diff --git a/configs/frameworks_example.ini b/configs/frameworks_example.ini index 20410cb..61a8af5 100755 --- a/configs/frameworks_example.ini +++ b/configs/frameworks_example.ini @@ -2,6 +2,8 @@ enabled=true hostname=localhost port=8834 +access_key= +secret_key= username=nessus_username password=nessus_password write_path=/opt/VulnWhisperer/data/nessus/ @@ -13,6 +15,8 @@ verbose=true enabled=true hostname=cloud.tenable.com port=443 +access_key= +secret_key= username=tenable.io_username password=tenable.io_password write_path=/opt/VulnWhisperer/data/tenable/ diff --git a/configs/test.ini b/configs/test.ini index b5f04b5..122c46c 100755 --- a/configs/test.ini +++ b/configs/test.ini @@ -2,6 +2,8 @@ enabled=true hostname=nessus port=443 +access_key= +secret_key= username=nessus_username password=nessus_password write_path=/opt/VulnWhisperer/data/nessus/ @@ -13,6 +15,8 @@ verbose=true enabled=true hostname=tenable port=443 +access_key= +secret_key= username=tenable.io_username password=tenable.io_password write_path=/opt/VulnWhisperer/data/tenable/ diff --git a/vulnwhisp/base/config.py b/vulnwhisp/base/config.py index e8490d6..630b21b 100644 --- a/vulnwhisp/base/config.py +++ b/vulnwhisp/base/config.py @@ -31,7 +31,7 @@ class vwConfig(object): for section in self.config.sections(): try: if self.get(section, attribute) in check: - sections.append(section) + sections.append(section) except: self.logger.warn("Section {} has no option '{}'".format(section, attribute)) return sections @@ -45,7 +45,7 @@ class vwConfig(object): return True def update_jira_profiles(self, profiles): - # create JIRA profiles in the ini config file + # create JIRA profiles in the ini config file self.logger.debug('Updating Jira profiles: {}'.format(str(profiles))) for profile in profiles: @@ -67,7 +67,7 @@ class vwConfig(object): self.config.set(section_name, 'min_critical_to_report', 'high') self.config.set(section_name, '; automatically report, boolean value ') self.config.set(section_name, 'autoreport', 'false') - + # TODO: try/catch this # writing changes back to file with open(self.config_in, 'w') as configfile: diff --git a/vulnwhisp/frameworks/nessus.py b/vulnwhisp/frameworks/nessus.py index 23c67d6..ed4a6e3 100755 --- a/vulnwhisp/frameworks/nessus.py +++ b/vulnwhisp/frameworks/nessus.py @@ -24,15 +24,19 @@ class NessusAPI(object): EXPORT_STATUS = EXPORT + '/{file_id}/status' EXPORT_HISTORY = EXPORT + '?history_id={history_id}' - def __init__(self, hostname=None, port=None, username=None, password=None, verbose=True): + def __init__(self, hostname=None, port=None, username=None, password=None, verbose=True, profile=None, access_key=None, secret_key=None): self.logger = logging.getLogger('NessusAPI') if verbose: self.logger.setLevel(logging.DEBUG) - if username is None or password is None: - raise Exception('ERROR: Missing username or password.') + if not all((username, password)) and not all((access_key, secret_key)): + raise Exception('ERROR: Missing username, password or API keys.') + self.profile = profile self.user = username self.password = password + self.api_keys = False + self.access_key = access_key + self.secret_key = secret_key self.base = 'https://{hostname}:{port}'.format(hostname=hostname, port=port) self.verbose = verbose @@ -52,7 +56,13 @@ class NessusAPI(object): 'X-Cookie': None } - self.login() + if all((self.access_key, self.secret_key)): + self.logger.debug('Using {} API keys'.format(self.profile)) + self.api_keys = True + self.session.headers['X-ApiKeys'] = 'accessKey={}; secretKey={}'.format(self.access_key, self.secret_key) + else: + self.login() + self.scans = self.get_scans() self.scan_ids = self.get_scan_ids() @@ -67,7 +77,7 @@ class NessusAPI(object): def request(self, url, data=None, headers=None, method='POST', download=False, json_output=False): timeout = 0 success = False - + method = method.lower() url = self.base + url self.logger.debug('Requesting to url {}'.format(url)) @@ -78,8 +88,10 @@ class NessusAPI(object): if url == self.base + self.SESSION: break try: - self.login() timeout += 1 + if self.api_keys: + continue + self.login() self.logger.info('Token refreshed') except Exception as e: self.logger.error('Could not refresh token\nReason: {}'.format(str(e))) @@ -114,7 +126,7 @@ class NessusAPI(object): data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json_output=True) return data['history'] - def download_scan(self, scan_id=None, history=None, export_format="", profile=""): + def download_scan(self, scan_id=None, history=None, export_format=""): running = True counter = 0 @@ -127,7 +139,8 @@ class NessusAPI(object): req = self.request(query, data=json.dumps(data), method='POST', json_output=True) try: file_id = req['file'] - token_id = req['token'] if 'token' in req else req['temp_token'] + if self.profile == 'nessus': + token_id = req['token'] if 'token' in req else req['temp_token'] except Exception as e: self.logger.error('{}'.format(str(e))) self.logger.info('Download for file id {}'.format(str(file_id))) @@ -143,7 +156,7 @@ class NessusAPI(object): if counter % 60 == 0: self.logger.info("Completed: {}".format(counter)) self.logger.info("Done: {}".format(counter)) - if profile == 'tenable': + if self.profile == 'tenable' or self.api_keys: content = self.request(self.EXPORT_FILE_DOWNLOAD.format(scan_id=scan_id, file_id=file_id), method='GET', download=True) else: content = self.request(self.EXPORT_TOKEN_DOWNLOAD.format(token_id=token_id), method='GET', download=True) diff --git a/vulnwhisp/frameworks/qualys_web.py b/vulnwhisp/frameworks/qualys_web.py index 4e50c5f..7d6b122 100644 --- a/vulnwhisp/frameworks/qualys_web.py +++ b/vulnwhisp/frameworks/qualys_web.py @@ -428,7 +428,7 @@ class qualysScanReport: merged_df = merged_df.drop(['QID_y', 'QID_x'], axis=1) merged_df = merged_df.rename(columns={'Id': 'QID'}) - + merged_df = merged_df.assign(**df_dict['SCAN_META'].to_dict(orient='records')[0]) merged_df = pd.merge(merged_df, df_dict['CATEGORY_HEADER'], how='left', left_on=['Category', 'Severity Level'], diff --git a/vulnwhisp/test/mock.py b/vulnwhisp/test/mock.py index 5d48729..ac32085 100644 --- a/vulnwhisp/test/mock.py +++ b/vulnwhisp/test/mock.py @@ -35,13 +35,13 @@ class mockAPI(object): elif 'fetch' in request.parsed_body['action']: try: response_body = open('{}/{}'.format( - self.qualys_vuln_path, + self.qualys_vuln_path, request.parsed_body['scan_ref'][0].replace('/', '_')) ).read() except: # Can't find the file, just send an empty response response_body = '' - return [200, response_headers, response_body] + return [200, response_headers, response_body] def create_nessus_resource(self, framework): for filename in self.get_files('{}/{}'.format(self.mock_dir, framework)): @@ -60,7 +60,7 @@ class mockAPI(object): httpretty.GET, 'https://{}:443/{}'.format(framework, 'msp/about.php'), body='') - + self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, 'POST', 'api/2.0/fo/scan')) httpretty.register_uri( httpretty.POST, 'https://{}:443/{}'.format(framework, 'api/2.0/fo/scan/'), diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index 0630724..a579fe4 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -55,8 +55,12 @@ class vulnWhispererBase(object): except: self.enabled = False self.hostname = self.config.get(self.CONFIG_SECTION, 'hostname') - self.username = self.config.get(self.CONFIG_SECTION, 'username') - self.password = self.config.get(self.CONFIG_SECTION, 'password') + try: + self.username = self.config.get(self.CONFIG_SECTION, 'username') + self.password = self.config.get(self.CONFIG_SECTION, 'password') + except: + self.username = None + self.password = None self.write_path = self.config.get(self.CONFIG_SECTION, 'write_path') self.db_path = self.config.get(self.CONFIG_SECTION, 'db_path') self.verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose') @@ -144,7 +148,7 @@ class vulnWhispererBase(object): def record_insert(self, record): #for backwards compatibility with older versions without "reported" field - + try: #-1 to get the latest column, 1 to get the column name (old version would be "processed", new "reported") #TODO delete backward compatibility check after some versions @@ -171,7 +175,7 @@ class vulnWhispererBase(object): return True except Exception as e: self.logger.error('Failed while setting scan with file {} as processed'.format(filename)) - + return False def retrieve_uuids(self): @@ -200,7 +204,7 @@ class vulnWhispererBase(object): def get_latest_results(self, source, scan_name): processed = 0 results = [] - + try: self.conn.text_factory = str self.cur.execute('SELECT filename FROM scan_history WHERE source="{}" AND scan_name="{}" ORDER BY last_modified DESC LIMIT 1;'.format(source, scan_name)) @@ -219,10 +223,10 @@ class vulnWhispererBase(object): except Exception as e: self.logger.error("Error when getting latest results from {}.{} : {}".format(source, scan_name, e)) return results, reported - + def get_scan_profiles(self): # Returns a list of source.scan_name elements from the database - + # we get the list of sources try: self.conn.text_factory = str @@ -231,7 +235,7 @@ class vulnWhispererBase(object): except: sources = [] self.logger.error("Process failed at executing 'SELECT DISTINCT source FROM scan_history;'") - + results = [] # we get the list of scans within each source @@ -274,6 +278,8 @@ class vulnWhispererNessus(vulnWhispererBase): self.develop = True self.purge = purge + self.access_key = None + self.secret_key = None if config is not None: try: @@ -283,19 +289,30 @@ class vulnWhispererNessus(vulnWhispererBase): 'trash') try: - self.logger.info('Attempting to connect to nessus...') + self.access_key = self.config.get(self.CONFIG_SECTION,'access_key') + self.secret_key = self.config.get(self.CONFIG_SECTION,'secret_key') + except: + pass + + try: + self.logger.info('Attempting to connect to {}...'.format(self.CONFIG_SECTION)) self.nessus = \ NessusAPI(hostname=self.hostname, port=self.nessus_port, username=self.username, - password=self.password) + password=self.password, + profile=self.CONFIG_SECTION, + access_key=self.access_key, + secret_key=self.secret_key + ) self.nessus_connect = True - self.logger.info('Connected to nessus on {host}:{port}'.format(host=self.hostname, + self.logger.info('Connected to {} on {host}:{port}'.format(self.CONFIG_SECTION, host=self.hostname, port=str(self.nessus_port))) except Exception as e: self.logger.error('Exception: {}'.format(str(e))) raise Exception( - 'Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format( + 'Could not connect to {} -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format( + self.CONFIG_SECTION, config=self.config.config_in, e=e)) except Exception as e: @@ -435,7 +452,7 @@ class vulnWhispererNessus(vulnWhispererBase): try: file_req = \ self.nessus.download_scan(scan_id=scan_id, history=history_id, - export_format='csv', profile=self.CONFIG_SECTION) + export_format='csv') except Exception as e: self.logger.error('Could not download {} scan {}: {}'.format(self.CONFIG_SECTION, scan_id, str(e))) self.exit_code += 1 @@ -556,7 +573,7 @@ class vulnWhispererQualys(vulnWhispererBase): self.logger = logging.getLogger('vulnWhispererQualys') if debug: self.logger.setLevel(logging.DEBUG) - + self.qualys_scan = qualysScanReport(config=config) self.latest_scans = self.qualys_scan.qw.get_all_scans() self.directory_check() @@ -643,8 +660,7 @@ class vulnWhispererQualys(vulnWhispererBase): if cleanup: self.logger.info('Removing report {} from Qualys Database'.format(generated_report_id)) - cleaning_up = \ - self.qualys_scan.qw.delete_report(generated_report_id) + cleaning_up = self.qualys_scan.qw.delete_report(generated_report_id) os.remove(self.path_check(str(generated_report_id) + '.csv')) self.logger.info('Deleted report from local disk: {}'.format(self.path_check(str(generated_report_id)))) else: @@ -838,7 +854,7 @@ class vulnWhispererQualysVuln(vulnWhispererBase): username=None, password=None, ): - + super(vulnWhispererQualysVuln, self).__init__(config=config) self.logger = logging.getLogger('vulnWhispererQualysVuln') if debug: @@ -967,8 +983,8 @@ class vulnWhispererJIRA(vulnWhispererBase): self.config_path = config self.config = vwConfig(config) self.host_resolv_cache = {} - self.directory_check() - + self.directory_check() + if config is not None: try: self.logger.info('Attempting to connect to jira...') @@ -985,16 +1001,16 @@ class vulnWhispererJIRA(vulnWhispererBase): 'Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format( config=self.config.config_in, e=e)) sys.exit(1) - + profiles = [] profiles = self.get_scan_profiles() - + if not self.config.exists_jira_profiles(profiles): self.config.update_jira_profiles(profiles) self.logger.info("Jira profiles have been created in {config}, please fill the variables before rerunning the module.".format(config=self.config_path)) sys.exit(0) - - + + def get_env_variables(self, source, scan_name): # function returns an array with [jira_project, jira_components, datafile_path] @@ -1005,32 +1021,32 @@ class vulnWhispererJIRA(vulnWhispererBase): if project == "": self.logger.error('JIRA project is missing on the configuration file!') sys.exit(0) - + # check that project actually exists if not self.jira.project_exists(project): self.logger.error("JIRA project '{project}' doesn't exist!".format(project=project)) sys.exit(0) - + components = self.config.get(jira_section,'components').split(',') - + #cleaning empty array from '' if not components[0]: components = [] - + min_critical = self.config.get(jira_section,'min_critical_to_report') if not min_critical: self.logger.error('"min_critical_to_report" variable on config file is empty.') sys.exit(0) - + #datafile path filename, reported = self.get_latest_results(source, scan_name) fullpath = "" - + # search data files under user specified directory for root, dirnames, filenames in os.walk(vwConfig(self.config_path).get(source,'write_path')): if filename in filenames: fullpath = "{}/{}".format(root,filename) - + if reported: self.logger.warn('Last Scan of "{scan_name}" for source "{source}" has already been reported; will be skipped.'.format(scan_name=scan_name, source=source)) return [False] * 5 @@ -1038,7 +1054,7 @@ class vulnWhispererJIRA(vulnWhispererBase): if not fullpath: self.logger.error('Scan of "{scan_name}" for source "{source}" has not been found. Please check that the scanner data files are in place.'.format(scan_name=scan_name, source=source)) sys.exit(1) - + dns_resolv = self.config.get('jira','dns_resolv') if dns_resolv in ('False', 'false', ''): dns_resolv = False @@ -1052,22 +1068,22 @@ class vulnWhispererJIRA(vulnWhispererBase): def parse_nessus_vulnerabilities(self, fullpath, source, scan_name, min_critical): - + vulnerabilities = [] # we need to parse the CSV - risks = ['none', 'low', 'medium', 'high', 'critical'] + risks = ['none', 'low', 'medium', 'high', 'critical'] min_risk = int([i for i,x in enumerate(risks) if x == min_critical][0]) df = pd.read_csv(fullpath, delimiter=',') - + #nessus fields we want - ['Host','Protocol','Port', 'Name', 'Synopsis', 'Description', 'Solution', 'See Also'] for index in range(len(df)): # filtering vulnerabilities by criticality, discarding low risk to_report = int([i for i,x in enumerate(risks) if x == df.loc[index]['Risk'].lower()][0]) if to_report < min_risk: continue - + if not vulnerabilities or df.loc[index]['Name'] not in [entry['title'] for entry in vulnerabilities]: vuln = {} #vulnerabilities should have all the info for creating all JIRA labels @@ -1081,7 +1097,7 @@ class vulnWhispererJIRA(vulnWhispererBase): vuln['ips'] = [] vuln['ips'].append("{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], df.loc[index]['Port'])) vuln['risk'] = df.loc[index]['Risk'].lower() - + # Nessus "nan" value gets automatically casted to float by python if not (type(df.loc[index]['See Also']) is float): vuln['references'] = df.loc[index]['See Also'].split("\\n") @@ -1094,24 +1110,24 @@ class vulnWhispererJIRA(vulnWhispererBase): for vuln in vulnerabilities: if vuln['title'] == df.loc[index]['Name']: vuln['ips'].append("{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], df.loc[index]['Port'])) - + return vulnerabilities - + def parse_qualys_vuln_vulnerabilities(self, fullpath, source, scan_name, min_critical, dns_resolv = False): #parsing of the qualys vulnerabilities schema #parse json vulnerabilities = [] - risks = ['info', 'low', 'medium', 'high', 'critical'] + risks = ['info', 'low', 'medium', 'high', 'critical'] # +1 as array is 0-4, but score is 1-5 min_risk = int([i for i,x in enumerate(risks) if x == min_critical][0])+1 - + try: - data=[json.loads(line) for line in open(fullpath).readlines()] + data=[json.loads(line) for line in open(fullpath).readlines()] except Exception as e: self.logger.warn("Scan has no vulnerabilities, skipping.") return vulnerabilities - + #qualys fields we want - [] for index in range(len(data)): if int(data[index]['risk']) < min_risk: @@ -1120,7 +1136,7 @@ class vulnWhispererJIRA(vulnWhispererBase): elif data[index]['type'] == 'Practice' or data[index]['type'] == 'Ig': self.logger.debug("Vulnerability '{vuln}' ignored, as it is 'Practice/Potential', not verified.".format(vuln=data[index]['plugin_name'])) continue - + if not vulnerabilities or data[index]['plugin_name'] not in [entry['title'] for entry in vulnerabilities]: vuln = {} #vulnerabilities should have all the info for creating all JIRA labels @@ -1133,12 +1149,12 @@ class vulnWhispererJIRA(vulnWhispererBase): vuln['solution'] = data[index]['solution'].replace('\\n',' ') vuln['ips'] = [] #TODO ADDED DNS RESOLUTION FROM QUALYS! \n SEPARATORS INSTEAD OF \\n! - + vuln['ips'].append("{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv))) #different risk system than Nessus! vuln['risk'] = risks[int(data[index]['risk'])-1] - + # Nessus "nan" value gets automatically casted to float by python if not (type(data[index]['vendor_reference']) is float or data[index]['vendor_reference'] == None): vuln['references'] = data[index]['vendor_reference'].split("\\n") @@ -1156,8 +1172,8 @@ class vulnWhispererJIRA(vulnWhispererBase): def get_asset_fields(self, vuln, dns_resolv): values = {} values['ip'] = vuln['ip'] - values['protocol'] = vuln['protocol'] - values['port'] = vuln['port'] + values['protocol'] = vuln['protocol'] + values['port'] = vuln['port'] values['dns'] = '' if dns_resolv: if vuln['dns']: @@ -1207,12 +1223,12 @@ class vulnWhispererJIRA(vulnWhispererBase): #***Qualys VM parsing*** if source == "qualys_vuln": vulnerabilities = self.parse_qualys_vuln_vulnerabilities(fullpath, source, scan_name, min_critical, dns_resolv) - + #***JIRA sync*** if vulnerabilities: self.logger.info('{source} data has been successfuly parsed'.format(source=source.upper())) self.logger.info('Starting JIRA sync') - + self.jira.sync(vulnerabilities, project, components) else: self.logger.info("[{source}.{scan_name}] No vulnerabilities or vulnerabilities not parsed.".format(source=source, scan_name=scan_name)) @@ -1259,9 +1275,6 @@ class vulnWhisperer(object): if self.profile == 'nessus': vw = vulnWhispererNessus(config=self.config, - username=self.username, - password=self.password, - verbose=self.verbose, profile=self.profile) self.exit_code += vw.whisper_nessus() @@ -1275,16 +1288,13 @@ class vulnWhisperer(object): elif self.profile == 'tenable': vw = vulnWhispererNessus(config=self.config, - username=self.username, - password=self.password, - verbose=self.verbose, profile=self.profile) self.exit_code += vw.whisper_nessus() elif self.profile == 'qualys_vuln': vw = vulnWhispererQualysVuln(config=self.config) self.exit_code += vw.process_vuln_scans() - + elif self.profile == 'jira': #first we check config fields are created, otherwise we create them vw = vulnWhispererJIRA(config=self.config)