diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index c4a214f..795826b 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -3,6 +3,7 @@ from __future__ import absolute_import from six.moves import range from functools import reduce + __author__ = 'Austin Taylor' from .base.config import vwConfig @@ -24,7 +25,6 @@ import socket class vulnWhispererBase(object): - CONFIG_SECTION = None def __init__( @@ -38,13 +38,13 @@ class vulnWhispererBase(object): password=None, section=None, develop=False, - ): + ): self.logger = logging.getLogger('vulnWhispererBase') if debug: self.logger.setLevel(logging.DEBUG) if self.CONFIG_SECTION is None: - raise Exception('Implementing class must define CONFIG_SECTION') + raise Exception('Implementing class must define CONFIG_SECTION') self.exit_code = 0 self.db_name = db_name @@ -68,8 +68,6 @@ class vulnWhispererBase(object): self.db_path = self.config.get(self.CONFIG_SECTION, 'db_path') self.verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose') - - if self.db_name is not None: if self.db_path: self.database = os.path.join(self.db_path, @@ -91,7 +89,8 @@ class vulnWhispererBase(object): self.cur = self.conn.cursor() self.logger.info('Connected to database at {loc}'.format(loc=self.database)) except Exception as e: - self.logger.error('Could not connect to database at {loc}\nReason: {e} - Please ensure the path exist'.format( + self.logger.error( + 'Could not connect to database at {loc}\nReason: {e} - Please ensure the path exist'.format( e=e, loc=self.database)) else: @@ -124,7 +123,7 @@ class vulnWhispererBase(object): ' scan_name TEXT, scan_id INTEGER, last_modified DATE, filename TEXT,' ' download_time DATE, record_count INTEGER, source TEXT,' ' uuid TEXT, processed INTEGER, reported INTEGER)' - ) + ) self.conn.commit() def delete_table(self): @@ -150,11 +149,11 @@ class vulnWhispererBase(object): return data def record_insert(self, record): - #for backwards compatibility with older versions without "reported" field + # for backwards compatibility with older versions without "reported" field try: - #-1 to get the latest column, 1 to get the column name (old version would be "processed", new "reported") - #TODO delete backward compatibility check after some versions + # -1 to get the latest column, 1 to get the column name (old version would be "processed", new "reported") + # TODO delete backward compatibility check after some versions last_column_table = self.cur.execute('PRAGMA table_info(scan_history)').fetchall()[-1][1] if last_column_table == self.table_columns[-1]: self.cur.execute('insert into scan_history({table_columns}) values (?,?,?,?,?,?,?,?,?,?)'.format( @@ -169,8 +168,8 @@ class vulnWhispererBase(object): sys.exit(1) def set_latest_scan_reported(self, filename): - #the reason to use the filename instead of the source/scan_name is because the filename already belongs to - #that latest scan, and we maintain integrity making sure that it is the exact scan we checked + # the reason to use the filename instead of the source/scan_name is because the filename already belongs to + # that latest scan, and we maintain integrity making sure that it is the exact scan we checked try: self.cur.execute('UPDATE scan_history SET reported = 1 WHERE filename="{}";'.format(filename)) self.conn.commit() @@ -188,7 +187,8 @@ class vulnWhispererBase(object): """ try: self.conn.text_factory = str - self.cur.execute('SELECT uuid FROM scan_history where source = "{config_section}"'.format(config_section=self.CONFIG_SECTION)) + self.cur.execute('SELECT uuid FROM scan_history where source = "{config_section}"'.format( + config_section=self.CONFIG_SECTION)) results = frozenset([r[0] for r in self.cur.fetchall()]) except: results = [] @@ -211,18 +211,23 @@ class vulnWhispererBase(object): try: self.conn.text_factory = str - self.cur.execute('SELECT filename FROM scan_history WHERE source="{}" AND scan_name="{}" ORDER BY last_modified DESC LIMIT 1;'.format(source, scan_name)) - #should always return just one filename + self.cur.execute( + 'SELECT filename FROM scan_history WHERE source="{}" AND scan_name="{}" ORDER BY last_modified DESC LIMIT 1;'.format( + source, scan_name)) + # should always return just one filename results = [r[0] for r in self.cur.fetchall()][0] - #-1 to get the latest column, 1 to get the column name (old version would be "processed", new "reported") - #TODO delete backward compatibility check after some versions + # -1 to get the latest column, 1 to get the column name (old version would be "processed", new "reported") + # TODO delete backward compatibility check after some versions last_column_table = self.cur.execute('PRAGMA table_info(scan_history)').fetchall()[-1][1] if results and last_column_table == self.table_columns[-1]: - reported = self.cur.execute('SELECT reported FROM scan_history WHERE filename="{}"'.format(results)).fetchall() + reported = self.cur.execute( + 'SELECT reported FROM scan_history WHERE filename="{}"'.format(results)).fetchall() reported = reported[0][0] if reported: - self.logger.debug("Last downloaded scan from source {source} scan_name {scan_name} has already been reported".format(source=source, scan_name=scan_name)) + self.logger.debug( + "Last downloaded scan from source {source} scan_name {scan_name} has already been reported".format( + source=source, scan_name=scan_name)) except Exception as e: self.logger.error("Error when getting latest results from {}.{} : {}".format(source, scan_name, e)) @@ -251,14 +256,14 @@ class vulnWhispererBase(object): self.cur.execute("SELECT DISTINCT scan_name FROM scan_history WHERE source='{}';".format(source)) scan_names = [r[0] for r in self.cur.fetchall()] for scan in scan_names: - results.append('{}.{}'.format(source,scan)) + results.append('{}.{}'.format(source, scan)) except: scan_names = [] return results -class vulnWhispererNessus(vulnWhispererBase): +class vulnWhispererNessus(vulnWhispererBase): CONFIG_SECTION = None def __init__( @@ -272,7 +277,7 @@ class vulnWhispererNessus(vulnWhispererBase): password=None, profile='nessus' ): - self.CONFIG_SECTION=profile + self.CONFIG_SECTION = profile super(vulnWhispererNessus, self).__init__(config=config) @@ -294,8 +299,8 @@ class vulnWhispererNessus(vulnWhispererBase): 'trash') try: - self.access_key = self.config.get(self.CONFIG_SECTION,'access_key') - self.secret_key = self.config.get(self.CONFIG_SECTION,'secret_key') + self.access_key = self.config.get(self.CONFIG_SECTION, 'access_key') + self.secret_key = self.config.get(self.CONFIG_SECTION, 'secret_key') except: pass @@ -312,7 +317,7 @@ class vulnWhispererNessus(vulnWhispererBase): ) self.nessus_connect = True self.logger.info('Connected to {} on {host}:{port}'.format(self.CONFIG_SECTION, host=self.hostname, - port=str(self.nessus_port))) + port=str(self.nessus_port))) except Exception as e: self.logger.error('Exception: {}'.format(str(e))) raise Exception( @@ -323,9 +328,7 @@ class vulnWhispererNessus(vulnWhispererBase): except Exception as e: self.logger.error('Could not properly load your config!\nReason: {e}'.format(e=e)) return False - #sys.exit(1) - - + # sys.exit(1) def scan_count(self, scans, completed=False): """ @@ -368,7 +371,6 @@ class vulnWhispererNessus(vulnWhispererBase): scan_records = [s for s in scan_records if s['status'] == 'completed'] return scan_records - def whisper_nessus(self): if self.nessus_connect: scan_data = self.nessus.scans @@ -423,7 +425,8 @@ class vulnWhispererNessus(vulnWhispererBase): s['uuid'], ) - # TODO Create directory sync function which scans the directory for files that exist already and populates the database + # TODO Create directory sync function which scans the directory for files that exist already and + # populates the database folder_id = s['folder_id'] if self.CONFIG_SECTION == 'tenable': @@ -453,22 +456,26 @@ class vulnWhispererNessus(vulnWhispererBase): 0, ) self.record_insert(record_meta) - self.logger.info('File {filename} already exist! Updating database'.format(filename=relative_path_name)) + self.logger.info( + 'File {filename} already exist! Updating database'.format(filename=relative_path_name)) else: try: file_req = \ self.nessus.download_scan(scan_id=scan_id, history=history_id, - export_format='csv') + export_format='csv') except Exception as e: - self.logger.error('Could not download {} scan {}: {}'.format(self.CONFIG_SECTION, scan_id, str(e))) + self.logger.error( + 'Could not download {} scan {}: {}'.format(self.CONFIG_SECTION, scan_id, str(e))) self.exit_code += 1 continue clean_csv = \ pd.read_csv(io.StringIO(file_req.decode('utf-8'))) if len(clean_csv) > 2: - self.logger.info('Processing {}/{} for scan: {}'.format(scan_count, len(scan_list), scan_name.encode('utf8'))) - columns_to_cleanse = ['CVSS','CVE','Description','Synopsis','Solution','See Also','Plugin Output', 'MAC Address'] + self.logger.info('Processing {}/{} for scan: {}'.format(scan_count, len(scan_list), + scan_name.encode('utf8'))) + columns_to_cleanse = ['CVSS', 'CVE', 'Description', 'Synopsis', 'Solution', 'See Also', + 'Plugin Output', 'MAC Address'] for col in columns_to_cleanse: if col in clean_csv: @@ -489,7 +496,8 @@ class vulnWhispererNessus(vulnWhispererBase): ) self.record_insert(record_meta) self.logger.info('{filename} records written to {path} '.format(filename=clean_csv.shape[0], - path=file_name.encode('utf8'))) + path=file_name.encode( + 'utf8'))) else: record_meta = ( scan_name, @@ -504,66 +512,68 @@ class vulnWhispererNessus(vulnWhispererBase): 0, ) self.record_insert(record_meta) - self.logger.warn('{} has no host available... Updating database and skipping!'.format(file_name)) + self.logger.warn( + '{} has no host available... Updating database and skipping!'.format(file_name)) self.conn.close() self.logger.info('Scan aggregation complete! Connection to database closed.') else: - self.logger.error('Failed to use scanner at {host}:{port}'.format(host=self.hostname, port=self.nessus_port)) + self.logger.error( + 'Failed to use scanner at {host}:{port}'.format(host=self.hostname, port=self.nessus_port)) self.exit_code += 1 return self.exit_code class vulnWhispererQualys(vulnWhispererBase): - CONFIG_SECTION = 'qualys_web' COLUMN_MAPPING = {'Access Path': 'access_path', - 'Ajax Request': 'ajax_request', - 'Ajax Request ID': 'ajax_request_id', - 'Authentication': 'authentication', - 'CVSS Base': 'cvss', - 'CVSS Temporal': 'cvss_temporal', - 'CWE': 'cwe', - 'Category': 'category', - 'Content': 'content', - 'DescriptionSeverity': 'severity_description', - 'DescriptionCatSev': 'category_description', - 'Detection ID': 'detection_id', - 'Evidence #1': 'evidence_1', - 'First Time Detected': 'first_time_detected', - 'Form Entry Point': 'form_entry_point', - 'Function': 'function', - 'Groups': 'groups', - 'ID': 'id', - 'Ignore Comments': 'ignore_comments', - 'Ignore Date': 'ignore_date', - 'Ignore Reason': 'ignore_reason', - 'Ignore User': 'ignore_user', - 'Ignored': 'ignored', - 'Impact': 'impact', - 'Last Time Detected': 'last_time_detected', - 'Last Time Tested': 'last_time_tested', - 'Level': 'level', - 'OWASP': 'owasp', - 'Operating System': 'operating_system', - 'Owner': 'owner', - 'Param': 'param', - 'Payload #1': 'payload_1', - 'QID': 'plugin_id', - 'Request Headers #1': 'request_headers_1', - 'Request Method #1': 'request_method_1', - 'Request URL #1': 'request_url_1', - 'Response #1': 'response_1', - 'Scope': 'scope', - 'Severity': 'risk', - 'Severity Level': 'security_level', - 'Solution': 'solution', - 'Times Detected': 'times_detected', - 'Title': 'plugin_name', - 'URL': 'url', - 'Url': 'uri', - 'Vulnerability Category': 'vulnerability_category', - 'WASC': 'wasc', - 'Web Application Name': 'web_application_name'} + 'Ajax Request': 'ajax_request', + 'Ajax Request ID': 'ajax_request_id', + 'Authentication': 'authentication', + 'CVSS Base': 'cvss', + 'CVSS Temporal': 'cvss_temporal', + 'CWE': 'cwe', + 'Category': 'category', + 'Content': 'content', + 'DescriptionSeverity': 'severity_description', + 'DescriptionCatSev': 'category_description', + 'Detection ID': 'detection_id', + 'Evidence #1': 'evidence_1', + 'First Time Detected': 'first_time_detected', + 'Form Entry Point': 'form_entry_point', + 'Function': 'function', + 'Groups': 'groups', + 'ID': 'id', + 'Ignore Comments': 'ignore_comments', + 'Ignore Date': 'ignore_date', + 'Ignore Reason': 'ignore_reason', + 'Ignore User': 'ignore_user', + 'Ignored': 'ignored', + 'Impact': 'impact', + 'Last Time Detected': 'last_time_detected', + 'Last Time Tested': 'last_time_tested', + 'Level': 'level', + 'OWASP': 'owasp', + 'Operating System': 'operating_system', + 'Owner': 'owner', + 'Param': 'param', + 'Payload #1': 'payload_1', + 'QID': 'plugin_id', + 'Request Headers #1': 'request_headers_1', + 'Request Method #1': 'request_method_1', + 'Request URL #1': 'request_url_1', + 'Response #1': 'response_1', + 'Scope': 'scope', + 'Severity': 'risk', + 'Severity Level': 'security_level', + 'Solution': 'solution', + 'Times Detected': 'times_detected', + 'Title': 'plugin_name', + 'URL': 'url', + 'Url': 'uri', + 'Vulnerability Category': 'vulnerability_category', + 'WASC': 'wasc', + 'Web Application Name': 'web_application_name'} + def __init__( self, config=None, @@ -573,7 +583,7 @@ class vulnWhispererQualys(vulnWhispererBase): debug=False, username=None, password=None, - ): + ): super(vulnWhispererQualys, self).__init__(config=config) self.logger = logging.getLogger('vulnWhispererQualys') @@ -611,7 +621,7 @@ class vulnWhispererQualys(vulnWhispererBase): relative_path_name = self.path_check(report_name).encode('utf8') if os.path.isfile(relative_path_name): - #TODO Possibly make this optional to sync directories + # TODO Possibly make this optional to sync directories file_length = len(open(relative_path_name).readlines()) record_meta = ( scan_name, @@ -663,7 +673,7 @@ class vulnWhispererQualys(vulnWhispererBase): f.write('\n') elif output_format == 'csv': - vuln_ready.to_csv(relative_path_name, index=False, header=True) # add when timestamp occured + vuln_ready.to_csv(relative_path_name, index=False, header=True) # add when timestamp occured self.logger.info('Report written to {}'.format(report_name)) @@ -671,7 +681,8 @@ class vulnWhispererQualys(vulnWhispererBase): self.logger.info('Removing report {} from Qualys Database'.format(generated_report_id)) cleaning_up = self.qualys_scan.qw.delete_report(generated_report_id) os.remove(self.path_check(str(generated_report_id) + '.csv')) - self.logger.info('Deleted report from local disk: {}'.format(self.path_check(str(generated_report_id)))) + self.logger.info( + 'Deleted report from local disk: {}'.format(self.path_check(str(generated_report_id)))) else: self.logger.error('Could not process report ID: {}'.format(status)) @@ -679,7 +690,6 @@ class vulnWhispererQualys(vulnWhispererBase): self.logger.error('Could not process {}: {}'.format(report_id, str(e))) return vuln_ready - def identify_scans_to_process(self): if self.uuids: self.scans_to_process = self.latest_scans[~self.latest_scans['id'].isin(self.uuids)] @@ -687,7 +697,6 @@ class vulnWhispererQualys(vulnWhispererBase): self.scans_to_process = self.latest_scans self.logger.info('Identified {new} scans to be processed'.format(new=len(self.scans_to_process))) - def process_web_assets(self): counter = 0 self.identify_scans_to_process() @@ -768,7 +777,6 @@ class vulnWhispererOpenVAS(vulnWhispererBase): if report_id: self.logger.info('Processing report ID: {}'.format(report_id)) - scan_name = report_id.replace('-', '') report_name = 'openvas_scan_{scan_name}_{last_updated}.{extension}'.format(scan_name=scan_name, last_updated=launched_date, @@ -836,7 +844,8 @@ class vulnWhispererOpenVAS(vulnWhispererBase): for scan in self.scans_to_process.iterrows(): counter += 1 info = scan[1] - self.logger.info('Processing {}/{} - Report ID: {}'.format(counter, len(self.scans_to_process), info['report_ids'])) + self.logger.info( + 'Processing {}/{} - Report ID: {}'.format(counter, len(self.scans_to_process), info['report_ids'])) self.whisper_reports(report_id=info['report_ids'], launched_date=info['epoch']) self.logger.info('Processing complete') @@ -847,15 +856,14 @@ class vulnWhispererOpenVAS(vulnWhispererBase): class vulnWhispererQualysVuln(vulnWhispererBase): - CONFIG_SECTION = 'qualys_vuln' COLUMN_MAPPING = {'cvss_base': 'cvss', - 'cvss3_base': 'cvss3', - 'cve_id': 'cve', - 'os': 'operating_system', - 'qid': 'plugin_id', - 'severity': 'risk', - 'title': 'plugin_name'} + 'cvss3_base': 'cvss3', + 'cve_id': 'cve', + 'os': 'operating_system', + 'qid': 'plugin_id', + 'severity': 'risk', + 'title': 'plugin_name'} def __init__( self, @@ -866,7 +874,7 @@ class vulnWhispererQualysVuln(vulnWhispererBase): debug=False, username=None, password=None, - ): + ): super(vulnWhispererQualysVuln, self).__init__(config=config) self.logger = logging.getLogger('vulnWhispererQualysVuln') @@ -888,66 +896,65 @@ class vulnWhispererQualysVuln(vulnWhispererBase): output_format='json', cleanup=True): - if 'Z' in launched_date: - launched_date = self.qualys_scan.utils.iso_to_epoch(launched_date) - report_name = 'qualys_vuln_' + report_id.replace('/','_') \ - + '_{last_updated}'.format(last_updated=launched_date) \ - + '.json' + if 'Z' in launched_date: + launched_date = self.qualys_scan.utils.iso_to_epoch(launched_date) + report_name = 'qualys_vuln_' + report_id.replace('/', '_') \ + + '_{last_updated}'.format(last_updated=launched_date) \ + + '.json' - relative_path_name = self.path_check(report_name).encode('utf8') + relative_path_name = self.path_check(report_name).encode('utf8') - if os.path.isfile(relative_path_name): - #TODO Possibly make this optional to sync directories - file_length = len(open(relative_path_name).readlines()) - record_meta = ( - scan_name, - scan_reference, - launched_date, - report_name, - time.time(), - file_length, - self.CONFIG_SECTION, - report_id, - 1, - 0, - ) - self.record_insert(record_meta) - self.logger.info('File {filename} already exist! Updating database'.format(filename=relative_path_name)) + if os.path.isfile(relative_path_name): + # TODO Possibly make this optional to sync directories + file_length = len(open(relative_path_name).readlines()) + record_meta = ( + scan_name, + scan_reference, + launched_date, + report_name, + time.time(), + file_length, + self.CONFIG_SECTION, + report_id, + 1, + 0, + ) + self.record_insert(record_meta) + self.logger.info('File {filename} already exist! Updating database'.format(filename=relative_path_name)) - else: - try: - self.logger.info('Processing report ID: {}'.format(report_id)) - vuln_ready = self.qualys_scan.process_data(scan_id=report_id) - vuln_ready['scan_name'] = scan_name - vuln_ready['scan_reference'] = report_id - vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True) - except Exception as e: - self.logger.error('Could not process {}: {}'.format(report_id, str(e))) - self.exit_code += 1 - return self.exit_code + else: + try: + self.logger.info('Processing report ID: {}'.format(report_id)) + vuln_ready = self.qualys_scan.process_data(scan_id=report_id) + vuln_ready['scan_name'] = scan_name + vuln_ready['scan_reference'] = report_id + vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True) + except Exception as e: + self.logger.error('Could not process {}: {}'.format(report_id, str(e))) + self.exit_code += 1 + return self.exit_code - record_meta = ( - scan_name, - scan_reference, - launched_date, - report_name, - time.time(), - vuln_ready.shape[0], - self.CONFIG_SECTION, - report_id, - 1, - 0, - ) - self.record_insert(record_meta) + record_meta = ( + scan_name, + scan_reference, + launched_date, + report_name, + time.time(), + vuln_ready.shape[0], + self.CONFIG_SECTION, + report_id, + 1, + 0, + ) + self.record_insert(record_meta) - if output_format == 'json': - with open(relative_path_name, 'w') as f: - f.write(vuln_ready.to_json(orient='records', lines=True)) - f.write('\n') - - self.logger.info('Report written to {}'.format(report_name)) - return self.exit_code + if output_format == 'json': + with open(relative_path_name, 'w') as f: + f.write(vuln_ready.to_json(orient='records', lines=True)) + f.write('\n') + self.logger.info('Report written to {}'.format(report_name)) + return self.exit_code def identify_scans_to_process(self): self.latest_scans = self.qualys_scan.qw.get_all_scans() @@ -959,7 +966,6 @@ class vulnWhispererQualysVuln(vulnWhispererBase): self.scans_to_process = self.latest_scans self.logger.info('Identified {new} scans to be processed'.format(new=len(self.scans_to_process))) - def process_vuln_scans(self): counter = 0 self.identify_scans_to_process() @@ -969,9 +975,9 @@ class vulnWhispererQualysVuln(vulnWhispererBase): r = app[1] self.logger.info('Processing {}/{}'.format(counter, len(self.scans_to_process))) self.exit_code += self.whisper_reports(report_id=r['id'], - launched_date=r['date'], - scan_name=r['name'], - scan_reference=r['type']) + launched_date=r['date'], + scan_name=r['name'], + scan_reference=r['type']) else: self.logger.info('No new scans to process. Exiting...') self.conn.close() @@ -979,7 +985,6 @@ class vulnWhispererQualysVuln(vulnWhispererBase): class vulnWhispererJIRA(vulnWhispererBase): - CONFIG_SECTION = 'jira' def __init__( @@ -991,7 +996,7 @@ class vulnWhispererJIRA(vulnWhispererBase): debug=False, username=None, password=None, - ): + ): super(vulnWhispererJIRA, self).__init__(config=config) self.logger = logging.getLogger('vulnWhispererJira') if debug: @@ -1001,8 +1006,8 @@ class vulnWhispererJIRA(vulnWhispererBase): self.host_resolv_cache = {} self.host_no_resolv = [] self.no_resolv_by_team_dict = {} - #Save locally those assets without DNS entry for flag to system owners - self.no_resolv_fname="no_resolv.txt" + # Save locally those assets without DNS entry for flag to system owners + self.no_resolv_fname = "no_resolv.txt" if os.path.isfile(self.no_resolv_fname): with open(self.no_resolv_fname, "r") as json_file: self.no_resolv_by_team_dict = json.load(json_file) @@ -1013,9 +1018,9 @@ class vulnWhispererJIRA(vulnWhispererBase): self.logger.info('Attempting to connect to jira...') self.jira = \ JiraAPI(hostname=self.hostname, - username=self.username, - password=self.password, - path=self.config.get('jira','write_path')) + username=self.username, + password=self.password, + path=self.config.get('jira', 'write_path')) self.jira_connect = True self.logger.info('Connected to jira on {host}'.format(host=self.hostname)) except Exception as e: @@ -1024,24 +1029,25 @@ class vulnWhispererJIRA(vulnWhispererBase): 'Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format( config=self.config.config_in, e=e)) return False - #sys.exit(1) + # sys.exit(1) profiles = [] profiles = self.get_scan_profiles() if not self.config.exists_jira_profiles(profiles): self.config.update_jira_profiles(profiles) - self.logger.info("Jira profiles have been created in {config}, please fill the variables before rerunning the module.".format(config=self.config_path)) + self.logger.info( + "Jira profiles have been created in {config}, please fill the variables before rerunning the module.".format( + config=self.config_path)) sys.exit(0) - def get_env_variables(self, source, scan_name): # function returns an array with [jira_project, jira_components, datafile_path] - #Jira variables - jira_section = self.config.normalize_section("{}.{}".format(source,scan_name)) + # Jira variables + jira_section = self.config.normalize_section("{}.{}".format(source, scan_name)) - project = self.config.get(jira_section,'jira_project') + project = self.config.get(jira_section, 'jira_project') if project == "": self.logger.error('JIRA project is missing on the configuration file!') sys.exit(0) @@ -1051,35 +1057,39 @@ class vulnWhispererJIRA(vulnWhispererBase): self.logger.error("JIRA project '{project}' doesn't exist!".format(project=project)) sys.exit(0) - components = self.config.get(jira_section,'components').split(',') + components = self.config.get(jira_section, 'components').split(',') - #cleaning empty array from '' + # cleaning empty array from '' if not components[0]: components = [] - min_critical = self.config.get(jira_section,'min_critical_to_report') + min_critical = self.config.get(jira_section, 'min_critical_to_report') if not min_critical: self.logger.error('"min_critical_to_report" variable on config file is empty.') sys.exit(0) - #datafile path + # datafile path filename, reported = self.get_latest_results(source, scan_name) fullpath = "" # search data files under user specified directory - for root, dirnames, filenames in os.walk(vwConfig(self.config_path).get(source,'write_path')): + for root, dirnames, filenames in os.walk(vwConfig(self.config_path).get(source, 'write_path')): if filename in filenames: - fullpath = "{}/{}".format(root,filename) + fullpath = "{}/{}".format(root, filename) if reported: - self.logger.warn('Last Scan of "{scan_name}" for source "{source}" has already been reported; will be skipped.'.format(scan_name=scan_name, source=source)) + self.logger.warn( + 'Last Scan of "{scan_name}" for source "{source}" has already been reported; will be skipped.'.format( + scan_name=scan_name, source=source)) return [False] * 5 if not fullpath: - self.logger.error('Scan of "{scan_name}" for source "{source}" has not been found. Please check that the scanner data files are in place.'.format(scan_name=scan_name, source=source)) + self.logger.error( + 'Scan of "{scan_name}" for source "{source}" has not been found. Please check that the scanner data files are in place.'.format( + scan_name=scan_name, source=source)) sys.exit(1) - dns_resolv = self.config.get('jira','dns_resolv') + dns_resolv = self.config.get('jira', 'dns_resolv') if dns_resolv in ('False', 'false', ''): dns_resolv = False elif dns_resolv in ('True', 'true'): @@ -1090,36 +1100,36 @@ class vulnWhispererJIRA(vulnWhispererBase): return project, components, fullpath, min_critical, dns_resolv - def parse_nessus_vulnerabilities(self, fullpath, source, scan_name, min_critical): vulnerabilities = [] # we need to parse the CSV risks = ['none', 'low', 'medium', 'high', 'critical'] - min_risk = int([i for i,x in enumerate(risks) if x == min_critical][0]) + min_risk = int([i for i, x in enumerate(risks) if x == min_critical][0]) df = pd.read_csv(fullpath, delimiter=',') - #nessus fields we want - ['Host','Protocol','Port', 'Name', 'Synopsis', 'Description', 'Solution', 'See Also'] + # nessus fields we want - ['Host','Protocol','Port', 'Name', 'Synopsis', 'Description', 'Solution', 'See Also'] for index in range(len(df)): # filtering vulnerabilities by criticality, discarding low risk - to_report = int([i for i,x in enumerate(risks) if x == df.loc[index]['Risk'].lower()][0]) + to_report = int([i for i, x in enumerate(risks) if x == df.loc[index]['Risk'].lower()][0]) if to_report < min_risk: continue if not vulnerabilities or df.loc[index]['Name'] not in [entry['title'] for entry in vulnerabilities]: vuln = {} - #vulnerabilities should have all the info for creating all JIRA labels + # vulnerabilities should have all the info for creating all JIRA labels vuln['source'] = source vuln['scan_name'] = scan_name - #vulnerability variables + # vulnerability variables vuln['title'] = df.loc[index]['Name'] - vuln['diagnosis'] = df.loc[index]['Synopsis'].replace('\\n',' ') - vuln['consequence'] = df.loc[index]['Description'].replace('\\n',' ') - vuln['solution'] = df.loc[index]['Solution'].replace('\\n',' ') + vuln['diagnosis'] = df.loc[index]['Synopsis'].replace('\\n', ' ') + vuln['consequence'] = df.loc[index]['Description'].replace('\\n', ' ') + vuln['solution'] = df.loc[index]['Solution'].replace('\\n', ' ') vuln['ips'] = [] - vuln['ips'].append("{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], df.loc[index]['Port'])) + vuln['ips'].append( + "{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], df.loc[index]['Port'])) vuln['risk'] = df.loc[index]['Risk'].lower() # Nessus "nan" value gets automatically casted to float by python @@ -1133,51 +1143,54 @@ class vulnWhispererJIRA(vulnWhispererBase): # grouping assets by vulnerability to open on single ticket, as each asset has its own nessus entry for vuln in vulnerabilities: if vuln['title'] == df.loc[index]['Name']: - vuln['ips'].append("{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], df.loc[index]['Port'])) + vuln['ips'].append("{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], + df.loc[index]['Port'])) return vulnerabilities - def parse_qualys_vuln_vulnerabilities(self, fullpath, source, scan_name, min_critical, dns_resolv = False): - #parsing of the qualys vulnerabilities schema - #parse json + def parse_qualys_vuln_vulnerabilities(self, fullpath, source, scan_name, min_critical, dns_resolv=False): + # parsing of the qualys vulnerabilities schema + # parse json vulnerabilities = [] risks = ['info', 'low', 'medium', 'high', 'critical'] # +1 as array is 0-4, but score is 1-5 - min_risk = int([i for i,x in enumerate(risks) if x == min_critical][0])+1 + min_risk = int([i for i, x in enumerate(risks) if x == min_critical][0]) + 1 try: - data=[json.loads(line) for line in open(fullpath).readlines()] + data = [json.loads(line) for line in open(fullpath).readlines()] except Exception as e: self.logger.warn("Scan has no vulnerabilities, skipping.") return vulnerabilities - #qualys fields we want - [] + # qualys fields we want - [] for index in range(len(data)): if int(data[index]['risk']) < min_risk: continue elif data[index]['type'] == 'Practice' or data[index]['type'] == 'Ig': - self.logger.debug("Vulnerability '{vuln}' ignored, as it is 'Practice/Potential', not verified.".format(vuln=data[index]['plugin_name'])) + self.logger.debug("Vulnerability '{vuln}' ignored, as it is 'Practice/Potential', not verified.".format( + vuln=data[index]['plugin_name'])) continue if not vulnerabilities or data[index]['plugin_name'] not in [entry['title'] for entry in vulnerabilities]: vuln = {} - #vulnerabilities should have all the info for creating all JIRA labels + # vulnerabilities should have all the info for creating all JIRA labels vuln['source'] = source vuln['scan_name'] = scan_name - #vulnerability variables + # vulnerability variables vuln['title'] = data[index]['plugin_name'] - vuln['diagnosis'] = data[index]['threat'].replace('\\n',' ') - vuln['consequence'] = data[index]['impact'].replace('\\n',' ') - vuln['solution'] = data[index]['solution'].replace('\\n',' ') + vuln['diagnosis'] = data[index]['threat'].replace('\\n', ' ') + vuln['consequence'] = data[index]['impact'].replace('\\n', ' ') + vuln['solution'] = data[index]['solution'].replace('\\n', ' ') vuln['ips'] = [] - #TODO ADDED DNS RESOLUTION FROM QUALYS! \n SEPARATORS INSTEAD OF \\n! + # TODO ADDED DNS RESOLUTION FROM QUALYS! \n SEPARATORS INSTEAD OF \\n! - vuln['ips'].append("{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv))) + vuln['ips'].append( + "{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv))) - #different risk system than Nessus! - vuln['risk'] = risks[int(data[index]['risk'])-1] + # different risk system than Nessus! + vuln['risk'] = risks[int(data[index]['risk']) - 1] # Nessus "nan" value gets automatically casted to float by python if not (type(data[index]['vendor_reference']) is float or data[index]['vendor_reference'] == None): @@ -1189,7 +1202,8 @@ class vulnWhispererJIRA(vulnWhispererBase): # grouping assets by vulnerability to open on single ticket, as each asset has its own nessus entry for vuln in vulnerabilities: if vuln['title'] == data[index]['plugin_name']: - vuln['ips'].append("{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv))) + vuln['ips'].append( + "{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv))) return vulnerabilities @@ -1224,50 +1238,55 @@ class vulnWhispererJIRA(vulnWhispererBase): return values def parse_vulnerabilities(self, fullpath, source, scan_name, min_critical): - #TODO: SINGLE LOCAL SAVE FORMAT FOR ALL SCANNERS - #JIRA standard vuln format - ['source', 'scan_name', 'title', 'diagnosis', 'consequence', 'solution', 'ips', 'references'] + # TODO: SINGLE LOCAL SAVE FORMAT FOR ALL SCANNERS + # JIRA standard vuln format - ['source', 'scan_name', 'title', 'diagnosis', 'consequence', 'solution', 'ips', 'references'] return 0 - def jira_sync(self, source, scan_name): - self.logger.info("Jira Sync triggered for source '{source}' and scan '{scan_name}'".format(source=source, scan_name=scan_name)) + self.logger.info("Jira Sync triggered for source '{source}' and scan '{scan_name}'".format(source=source, + scan_name=scan_name)) project, components, fullpath, min_critical, dns_resolv = self.get_env_variables(source, scan_name) if not project: - self.logger.debug("Skipping scan for source '{source}' and scan '{scan_name}': vulnerabilities have already been reported.".format(source=source, scan_name=scan_name)) + self.logger.debug( + "Skipping scan for source '{source}' and scan '{scan_name}': vulnerabilities have already been reported.".format( + source=source, scan_name=scan_name)) return False vulnerabilities = [] - #***Nessus parsing*** + # ***Nessus parsing*** if source == "nessus": vulnerabilities = self.parse_nessus_vulnerabilities(fullpath, source, scan_name, min_critical) - #***Qualys VM parsing*** + # ***Qualys VM parsing*** if source == "qualys_vuln": - vulnerabilities = self.parse_qualys_vuln_vulnerabilities(fullpath, source, scan_name, min_critical, dns_resolv) + vulnerabilities = self.parse_qualys_vuln_vulnerabilities(fullpath, source, scan_name, min_critical, + dns_resolv) - #***JIRA sync*** + # ***JIRA sync*** if vulnerabilities: self.logger.info('{source} data has been successfuly parsed'.format(source=source.upper())) self.logger.info('Starting JIRA sync') self.jira.sync(vulnerabilities, project, components) else: - self.logger.info("[{source}.{scan_name}] No vulnerabilities or vulnerabilities not parsed.".format(source=source, scan_name=scan_name)) + self.logger.info( + "[{source}.{scan_name}] No vulnerabilities or vulnerabilities not parsed.".format(source=source, + scan_name=scan_name)) self.set_latest_scan_reported(fullpath.split("/")[-1]) return False - #writing to file those assets without DNS resolution - #if its not empty + # writing to file those assets without DNS resolution + # if its not empty if self.host_no_resolv: - #we will replace old list of non resolved for the new one or create if it doesn't exist already + # we will replace old list of non resolved for the new one or create if it doesn't exist already self.no_resolv_by_team_dict[scan_name] = self.host_no_resolv - with open(self.no_resolv_fname, 'w') as outfile: - json.dump(self.no_resolv_by_team_dict, outfile) - + with open(self.no_resolv_fname, 'w') as outfile: + json.dump(self.no_resolv_by_team_dict, outfile) + self.set_latest_scan_reported(fullpath.split("/")[-1]) return True @@ -1279,10 +1298,13 @@ class vulnWhispererJIRA(vulnWhispererBase): try: self.jira_sync(self.config.get(scan, 'source'), self.config.get(scan, 'scan_name')) except Exception as e: - self.logger.error("VulnWhisperer wasn't able to report the vulnerabilities from the '{}'s source".format(self.config.get(scan, 'source'))) + self.logger.error( + "VulnWhisperer wasn't able to report the vulnerabilities from the '{}'s source".format( + self.config.get(scan, 'source'))) return True return False + class vulnWhisperer(object): def __init__(self, @@ -1306,7 +1328,6 @@ class vulnWhisperer(object): self.scanname = scanname self.exit_code = 0 - def whisper_vulnerabilities(self): if self.profile == 'nessus': @@ -1337,7 +1358,7 @@ class vulnWhisperer(object): self.exit_code += vw.process_vuln_scans() elif self.profile == 'jira': - #first we check config fields are created, otherwise we create them + # first we check config fields are created, otherwise we create them vw = vulnWhispererJIRA(config=self.config) if vw: if not (self.source and self.scanname):