diff --git a/vulnwhisp/frameworks/qualys_vuln.py b/vulnwhisp/frameworks/qualys_vuln.py index 69cddfa..dace770 100644 --- a/vulnwhisp/frameworks/qualys_vuln.py +++ b/vulnwhisp/frameworks/qualys_vuln.py @@ -11,6 +11,7 @@ import pandas as pd import qualysapi + class qualysWhisperAPI(object): SCANS = 'api/2.0/fo/scan' @@ -78,6 +79,15 @@ class qualysUtils: class qualysVulnScan: + COLUMN_MAPPING = { + 'cve_id': 'cve', + 'cvss_base': 'cvss', + 'cvss3_base': 'cvss3', + 'ip_status': 'state', + 'os': 'operating_system', + 'qid': 'plugin_id' + } + def __init__( self, config=None, @@ -122,3 +132,76 @@ class qualysVulnScan: return scan_report return scan_report + + def normalise(self, dataframe): + self.logger.debug('Normalising data') + self.map_fields(dataframe) + self.transform_values(dataframe) + return dataframe + + def map_fields(self, dataframe): + self.logger.info('Mapping fields') + + # Map fields from COLUMN_MAPPING + fields = [x.lower() for x in dataframe.columns] + for field, replacement in self.COLUMN_MAPPING.iteritems(): + if field in fields: + self.logger.info('Renaming "{}" to "{}"'.format(field, replacement)) + fields[fields.index(field)] = replacement + + fields = [x.replace(' ', '_') for x in fields] + dataframe.columns = fields + + return dataframe + + def transform_values(self, dataframe): + self.logger.info('Transforming values') + + # upper/lowercase fields + self.logger.info('Changing case of fields') + dataframe['cve'] = dataframe['cve'].str.upper() + dataframe['protocol'] = dataframe['protocol'].str.lower() + + # Contruct the CVSS vector + dataframe['cvss_vector'] = '' + dataframe.loc[dataframe["cvss"].notnull(), "cvss_vector"] = ( + dataframe.loc[dataframe["cvss"].notnull(), "cvss"] + .str.split() + .apply(lambda x: x[1]) + .str.replace("(", "") + .str.replace(")", "") + ) + dataframe.loc[dataframe["cvss"].notnull(), "cvss"] = ( + dataframe.loc[dataframe["cvss"].notnull(), "cvss"] + .str.split() + .apply(lambda x: x[0]) + ) + dataframe['cvss_temporal_vector'] = '' + dataframe.loc[dataframe["cvss_temporal"].notnull(), "cvss_temporal_vector"] = ( + dataframe.loc[dataframe["cvss_temporal"].notnull(), "cvss_temporal"] + .str.split() + .apply(lambda x: x[1]) + .str.replace("(", "") + .str.replace(")", "") + ) + dataframe.loc[dataframe["cvss_temporal"].notnull(), "cvss_temporal"] = ( + dataframe.loc[dataframe["cvss_temporal"].notnull(), "cvss_temporal"] + .str.split() + .apply(lambda x: x[0]) + .fillna('') + ) + + # Combine base and temporal + dataframe["cvss_vector"] = ( + dataframe[["cvss_vector", "cvss_temporal_vector"]] + .apply(lambda x: "{}/{}".format(x[0], x[1]), axis=1) + .str.rstrip("/nan") + .fillna("") + ) + + # Convert Qualys severity to standardised risk number + dataframe['risk_number'] = dataframe['severity'].astype(int)-1 + + dataframe.fillna('', inplace=True) + + return dataframe \ No newline at end of file diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index 7d0c95d..e7614a1 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -633,7 +633,7 @@ class vulnWhispererQualys(vulnWhispererBase): vuln_ready.to_json(relative_path_name, orient='records', lines=True) elif output_format == 'csv': - vuln_ready.to_csv(relative_path_name, index=False, header=True) # add when timestamp occured + vuln_ready.to_csv(relative_path_name, index=False, header=True) # add when timestamp occured self.logger.info('Report written to {}'.format(report_name)) @@ -815,13 +815,6 @@ class vulnWhispererOpenVAS(vulnWhispererBase): class vulnWhispererQualysVuln(vulnWhispererBase): CONFIG_SECTION = 'qualys_vuln' - COLUMN_MAPPING = {'cvss_base': 'cvss', - 'cvss3_base': 'cvss3', - 'cve_id': 'cve', - 'os': 'operating_system', - 'qid': 'plugin_id', - 'severity': 'risk', - 'title': 'plugin_name'} def __init__( self, @@ -850,12 +843,11 @@ class vulnWhispererQualysVuln(vulnWhispererBase): scan_reference=None, output_format='json', cleanup=True): - launched_date if 'Z' in launched_date: launched_date = self.qualys_scan.utils.iso_to_epoch(launched_date) report_name = 'qualys_vuln_' + report_id.replace('/','_') \ + '_{last_updated}'.format(last_updated=launched_date) \ - + '.json' + + '.{extension}'.format(extension=output_format) relative_path_name = self.path_check(report_name) @@ -883,7 +875,8 @@ class vulnWhispererQualysVuln(vulnWhispererBase): vuln_ready = self.qualys_scan.process_data(scan_id=report_id) vuln_ready['scan_name'] = scan_name vuln_ready['scan_reference'] = report_id - vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True) + # Map and transform fields + vuln_ready = self.qualys_scan.normalise(vuln_ready) except Exception as e: self.logger.error('Could not process {}: {}'.format(report_id, str(e))) self.exit_code += 1 @@ -904,9 +897,7 @@ class vulnWhispererQualysVuln(vulnWhispererBase): self.record_insert(record_meta) if output_format == 'json': - with open(relative_path_name, 'w') as f: - f.write(vuln_ready.to_json(orient='records', lines=True)) - f.write('\n') + vuln_ready.to_json(relative_path_name, orient='records', lines=True) self.logger.info('Report written to {}'.format(report_name)) return self.exit_code