From 92cad06b2b8b9a81f50c71382a637095591581d3 Mon Sep 17 00:00:00 2001 From: pemontto Date: Sat, 27 Apr 2019 07:26:35 +0100 Subject: [PATCH] Update Qualys WAS mapping and transforms --- vulnwhisp/frameworks/qualys_web.py | 44 ++++++++++++++++++------ vulnwhisp/vulnwhisp.py | 55 ++---------------------------- 2 files changed, 36 insertions(+), 63 deletions(-) diff --git a/vulnwhisp/frameworks/qualys_web.py b/vulnwhisp/frameworks/qualys_web.py index 34547f8..82f93f3 100644 --- a/vulnwhisp/frameworks/qualys_web.py +++ b/vulnwhisp/frameworks/qualys_web.py @@ -282,13 +282,24 @@ class qualysUtils: def iso_to_epoch(self, dt): return dp.parse(dt).strftime('%s') - def cleanser(self, _data): - repls = (('\n', '|||'), ('\r', '|||'), (',', ';'), ('\t', '|||')) - if _data: - _data = reduce(lambda a, kv: a.replace(*kv), repls, str(_data)) - return _data - class qualysScanReport: + + COLUMN_MAPPING = { + 'DescriptionCatSev': 'category_description', + 'DescriptionSeverity': 'severity_description', + 'Evidence #1': 'evidence', + 'Payload #1': 'payload', + 'Request Headers #1': 'request_headers', + 'Request Method #1': 'request_method', + 'Request URL #1': 'request_url', + 'Response #1': 'response', + 'URL': 'url', + 'Url': 'uri', + 'QID': 'plugin_id', + } + + SEVERITY_MAPPING = {0: 'none', 1: 'low', 2: 'medium', 3: 'high',4: 'critical'} + # URL Vulnerability Information WEB_SCAN_VULN_BLOCK = list(qualysReportFields.VULN_BLOCK) WEB_SCAN_VULN_BLOCK.insert(WEB_SCAN_VULN_BLOCK.index('QID'), 'Detection ID') @@ -444,9 +455,6 @@ class qualysScanReport: 'Request Headers #1', 'Response #1', 'Evidence #1', 'Description', 'Impact', 'Solution', 'Url', 'Content'] - for col in columns_to_cleanse: - merged_df[col] = merged_df[col].apply(self.utils.cleanser) - merged_df = merged_df.drop(['QID_y', 'QID_x'], axis=1) merged_df = merged_df.rename(columns={'Id': 'QID'}) @@ -493,9 +501,25 @@ class qualysScanReport: def map_fields(self, df): self.logger.debug('Mapping fields') + + df.rename(columns=self.COLUMN_MAPPING, inplace=True) + + # Lowercase and map fields from COLUMN_MAPPING + df.columns = [x.lower() for x in df.columns] + df.columns = [x.replace(' ', '_') for x in df.columns] + return df - + def transform_values(self, df): self.logger.debug('Transforming values') + df.fillna('', inplace=True) + + self.logger.info('Changing case of fields') + df['cwe'] = df['cwe'].str.upper() + + # Convert Qualys severity to standardised risk number + df['risk_number'] = df['severity'].astype(int)-1 + df['risk'] = df['risk_number'].map(self.SEVERITY_MAPPING) + df.fillna('', inplace=True) return df diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index 1c06de2..2da3878 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -532,54 +532,6 @@ class vulnWhispererNessus(vulnWhispererBase): class vulnWhispererQualys(vulnWhispererBase): CONFIG_SECTION = 'qualys_web' - COLUMN_MAPPING = {'Access Path': 'access_path', - 'Ajax Request': 'ajax_request', - 'Ajax Request ID': 'ajax_request_id', - 'Authentication': 'authentication', - 'CVSS Base': 'cvss', - 'CVSS Temporal': 'cvss_temporal', - 'CWE': 'cwe', - 'Category': 'category', - 'Content': 'content', - 'DescriptionSeverity': 'severity_description', - 'DescriptionCatSev': 'category_description', - 'Detection ID': 'detection_id', - 'Evidence #1': 'evidence_1', - 'First Time Detected': 'first_time_detected', - 'Form Entry Point': 'form_entry_point', - 'Function': 'function', - 'Groups': 'groups', - 'ID': 'id', - 'Ignore Comments': 'ignore_comments', - 'Ignore Date': 'ignore_date', - 'Ignore Reason': 'ignore_reason', - 'Ignore User': 'ignore_user', - 'Ignored': 'ignored', - 'Impact': 'impact', - 'Last Time Detected': 'last_time_detected', - 'Last Time Tested': 'last_time_tested', - 'Level': 'level', - 'OWASP': 'owasp', - 'Operating System': 'operating_system', - 'Owner': 'owner', - 'Param': 'param', - 'Payload #1': 'payload_1', - 'QID': 'plugin_id', - 'Request Headers #1': 'request_headers_1', - 'Request Method #1': 'request_method_1', - 'Request URL #1': 'request_url_1', - 'Response #1': 'response_1', - 'Scope': 'scope', - 'Severity': 'risk', - 'Severity Level': 'security_level', - 'Solution': 'solution', - 'Times Detected': 'times_detected', - 'Title': 'plugin_name', - 'URL': 'url', - 'Url': 'uri', - 'Vulnerability Category': 'vulnerability_category', - 'WASC': 'wasc', - 'Web Application Name': 'web_application_name'} def __init__( self, config=None, @@ -654,8 +606,6 @@ class vulnWhispererQualys(vulnWhispererBase): # Map and transform fields vuln_ready = self.qualys_scan.normalise(vuln_ready) vuln_ready = self.common_normalise(vuln_ready) - # TODO remove the line below once normalising complete - vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True) # Set common fields vuln_ready['app_id'] = report_id @@ -690,9 +640,8 @@ class vulnWhispererQualys(vulnWhispererBase): if cleanup: self.logger.info('Removing report {} from Qualys Database'.format(generated_report_id)) - cleaning_up = \ - self.qualys_scan.qw.delete_report(generated_report_id) - os.remove(self.path_check(str(generated_report_id) + '.csv')) + cleaning_up = self.qualys_scan.qw.delete_report(generated_report_id) + # os.remove(self.path_check(str(generated_report_id) + '.csv')) self.logger.info('Deleted report from local disk: {}'.format(self.path_check(str(generated_report_id)))) else: self.logger.error('Could not process report ID: {}'.format(status))