From ff5f4cb33179d85343fee08efdfd6efb8341ebd7 Mon Sep 17 00:00:00 2001 From: nate Date: Tue, 3 Aug 2021 16:39:24 -0500 Subject: [PATCH] renamed and cleaned columns --- vulnwhisp/frameworks/qualys_web.py | 48 +++++++++++++++++------------- vulnwhisp/vulnwhisp.py | 13 +++++++- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/vulnwhisp/frameworks/qualys_web.py b/vulnwhisp/frameworks/qualys_web.py index 2978656..1c9d122 100644 --- a/vulnwhisp/frameworks/qualys_web.py +++ b/vulnwhisp/frameworks/qualys_web.py @@ -196,12 +196,11 @@ class qualysScanReport: "Info#1", "CVSS V3 Base", "CVSS V3 Temporal", "CVSS V3 Attack Vector", "Request Body #1" ] - WEB_SCAN_VULN_BLOCK = [CATEGORIES[0]] + WEB_SCAN_BLOCK - WEB_SCAN_VULN_HEADER = WEB_SCAN_VULN_BLOCK - WEB_SCAN_SENSITIVE_BLOCK = [CATEGORIES[1]] + WEB_SCAN_BLOCK - WEB_SCAN_SENSITIVE_HEADER = WEB_SCAN_SENSITIVE_BLOCK + + WEB_SCAN_HEADER = ["Vulnerability Category"] + WEB_SCAN_BLOCK + WEB_SCAN_INFO_BLOCK = [ "INFORMATION_GATHERED", "ID", "Detection ID", "QID", "Results", "Detection Date", @@ -210,7 +209,7 @@ class qualysScanReport: ] WEB_SCAN_INFO_HEADER = [ - "Vulnerability Category", "ID", "Detection ID", "QID", "Results", "Detection Date", + "Vulnerability Category", "ID", "Detection ID", "QID", "Results", "Last Time Detected", "Unique ID", "Flags", "Protocol", "Virtual Host", "IP", "Port", "Result", "Info#1" ] @@ -259,21 +258,20 @@ class qualysScanReport: self.downloaded_file = None def grab_sections(self, report): - all_dataframes = [] - dict_tracker = { + return { 'WEB_SCAN_VULN_BLOCK': pd.DataFrame( self.utils.grab_section( report, self.WEB_SCAN_VULN_BLOCK, end=[self.WEB_SCAN_SENSITIVE_BLOCK, self.WEB_SCAN_INFO_BLOCK], pop_last=True), - columns=self.WEB_SCAN_VULN_HEADER), + columns=self.WEB_SCAN_HEADER), 'WEB_SCAN_SENSITIVE_BLOCK': pd.DataFrame( self.utils.grab_section(report, self.WEB_SCAN_SENSITIVE_BLOCK, end=[self.WEB_SCAN_INFO_BLOCK, self.WEB_SCAN_SENSITIVE_BLOCK], pop_last=True), - columns=self.WEB_SCAN_SENSITIVE_HEADER), + columns=self.WEB_SCAN_HEADER), 'WEB_SCAN_INFO_BLOCK': pd.DataFrame( self.utils.grab_section( report, @@ -321,10 +319,6 @@ class qualysScanReport: self.CATEGORY_HEADER), columns=self.CATEGORY_HEADER) } - all_dataframes.append(dict_tracker) - - - return all_dataframes def data_normalizer(self, dataframes): """ @@ -332,12 +326,21 @@ class qualysScanReport: :param dataframes: :return: """ - df_dict = dataframes[0] - merged_df = pd.concat([df_dict['WEB_SCAN_VULN_BLOCK'], df_dict['WEB_SCAN_SENSITIVE_BLOCK'], - df_dict['WEB_SCAN_INFO_BLOCK']], axis=0, - ignore_index=False) - merged_df = pd.merge(merged_df, df_dict['QID_HEADER'], left_on='QID', - right_on='Id') + df_dict = dataframes + merged_df = pd.concat([ + df_dict['WEB_SCAN_VULN_BLOCK'], + df_dict['WEB_SCAN_SENSITIVE_BLOCK'], + df_dict['WEB_SCAN_INFO_BLOCK'] + ], axis=0, ignore_index=False) + + merged_df = pd.merge( + merged_df, + df_dict['QID_HEADER'].drop( + #these columns always seem to be the same as what we're merging into + ['CVSS V3 Attack Vector', 'CVSS V3 Base', 'CVSS V3 Temporal'], + axis=1), + left_on='QID', right_on='Id' + ) if 'Content' not in merged_df: merged_df['Content'] = '' @@ -354,8 +357,11 @@ class qualysScanReport: merged_df = merged_df.assign(**df_dict['SCAN_META'].to_dict(orient='records')[0]) - merged_df = pd.merge(merged_df, df_dict['CATEGORY_HEADER'], how='left', left_on=['Category', 'Severity Level'], - right_on=['Category', 'Severity'], suffixes=('Severity', 'CatSev')) + merged_df = pd.merge( + merged_df, df_dict['CATEGORY_HEADER'], + how='left', left_on=['Category', 'Severity Level'], + right_on=['Category', 'Severity'], suffixes=('Severity', 'CatSev') + ) merged_df = merged_df.replace('N/A', '').fillna('') diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index 795826b..998763e 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -530,10 +530,14 @@ class vulnWhispererQualys(vulnWhispererBase): 'Ajax Request ID': 'ajax_request_id', 'Authentication': 'authentication', 'CVSS Base': 'cvss', + 'CVSS V3 Attack Vector': 'cvss_v3_attack_vector', + 'CVSS V3 Base': 'cvss_v3_base', + 'CVSS V3 Temporal': 'cvss_v3_temporal', 'CVSS Temporal': 'cvss_temporal', 'CWE': 'cwe', 'Category': 'category', 'Content': 'content', + 'Custom Attributes': 'custom_attributes', 'DescriptionSeverity': 'severity_description', 'DescriptionCatSev': 'category_description', 'Detection ID': 'detection_id', @@ -549,15 +553,19 @@ class vulnWhispererQualys(vulnWhispererBase): 'Ignore User': 'ignore_user', 'Ignored': 'ignored', 'Impact': 'impact', + 'Info#1': 'info_1', 'Last Time Detected': 'last_time_detected', 'Last Time Tested': 'last_time_tested', 'Level': 'level', 'OWASP': 'owasp', 'Operating System': 'operating_system', 'Owner': 'owner', - 'Param': 'param', + 'Param/Cookie': 'param', 'Payload #1': 'payload_1', + 'Port': 'port', + 'Protocol': 'protocol', 'QID': 'plugin_id', + 'Request Body #1': 'request_body_1', 'Request Headers #1': 'request_headers_1', 'Request Method #1': 'request_method_1', 'Request URL #1': 'request_url_1', @@ -566,11 +574,14 @@ class vulnWhispererQualys(vulnWhispererBase): 'Severity': 'risk', 'Severity Level': 'security_level', 'Solution': 'solution', + 'Tags': 'tags', 'Times Detected': 'times_detected', 'Title': 'plugin_name', 'URL': 'url', + 'Unique ID': 'unique_id', 'Url': 'uri', 'Vulnerability Category': 'vulnerability_category', + 'Virtual Host': 'virutal_host', 'WASC': 'wasc', 'Web Application Name': 'web_application_name'}