Merge pull request #230 from nfalke-/2to3

2to3

vulnwhisp/frameworks/qualys_web.py

@@ -17,24 +17,16 @@ import os
 import csv
 import logging
 import dateutil.parser as dp
+csv.field_size_limit(sys.maxsize)
 
 
 class qualysWhisperAPI(object):
-    COUNT_WEBAPP = '/count/was/webapp'
     COUNT_WASSCAN = '/count/was/wasscan'
     DELETE_REPORT = '/delete/was/report/{report_id}'
-    GET_WEBAPP_DETAILS = '/get/was/webapp/{was_id}'
-    QPS_REST_3 = '/qps/rest/3.0'
-    REPORT_DETAILS = '/get/was/report/{report_id}'
     REPORT_STATUS = '/status/was/report/{report_id}'
     REPORT_CREATE = '/create/was/report'
     REPORT_DOWNLOAD = '/download/was/report/{report_id}'
-    SCAN_DETAILS = '/get/was/wasscan/{scan_id}'
-    SCAN_DOWNLOAD = '/download/was/wasscan/{scan_id}'
-    SEARCH_REPORTS = '/search/was/report'
-    SEARCH_WEB_APPS = '/search/was/webapp'
     SEARCH_WAS_SCAN = '/search/was/wasscan'
-    VERSION = '/qps/rest/portal/version'
 
     def __init__(self, config=None):
         self.logger = logging.getLogger('qualysWhisperAPI')
@@ -44,10 +36,6 @@ class qualysWhisperAPI(object):
             self.logger.info('Connected to Qualys at {}'.format(self.qgc.server))
         except Exception as e:
             self.logger.error('Could not connect to Qualys: {}'.format(str(e)))
-        self.headers = {
-            #"content-type": "text/xml"}
-            "Accept" : "application/json",
-            "Content-Type": "application/json"}
         self.config_parse = qcconf.QualysConnectConfig(config, 'qualys_web')
         try:
             self.template_id = self.config_parse.get_template_id()
@@ -72,14 +60,8 @@ class qualysWhisperAPI(object):
 
     def generate_scan_result_XML(self, limit=1000, offset=1, status='FINISHED'):
         report_xml = E.ServiceRequest(
-            E.filters(
+            E.filters(E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status)),
-                E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status
+            E.preferences(E.startFromOffset(str(offset)), E.limitResults(str(limit))),
-                           ),
-            ),
-            E.preferences(
-                E.startFromOffset(str(offset)),
-                E.limitResults(str(limit))
-            ),
         )
         return report_xml
 
@@ -118,8 +100,10 @@ class qualysWhisperAPI(object):
                 if i % limit == 0:
                     if (total - i) < limit:
                         qualys_api_limit = total - i
-                    self.logger.info('Making a request with a limit of {} at offset {}'.format((str(qualys_api_limit)), str(i + 1)))
+                    self.logger.info('Making a request with a limit of {} at offset {}'
-                    scan_info = self.get_scan_info(limit=qualys_api_limit, offset=i + 1, status=status)
+                            .format((str(qualys_api_limit)), str(i + 1)))
+                    scan_info = self.get_scan_info(
+                            limit=qualys_api_limit, offset=i + 1, status=status)
                     _records.append(scan_info)
             self.logger.debug('Converting XML to DataFrame')
             dataframes = [self.xml_parser(xml) for xml in _records]
@@ -136,7 +120,8 @@ class qualysWhisperAPI(object):
         return self.qgc.request(self.REPORT_STATUS.format(report_id=report_id))
 
     def download_report(self, report_id):
-        return self.qgc.request(self.REPORT_DOWNLOAD.format(report_id=report_id))
+        return self.qgc.request(
+                self.REPORT_DOWNLOAD.format(report_id=report_id), http_method='get')
 
     def generate_scan_report_XML(self, scan_id):
         """Generates a CSV report for an asset based on template defined in .ini file"""
@@ -148,20 +133,8 @@ class qualysWhisperAPI(object):
                     E.format('CSV'),
                     #type is not needed, as the template already has it
                     E.type('WAS_SCAN_REPORT'),
-                    E.template(
+                    E.template(E.id(self.template_id)),
-                        E.id(self.template_id)
+                    E.config(E.scanReport(E.target(E.scans(E.WasScan(E.id(scan_id))))))
-                    ),
-                    E.config(
-                        E.scanReport(
-                            E.target(
-                                E.scans(
-                                    E.WasScan(
-                                        E.id(scan_id)
-                                    )
-                                ),
-                            ),
-                        ),
-                    )
                 )
             )
         )
@@ -178,95 +151,14 @@ class qualysWhisperAPI(object):
     def delete_report(self, report_id):
         return self.qgc.request(self.DELETE_REPORT.format(report_id=report_id))
 
-
-class qualysReportFields:
-    CATEGORIES = ['VULNERABILITY',
-                  'SENSITIVECONTENT',
-                  'INFORMATION_GATHERED']
-
-    # URL Vulnerability Information
-
-    VULN_BLOCK = [
-        CATEGORIES[0],
-        'ID',
-        'QID',
-        'Url',
-        'Param',
-        'Function',
-        'Form Entry Point',
-        'Access Path',
-        'Authentication',
-        'Ajax Request',
-        'Ajax Request ID',
-        'Ignored',
-        'Ignore Reason',
-        'Ignore Date',
-        'Ignore User',
-        'Ignore Comments',
-        'First Time Detected',
-        'Last Time Detected',
-        'Last Time Tested',
-        'Times Detected',
-        'Payload #1',
-        'Request Method #1',
-        'Request URL #1',
-        'Request Headers #1',
-        'Response #1',
-        'Evidence #1',
-    ]
-
-    INFO_HEADER = [
-        'Vulnerability Category',
-        'ID',
-        'QID',
-        'Response #1',
-        'Last Time Detected',
-    ]
-    INFO_BLOCK = [
-        CATEGORIES[2],
-        'ID',
-        'QID',
-        'Results',
-        'Detection Date',
-    ]
-
-    QID_HEADER = [
-        'QID',
-        'Id',
-        'Title',
-        'Category',
-        'Severity Level',
-        'Groups',
-        'OWASP',
-        'WASC',
-        'CWE',
-        'CVSS Base',
-        'CVSS Temporal',
-        'Description',
-        'Impact',
-        'Solution',
-    ]
-    GROUP_HEADER = ['GROUP', 'Name', 'Category']
-    OWASP_HEADER = ['OWASP', 'Code', 'Name']
-    WASC_HEADER = ['WASC', 'Code', 'Name']
-    SCAN_META = ['Web Application Name', 'URL', 'Owner', 'Scope', 'Operating System']
-    CATEGORY_HEADER = ['Category', 'Severity', 'Level', 'Description']
-
-
 class qualysUtils:
     def __init__(self):
         self.logger = logging.getLogger('qualysUtils')
 
-    def grab_section(
+    def grab_section(self, report, section, end=[], pop_last=False):
-            self,
-            report,
-            section,
-            end=[],
-            pop_last=False,
-    ):
         temp_list = []
         max_col_count = 0
-        with open(report, 'rb') as csvfile:
+        with open(report, 'rt') as csvfile:
             q_report = csv.reader(csvfile, delimiter=',', quotechar='"')
             for line in q_report:
                 if set(line) == set(section):
@@ -292,44 +184,53 @@ class qualysUtils:
         return _data
 
 class qualysScanReport:
-    # URL Vulnerability Information
+    CATEGORIES = ['VULNERABILITY', 'SENSITIVECONTENT', 'INFORMATION_GATHERED']
-    WEB_SCAN_VULN_BLOCK = list(qualysReportFields.VULN_BLOCK)
-    WEB_SCAN_VULN_BLOCK.insert(WEB_SCAN_VULN_BLOCK.index('QID'), 'Detection ID')
 
-    WEB_SCAN_VULN_HEADER = list(WEB_SCAN_VULN_BLOCK)
+    WEB_SCAN_BLOCK = [
-    WEB_SCAN_VULN_HEADER[WEB_SCAN_VULN_BLOCK.index(qualysReportFields.CATEGORIES[0])] = \
+        "ID", "Detection ID", "QID", "Url", "Param/Cookie", "Function",
-        'Vulnerability Category'
+        "Form Entry Point", "Access Path", "Authentication", "Ajax Request",
+        "Ajax Request ID", "Ignored", "Ignore Reason", "Ignore Date", "Ignore User",
+        "Ignore Comments", "Detection Date", "Payload #1", "Request Method #1",
+        "Request URL #1", "Request Headers #1", "Response #1", "Evidence #1",
+        "Unique ID", "Flags", "Protocol", "Virtual Host", "IP", "Port", "Result",
+        "Info#1", "CVSS V3 Base", "CVSS V3 Temporal", "CVSS V3 Attack Vector",
+        "Request Body #1"
+    ]
+    WEB_SCAN_VULN_BLOCK = [CATEGORIES[0]] + WEB_SCAN_BLOCK
+    WEB_SCAN_SENSITIVE_BLOCK = [CATEGORIES[1]] + WEB_SCAN_BLOCK
 
-    WEB_SCAN_SENSITIVE_HEADER = list(WEB_SCAN_VULN_HEADER)
+    WEB_SCAN_HEADER = ["Vulnerability Category"] + WEB_SCAN_BLOCK
-    WEB_SCAN_SENSITIVE_HEADER.insert(WEB_SCAN_SENSITIVE_HEADER.index('Url'
+    WEB_SCAN_HEADER[WEB_SCAN_HEADER.index("Detection Date")] = "Last Time Detected"
-                                                                     ), 'Content')
 
-    WEB_SCAN_SENSITIVE_BLOCK = list(WEB_SCAN_SENSITIVE_HEADER)
-    WEB_SCAN_SENSITIVE_BLOCK.insert(WEB_SCAN_SENSITIVE_BLOCK.index('QID'), 'Detection ID')
-    WEB_SCAN_SENSITIVE_BLOCK[WEB_SCAN_SENSITIVE_BLOCK.index('Vulnerability Category'
-                                                            )] = qualysReportFields.CATEGORIES[1]
 
-    WEB_SCAN_INFO_HEADER = list(qualysReportFields.INFO_HEADER)
+    WEB_SCAN_INFO_BLOCK = [
-    WEB_SCAN_INFO_HEADER.insert(WEB_SCAN_INFO_HEADER.index('QID'), 'Detection ID')
+        "INFORMATION_GATHERED", "ID", "Detection ID", "QID", "Results", "Detection Date",
+        "Unique ID", "Flags", "Protocol", "Virtual Host", "IP", "Port", "Result",
+        "Info#1"
+    ]
 
-    WEB_SCAN_INFO_BLOCK = list(qualysReportFields.INFO_BLOCK)
+    WEB_SCAN_INFO_HEADER = [
-    WEB_SCAN_INFO_BLOCK.insert(WEB_SCAN_INFO_BLOCK.index('QID'), 'Detection ID')
+        "Vulnerability Category", "ID", "Detection ID", "QID", "Results", "Last Time Detected",
+        "Unique ID", "Flags", "Protocol", "Virtual Host", "IP", "Port", "Result",
+        "Info#1"
+    ]
 
-    QID_HEADER = list(qualysReportFields.QID_HEADER)
+    QID_HEADER = [
-    GROUP_HEADER = list(qualysReportFields.GROUP_HEADER)
+        "QID", "Id", "Title", "Category", "Severity Level", "Groups", "OWASP", "WASC",
-    OWASP_HEADER = list(qualysReportFields.OWASP_HEADER)
+        "CWE", "CVSS Base", "CVSS Temporal", "Description", "Impact", "Solution",
-    WASC_HEADER = list(qualysReportFields.WASC_HEADER)
+        "CVSS V3 Base", "CVSS V3 Temporal", "CVSS V3 Attack Vector"
-    SCAN_META = list(qualysReportFields.SCAN_META)
+    ]
-    CATEGORY_HEADER = list(qualysReportFields.CATEGORY_HEADER)
+    GROUP_HEADER = ['GROUP', 'Name', 'Category']
+    OWASP_HEADER = ['OWASP', 'Code', 'Name']
+    WASC_HEADER = ['WASC', 'Code', 'Name']
+    SCAN_META = [
+        "Web Application Name", "URL", "Owner", "Scope", "ID", "Tags",
+        "Custom Attributes"
+    ]
+    CATEGORY_HEADER = ['Category', 'Severity', 'Level', 'Description']
 
-    def __init__(
+    def __init__(self, config=None, file_in=None, 
-            self,
+            file_stream=False, delimiter=',', quotechar='"'):
-            config=None,
-            file_in=None,
-            file_stream=False,
-            delimiter=',',
-            quotechar='"',
-    ):
         self.logger = logging.getLogger('qualysScanReport')
         self.file_in = file_in
         self.file_stream = file_stream
@@ -340,71 +241,79 @@ class qualysScanReport:
             try:
                 self.qw = qualysWhisperAPI(config=config)
             except Exception as e:
-                self.logger.error('Could not load config! Please check settings. Error: {}'.format(str(e)))
+                self.logger.error(
+                        'Could not load config! Please check settings. Error: {}'.format(
+                            str(e)))
 
         if file_stream:
             self.open_file = file_in.splitlines()
         elif file_in:
-
             self.open_file = open(file_in, 'rb')
 
         self.downloaded_file = None
 
     def grab_sections(self, report):
-        all_dataframes = []
+        return {
-        dict_tracker = {}
+            'WEB_SCAN_VULN_BLOCK': pd.DataFrame(
-        with open(report, 'rb') as csvfile:
+                self.utils.grab_section(
-            dict_tracker['WEB_SCAN_VULN_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
+                    report,
-                                                                                       self.WEB_SCAN_VULN_BLOCK,
+                    self.WEB_SCAN_VULN_BLOCK,
-                                                                                       end=[
+                    end=[self.WEB_SCAN_SENSITIVE_BLOCK, self.WEB_SCAN_INFO_BLOCK],
-                                                                                           self.WEB_SCAN_SENSITIVE_BLOCK,
+                    pop_last=True),
-                                                                                           self.WEB_SCAN_INFO_BLOCK],
+                columns=self.WEB_SCAN_HEADER),
-                                                                                       pop_last=True),
+            'WEB_SCAN_SENSITIVE_BLOCK': pd.DataFrame(
-                                                               columns=self.WEB_SCAN_VULN_HEADER)
+                self.utils.grab_section(report,
-            dict_tracker['WEB_SCAN_SENSITIVE_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
+                    self.WEB_SCAN_SENSITIVE_BLOCK,
-                                                                                            self.WEB_SCAN_SENSITIVE_BLOCK,
+                    end=[self.WEB_SCAN_INFO_BLOCK, self.WEB_SCAN_SENSITIVE_BLOCK],
-                                                                                            end=[
+                    pop_last=True),
-                                                                                                self.WEB_SCAN_INFO_BLOCK,
+                columns=self.WEB_SCAN_HEADER),
-                                                                                                self.WEB_SCAN_SENSITIVE_BLOCK],
+            'WEB_SCAN_INFO_BLOCK': pd.DataFrame(
-                                                                                            pop_last=True),
+                self.utils.grab_section(
-                                                                columns=self.WEB_SCAN_SENSITIVE_HEADER)
+                    report,
-            dict_tracker['WEB_SCAN_INFO_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
+                    self.WEB_SCAN_INFO_BLOCK,
-                                                                                       self.WEB_SCAN_INFO_BLOCK,
+                    end=[self.QID_HEADER],
-                                                                                       end=[self.QID_HEADER],
+                    pop_last=True),
-                                                                                       pop_last=True),
+                columns=self.WEB_SCAN_INFO_HEADER),
-                                                                columns=self.WEB_SCAN_INFO_HEADER)
-            dict_tracker['QID_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
-                                                                              self.QID_HEADER,
-                                                                              end=[self.GROUP_HEADER],
-                                                                              pop_last=True),
-                                                                columns=self.QID_HEADER)
-            dict_tracker['GROUP_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
-                                                                                self.GROUP_HEADER,
-                                                                                end=[self.OWASP_HEADER],
-                                                                                pop_last=True),
-                                                                columns=self.GROUP_HEADER)
-            dict_tracker['OWASP_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
-                                                                                self.OWASP_HEADER,
-                                                                                end=[self.WASC_HEADER],
-                                                                                pop_last=True),
-                                                                columns=self.OWASP_HEADER)
-            dict_tracker['WASC_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
-                                                                               self.WASC_HEADER, end=[['APPENDIX']],
-                                                                               pop_last=True),
-                                                                columns=self.WASC_HEADER)
 
-            dict_tracker['SCAN_META'] = pd.DataFrame(self.utils.grab_section(report,
+            'QID_HEADER': pd.DataFrame(
-                                                                             self.SCAN_META,
+                self.utils.grab_section(
-                                                                             end=[self.CATEGORY_HEADER],
+                    report,
-                                                                             pop_last=True),
+                    self.QID_HEADER,
-                                                                columns=self.SCAN_META)
+                    end=[self.GROUP_HEADER],
-
+                    pop_last=True),
-            dict_tracker['CATEGORY_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
+                columns=self.QID_HEADER),
-                                                                                   self.CATEGORY_HEADER),
+            'GROUP_HEADER': pd.DataFrame(
-                                                                columns=self.CATEGORY_HEADER)
+                self.utils.grab_section(
-            all_dataframes.append(dict_tracker)
+                    report,
-
+                    self.GROUP_HEADER,
-        return all_dataframes
+                    end=[self.OWASP_HEADER],
+                    pop_last=True),
+                columns=self.GROUP_HEADER),
+            'OWASP_HEADER': pd.DataFrame(
+                self.utils.grab_section(
+                    report,
+                    self.OWASP_HEADER,
+                    end=[self.WASC_HEADER],
+                    pop_last=True),
+                columns=self.OWASP_HEADER),
+            'WASC_HEADER': pd.DataFrame(
+                self.utils.grab_section(
+                    report,
+                    self.WASC_HEADER,
+                    end=[['APPENDIX']],
+                    pop_last=True),
+                columns=self.WASC_HEADER),
+            'SCAN_META': pd.DataFrame(
+                self.utils.grab_section(report,
+                    self.SCAN_META,
+                    end=[self.CATEGORY_HEADER],
+                    pop_last=True),
+                columns=self.SCAN_META),
+            'CATEGORY_HEADER': pd.DataFrame(
+                self.utils.grab_section(report,
+                    self.CATEGORY_HEADER),
+                columns=self.CATEGORY_HEADER)
+        }
 
     def data_normalizer(self, dataframes):
         """
@@ -412,12 +321,21 @@ class qualysScanReport:
         :param dataframes:
         :return:
         """
-        df_dict = dataframes[0]
+        df_dict = dataframes
-        merged_df = pd.concat([df_dict['WEB_SCAN_VULN_BLOCK'], df_dict['WEB_SCAN_SENSITIVE_BLOCK'],
+        merged_df = pd.concat([
-                               df_dict['WEB_SCAN_INFO_BLOCK']], axis=0,
+            df_dict['WEB_SCAN_VULN_BLOCK'],
-                              ignore_index=False)
+            df_dict['WEB_SCAN_SENSITIVE_BLOCK'],
-        merged_df = pd.merge(merged_df, df_dict['QID_HEADER'], left_on='QID',
+            df_dict['WEB_SCAN_INFO_BLOCK']
-                             right_on='Id')
+        ], axis=0, ignore_index=False)
+
+        merged_df = pd.merge(
+                merged_df,
+                df_dict['QID_HEADER'].drop(
+                    #these columns always seem to be the same as what we're merging into
+                    ['CVSS V3 Attack Vector', 'CVSS V3 Base', 'CVSS V3 Temporal'],
+                    axis=1),
+                left_on='QID', right_on='Id'
+        )
 
         if 'Content' not in merged_df:
             merged_df['Content'] = ''
@@ -434,8 +352,11 @@ class qualysScanReport:
 
         merged_df = merged_df.assign(**df_dict['SCAN_META'].to_dict(orient='records')[0])
 
-        merged_df = pd.merge(merged_df, df_dict['CATEGORY_HEADER'], how='left', left_on=['Category', 'Severity Level'],
+        merged_df = pd.merge(
-                             right_on=['Category', 'Severity'], suffixes=('Severity', 'CatSev'))
+                merged_df, df_dict['CATEGORY_HEADER'],
+                how='left', left_on=['Category', 'Severity Level'],
+                right_on=['Category', 'Severity'], suffixes=('Severity', 'CatSev')
+        )
 
         merged_df = merged_df.replace('N/A', '').fillna('')
 

vulnwhisp/vulnwhisp.py

@@ -530,10 +530,14 @@ class vulnWhispererQualys(vulnWhispererBase):
                       'Ajax Request ID': 'ajax_request_id',
                       'Authentication': 'authentication',
                       'CVSS Base': 'cvss',
+                      'CVSS V3 Attack Vector': 'cvss_v3_attack_vector',
+                      'CVSS V3 Base': 'cvss_v3_base',
+                      'CVSS V3 Temporal': 'cvss_v3_temporal',
                       'CVSS Temporal': 'cvss_temporal',
                       'CWE': 'cwe',
                       'Category': 'category',
                       'Content': 'content',
+                      'Custom Attributes': 'custom_attributes',
                       'DescriptionSeverity': 'severity_description',
                       'DescriptionCatSev': 'category_description',
                       'Detection ID': 'detection_id',
@@ -549,15 +553,19 @@ class vulnWhispererQualys(vulnWhispererBase):
                       'Ignore User': 'ignore_user',
                       'Ignored': 'ignored',
                       'Impact': 'impact',
+                      'Info#1': 'info_1',
                       'Last Time Detected': 'last_time_detected',
                       'Last Time Tested': 'last_time_tested',
                       'Level': 'level',
                       'OWASP': 'owasp',
                       'Operating System': 'operating_system',
                       'Owner': 'owner',
-                      'Param': 'param',
+                      'Param/Cookie': 'param',
                       'Payload #1': 'payload_1',
+                      'Port': 'port',
+                      'Protocol': 'protocol',
                       'QID': 'plugin_id',
+                      'Request Body #1': 'request_body_1',
                       'Request Headers #1': 'request_headers_1',
                       'Request Method #1': 'request_method_1',
                       'Request URL #1': 'request_url_1',
@@ -566,11 +574,14 @@ class vulnWhispererQualys(vulnWhispererBase):
                       'Severity': 'risk',
                       'Severity Level': 'security_level',
                       'Solution': 'solution',
+                      'Tags': 'tags',
                       'Times Detected': 'times_detected',
                       'Title': 'plugin_name',
                       'URL': 'url',
+                      'Unique ID': 'unique_id',
                       'Url': 'uri',
                       'Vulnerability Category': 'vulnerability_category',
+                      'Virtual Host': 'virutal_host',
                       'WASC': 'wasc',
                       'Web Application Name': 'web_application_name'}