diff --git a/vulnwhisp/frameworks/qualys_web.py b/vulnwhisp/frameworks/qualys_web.py index 158c886..8ce5f3e 100644 --- a/vulnwhisp/frameworks/qualys_web.py +++ b/vulnwhisp/frameworks/qualys_web.py @@ -17,24 +17,16 @@ import os import csv import logging import dateutil.parser as dp +csv.field_size_limit(sys.maxsize) class qualysWhisperAPI(object): - COUNT_WEBAPP = '/count/was/webapp' COUNT_WASSCAN = '/count/was/wasscan' DELETE_REPORT = '/delete/was/report/{report_id}' - GET_WEBAPP_DETAILS = '/get/was/webapp/{was_id}' - QPS_REST_3 = '/qps/rest/3.0' - REPORT_DETAILS = '/get/was/report/{report_id}' REPORT_STATUS = '/status/was/report/{report_id}' REPORT_CREATE = '/create/was/report' REPORT_DOWNLOAD = '/download/was/report/{report_id}' - SCAN_DETAILS = '/get/was/wasscan/{scan_id}' - SCAN_DOWNLOAD = '/download/was/wasscan/{scan_id}' - SEARCH_REPORTS = '/search/was/report' - SEARCH_WEB_APPS = '/search/was/webapp' SEARCH_WAS_SCAN = '/search/was/wasscan' - VERSION = '/qps/rest/portal/version' def __init__(self, config=None): self.logger = logging.getLogger('qualysWhisperAPI') @@ -44,10 +36,6 @@ class qualysWhisperAPI(object): self.logger.info('Connected to Qualys at {}'.format(self.qgc.server)) except Exception as e: self.logger.error('Could not connect to Qualys: {}'.format(str(e))) - self.headers = { - #"content-type": "text/xml"} - "Accept" : "application/json", - "Content-Type": "application/json"} self.config_parse = qcconf.QualysConnectConfig(config, 'qualys_web') try: self.template_id = self.config_parse.get_template_id() @@ -72,14 +60,8 @@ class qualysWhisperAPI(object): def generate_scan_result_XML(self, limit=1000, offset=1, status='FINISHED'): report_xml = E.ServiceRequest( - E.filters( - E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status - ), - ), - E.preferences( - E.startFromOffset(str(offset)), - E.limitResults(str(limit)) - ), + E.filters(E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status)), + E.preferences(E.startFromOffset(str(offset)), E.limitResults(str(limit))), ) return report_xml @@ -118,8 +100,10 @@ class qualysWhisperAPI(object): if i % limit == 0: if (total - i) < limit: qualys_api_limit = total - i - self.logger.info('Making a request with a limit of {} at offset {}'.format((str(qualys_api_limit)), str(i + 1))) - scan_info = self.get_scan_info(limit=qualys_api_limit, offset=i + 1, status=status) + self.logger.info('Making a request with a limit of {} at offset {}' + .format((str(qualys_api_limit)), str(i + 1))) + scan_info = self.get_scan_info( + limit=qualys_api_limit, offset=i + 1, status=status) _records.append(scan_info) self.logger.debug('Converting XML to DataFrame') dataframes = [self.xml_parser(xml) for xml in _records] @@ -136,7 +120,8 @@ class qualysWhisperAPI(object): return self.qgc.request(self.REPORT_STATUS.format(report_id=report_id)) def download_report(self, report_id): - return self.qgc.request(self.REPORT_DOWNLOAD.format(report_id=report_id)) + return self.qgc.request( + self.REPORT_DOWNLOAD.format(report_id=report_id), http_method='get') def generate_scan_report_XML(self, scan_id): """Generates a CSV report for an asset based on template defined in .ini file""" @@ -148,20 +133,8 @@ class qualysWhisperAPI(object): E.format('CSV'), #type is not needed, as the template already has it E.type('WAS_SCAN_REPORT'), - E.template( - E.id(self.template_id) - ), - E.config( - E.scanReport( - E.target( - E.scans( - E.WasScan( - E.id(scan_id) - ) - ), - ), - ), - ) + E.template(E.id(self.template_id)), + E.config(E.scanReport(E.target(E.scans(E.WasScan(E.id(scan_id)))))) ) ) ) @@ -178,95 +151,14 @@ class qualysWhisperAPI(object): def delete_report(self, report_id): return self.qgc.request(self.DELETE_REPORT.format(report_id=report_id)) - -class qualysReportFields: - CATEGORIES = ['VULNERABILITY', - 'SENSITIVECONTENT', - 'INFORMATION_GATHERED'] - - # URL Vulnerability Information - - VULN_BLOCK = [ - CATEGORIES[0], - 'ID', - 'QID', - 'Url', - 'Param', - 'Function', - 'Form Entry Point', - 'Access Path', - 'Authentication', - 'Ajax Request', - 'Ajax Request ID', - 'Ignored', - 'Ignore Reason', - 'Ignore Date', - 'Ignore User', - 'Ignore Comments', - 'First Time Detected', - 'Last Time Detected', - 'Last Time Tested', - 'Times Detected', - 'Payload #1', - 'Request Method #1', - 'Request URL #1', - 'Request Headers #1', - 'Response #1', - 'Evidence #1', - ] - - INFO_HEADER = [ - 'Vulnerability Category', - 'ID', - 'QID', - 'Response #1', - 'Last Time Detected', - ] - INFO_BLOCK = [ - CATEGORIES[2], - 'ID', - 'QID', - 'Results', - 'Detection Date', - ] - - QID_HEADER = [ - 'QID', - 'Id', - 'Title', - 'Category', - 'Severity Level', - 'Groups', - 'OWASP', - 'WASC', - 'CWE', - 'CVSS Base', - 'CVSS Temporal', - 'Description', - 'Impact', - 'Solution', - ] - GROUP_HEADER = ['GROUP', 'Name', 'Category'] - OWASP_HEADER = ['OWASP', 'Code', 'Name'] - WASC_HEADER = ['WASC', 'Code', 'Name'] - SCAN_META = ['Web Application Name', 'URL', 'Owner', 'Scope', 'Operating System'] - CATEGORY_HEADER = ['Category', 'Severity', 'Level', 'Description'] - - class qualysUtils: def __init__(self): self.logger = logging.getLogger('qualysUtils') - def grab_section( - self, - report, - section, - end=[], - pop_last=False, - ): + def grab_section(self, report, section, end=[], pop_last=False): temp_list = [] max_col_count = 0 - with open(report, 'rb') as csvfile: + with open(report, 'rt') as csvfile: q_report = csv.reader(csvfile, delimiter=',', quotechar='"') for line in q_report: if set(line) == set(section): @@ -292,44 +184,53 @@ class qualysUtils: return _data class qualysScanReport: - # URL Vulnerability Information - WEB_SCAN_VULN_BLOCK = list(qualysReportFields.VULN_BLOCK) - WEB_SCAN_VULN_BLOCK.insert(WEB_SCAN_VULN_BLOCK.index('QID'), 'Detection ID') + CATEGORIES = ['VULNERABILITY', 'SENSITIVECONTENT', 'INFORMATION_GATHERED'] - WEB_SCAN_VULN_HEADER = list(WEB_SCAN_VULN_BLOCK) - WEB_SCAN_VULN_HEADER[WEB_SCAN_VULN_BLOCK.index(qualysReportFields.CATEGORIES[0])] = \ - 'Vulnerability Category' + WEB_SCAN_BLOCK = [ + "ID", "Detection ID", "QID", "Url", "Param/Cookie", "Function", + "Form Entry Point", "Access Path", "Authentication", "Ajax Request", + "Ajax Request ID", "Ignored", "Ignore Reason", "Ignore Date", "Ignore User", + "Ignore Comments", "Detection Date", "Payload #1", "Request Method #1", + "Request URL #1", "Request Headers #1", "Response #1", "Evidence #1", + "Unique ID", "Flags", "Protocol", "Virtual Host", "IP", "Port", "Result", + "Info#1", "CVSS V3 Base", "CVSS V3 Temporal", "CVSS V3 Attack Vector", + "Request Body #1" + ] + WEB_SCAN_VULN_BLOCK = [CATEGORIES[0]] + WEB_SCAN_BLOCK + WEB_SCAN_SENSITIVE_BLOCK = [CATEGORIES[1]] + WEB_SCAN_BLOCK - WEB_SCAN_SENSITIVE_HEADER = list(WEB_SCAN_VULN_HEADER) - WEB_SCAN_SENSITIVE_HEADER.insert(WEB_SCAN_SENSITIVE_HEADER.index('Url' - ), 'Content') + WEB_SCAN_HEADER = ["Vulnerability Category"] + WEB_SCAN_BLOCK + WEB_SCAN_HEADER[WEB_SCAN_HEADER.index("Detection Date")] = "Last Time Detected" - WEB_SCAN_SENSITIVE_BLOCK = list(WEB_SCAN_SENSITIVE_HEADER) - WEB_SCAN_SENSITIVE_BLOCK.insert(WEB_SCAN_SENSITIVE_BLOCK.index('QID'), 'Detection ID') - WEB_SCAN_SENSITIVE_BLOCK[WEB_SCAN_SENSITIVE_BLOCK.index('Vulnerability Category' - )] = qualysReportFields.CATEGORIES[1] - WEB_SCAN_INFO_HEADER = list(qualysReportFields.INFO_HEADER) - WEB_SCAN_INFO_HEADER.insert(WEB_SCAN_INFO_HEADER.index('QID'), 'Detection ID') + WEB_SCAN_INFO_BLOCK = [ + "INFORMATION_GATHERED", "ID", "Detection ID", "QID", "Results", "Detection Date", + "Unique ID", "Flags", "Protocol", "Virtual Host", "IP", "Port", "Result", + "Info#1" + ] - WEB_SCAN_INFO_BLOCK = list(qualysReportFields.INFO_BLOCK) - WEB_SCAN_INFO_BLOCK.insert(WEB_SCAN_INFO_BLOCK.index('QID'), 'Detection ID') + WEB_SCAN_INFO_HEADER = [ + "Vulnerability Category", "ID", "Detection ID", "QID", "Results", "Last Time Detected", + "Unique ID", "Flags", "Protocol", "Virtual Host", "IP", "Port", "Result", + "Info#1" + ] - QID_HEADER = list(qualysReportFields.QID_HEADER) - GROUP_HEADER = list(qualysReportFields.GROUP_HEADER) - OWASP_HEADER = list(qualysReportFields.OWASP_HEADER) - WASC_HEADER = list(qualysReportFields.WASC_HEADER) - SCAN_META = list(qualysReportFields.SCAN_META) - CATEGORY_HEADER = list(qualysReportFields.CATEGORY_HEADER) + QID_HEADER = [ + "QID", "Id", "Title", "Category", "Severity Level", "Groups", "OWASP", "WASC", + "CWE", "CVSS Base", "CVSS Temporal", "Description", "Impact", "Solution", + "CVSS V3 Base", "CVSS V3 Temporal", "CVSS V3 Attack Vector" + ] + GROUP_HEADER = ['GROUP', 'Name', 'Category'] + OWASP_HEADER = ['OWASP', 'Code', 'Name'] + WASC_HEADER = ['WASC', 'Code', 'Name'] + SCAN_META = [ + "Web Application Name", "URL", "Owner", "Scope", "ID", "Tags", + "Custom Attributes" + ] + CATEGORY_HEADER = ['Category', 'Severity', 'Level', 'Description'] - def __init__( - self, - config=None, - file_in=None, - file_stream=False, - delimiter=',', - quotechar='"', - ): + def __init__(self, config=None, file_in=None, + file_stream=False, delimiter=',', quotechar='"'): self.logger = logging.getLogger('qualysScanReport') self.file_in = file_in self.file_stream = file_stream @@ -340,71 +241,79 @@ class qualysScanReport: try: self.qw = qualysWhisperAPI(config=config) except Exception as e: - self.logger.error('Could not load config! Please check settings. Error: {}'.format(str(e))) + self.logger.error( + 'Could not load config! Please check settings. Error: {}'.format( + str(e))) if file_stream: self.open_file = file_in.splitlines() elif file_in: - self.open_file = open(file_in, 'rb') self.downloaded_file = None def grab_sections(self, report): - all_dataframes = [] - dict_tracker = {} - with open(report, 'rb') as csvfile: - dict_tracker['WEB_SCAN_VULN_BLOCK'] = pd.DataFrame(self.utils.grab_section(report, - self.WEB_SCAN_VULN_BLOCK, - end=[ - self.WEB_SCAN_SENSITIVE_BLOCK, - self.WEB_SCAN_INFO_BLOCK], - pop_last=True), - columns=self.WEB_SCAN_VULN_HEADER) - dict_tracker['WEB_SCAN_SENSITIVE_BLOCK'] = pd.DataFrame(self.utils.grab_section(report, - self.WEB_SCAN_SENSITIVE_BLOCK, - end=[ - self.WEB_SCAN_INFO_BLOCK, - self.WEB_SCAN_SENSITIVE_BLOCK], - pop_last=True), - columns=self.WEB_SCAN_SENSITIVE_HEADER) - dict_tracker['WEB_SCAN_INFO_BLOCK'] = pd.DataFrame(self.utils.grab_section(report, - self.WEB_SCAN_INFO_BLOCK, - end=[self.QID_HEADER], - pop_last=True), - columns=self.WEB_SCAN_INFO_HEADER) - dict_tracker['QID_HEADER'] = pd.DataFrame(self.utils.grab_section(report, - self.QID_HEADER, - end=[self.GROUP_HEADER], - pop_last=True), - columns=self.QID_HEADER) - dict_tracker['GROUP_HEADER'] = pd.DataFrame(self.utils.grab_section(report, - self.GROUP_HEADER, - end=[self.OWASP_HEADER], - pop_last=True), - columns=self.GROUP_HEADER) - dict_tracker['OWASP_HEADER'] = pd.DataFrame(self.utils.grab_section(report, - self.OWASP_HEADER, - end=[self.WASC_HEADER], - pop_last=True), - columns=self.OWASP_HEADER) - dict_tracker['WASC_HEADER'] = pd.DataFrame(self.utils.grab_section(report, - self.WASC_HEADER, end=[['APPENDIX']], - pop_last=True), - columns=self.WASC_HEADER) + return { + 'WEB_SCAN_VULN_BLOCK': pd.DataFrame( + self.utils.grab_section( + report, + self.WEB_SCAN_VULN_BLOCK, + end=[self.WEB_SCAN_SENSITIVE_BLOCK, self.WEB_SCAN_INFO_BLOCK], + pop_last=True), + columns=self.WEB_SCAN_HEADER), + 'WEB_SCAN_SENSITIVE_BLOCK': pd.DataFrame( + self.utils.grab_section(report, + self.WEB_SCAN_SENSITIVE_BLOCK, + end=[self.WEB_SCAN_INFO_BLOCK, self.WEB_SCAN_SENSITIVE_BLOCK], + pop_last=True), + columns=self.WEB_SCAN_HEADER), + 'WEB_SCAN_INFO_BLOCK': pd.DataFrame( + self.utils.grab_section( + report, + self.WEB_SCAN_INFO_BLOCK, + end=[self.QID_HEADER], + pop_last=True), + columns=self.WEB_SCAN_INFO_HEADER), - dict_tracker['SCAN_META'] = pd.DataFrame(self.utils.grab_section(report, - self.SCAN_META, - end=[self.CATEGORY_HEADER], - pop_last=True), - columns=self.SCAN_META) - - dict_tracker['CATEGORY_HEADER'] = pd.DataFrame(self.utils.grab_section(report, - self.CATEGORY_HEADER), - columns=self.CATEGORY_HEADER) - all_dataframes.append(dict_tracker) - - return all_dataframes + 'QID_HEADER': pd.DataFrame( + self.utils.grab_section( + report, + self.QID_HEADER, + end=[self.GROUP_HEADER], + pop_last=True), + columns=self.QID_HEADER), + 'GROUP_HEADER': pd.DataFrame( + self.utils.grab_section( + report, + self.GROUP_HEADER, + end=[self.OWASP_HEADER], + pop_last=True), + columns=self.GROUP_HEADER), + 'OWASP_HEADER': pd.DataFrame( + self.utils.grab_section( + report, + self.OWASP_HEADER, + end=[self.WASC_HEADER], + pop_last=True), + columns=self.OWASP_HEADER), + 'WASC_HEADER': pd.DataFrame( + self.utils.grab_section( + report, + self.WASC_HEADER, + end=[['APPENDIX']], + pop_last=True), + columns=self.WASC_HEADER), + 'SCAN_META': pd.DataFrame( + self.utils.grab_section(report, + self.SCAN_META, + end=[self.CATEGORY_HEADER], + pop_last=True), + columns=self.SCAN_META), + 'CATEGORY_HEADER': pd.DataFrame( + self.utils.grab_section(report, + self.CATEGORY_HEADER), + columns=self.CATEGORY_HEADER) + } def data_normalizer(self, dataframes): """ @@ -412,12 +321,21 @@ class qualysScanReport: :param dataframes: :return: """ - df_dict = dataframes[0] - merged_df = pd.concat([df_dict['WEB_SCAN_VULN_BLOCK'], df_dict['WEB_SCAN_SENSITIVE_BLOCK'], - df_dict['WEB_SCAN_INFO_BLOCK']], axis=0, - ignore_index=False) - merged_df = pd.merge(merged_df, df_dict['QID_HEADER'], left_on='QID', - right_on='Id') + df_dict = dataframes + merged_df = pd.concat([ + df_dict['WEB_SCAN_VULN_BLOCK'], + df_dict['WEB_SCAN_SENSITIVE_BLOCK'], + df_dict['WEB_SCAN_INFO_BLOCK'] + ], axis=0, ignore_index=False) + + merged_df = pd.merge( + merged_df, + df_dict['QID_HEADER'].drop( + #these columns always seem to be the same as what we're merging into + ['CVSS V3 Attack Vector', 'CVSS V3 Base', 'CVSS V3 Temporal'], + axis=1), + left_on='QID', right_on='Id' + ) if 'Content' not in merged_df: merged_df['Content'] = '' @@ -434,8 +352,11 @@ class qualysScanReport: merged_df = merged_df.assign(**df_dict['SCAN_META'].to_dict(orient='records')[0]) - merged_df = pd.merge(merged_df, df_dict['CATEGORY_HEADER'], how='left', left_on=['Category', 'Severity Level'], - right_on=['Category', 'Severity'], suffixes=('Severity', 'CatSev')) + merged_df = pd.merge( + merged_df, df_dict['CATEGORY_HEADER'], + how='left', left_on=['Category', 'Severity Level'], + right_on=['Category', 'Severity'], suffixes=('Severity', 'CatSev') + ) merged_df = merged_df.replace('N/A', '').fillna('') diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index 795826b..998763e 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -530,10 +530,14 @@ class vulnWhispererQualys(vulnWhispererBase): 'Ajax Request ID': 'ajax_request_id', 'Authentication': 'authentication', 'CVSS Base': 'cvss', + 'CVSS V3 Attack Vector': 'cvss_v3_attack_vector', + 'CVSS V3 Base': 'cvss_v3_base', + 'CVSS V3 Temporal': 'cvss_v3_temporal', 'CVSS Temporal': 'cvss_temporal', 'CWE': 'cwe', 'Category': 'category', 'Content': 'content', + 'Custom Attributes': 'custom_attributes', 'DescriptionSeverity': 'severity_description', 'DescriptionCatSev': 'category_description', 'Detection ID': 'detection_id', @@ -549,15 +553,19 @@ class vulnWhispererQualys(vulnWhispererBase): 'Ignore User': 'ignore_user', 'Ignored': 'ignored', 'Impact': 'impact', + 'Info#1': 'info_1', 'Last Time Detected': 'last_time_detected', 'Last Time Tested': 'last_time_tested', 'Level': 'level', 'OWASP': 'owasp', 'Operating System': 'operating_system', 'Owner': 'owner', - 'Param': 'param', + 'Param/Cookie': 'param', 'Payload #1': 'payload_1', + 'Port': 'port', + 'Protocol': 'protocol', 'QID': 'plugin_id', + 'Request Body #1': 'request_body_1', 'Request Headers #1': 'request_headers_1', 'Request Method #1': 'request_method_1', 'Request URL #1': 'request_url_1', @@ -566,11 +574,14 @@ class vulnWhispererQualys(vulnWhispererBase): 'Severity': 'risk', 'Severity Level': 'security_level', 'Solution': 'solution', + 'Tags': 'tags', 'Times Detected': 'times_detected', 'Title': 'plugin_name', 'URL': 'url', + 'Unique ID': 'unique_id', 'Url': 'uri', 'Vulnerability Category': 'vulnerability_category', + 'Virtual Host': 'virutal_host', 'WASC': 'wasc', 'Web Application Name': 'web_application_name'}