From 183e3b3e720023fb7272cd16c6e96898cd205626 Mon Sep 17 00:00:00 2001 From: nate Date: Tue, 3 Aug 2021 13:01:22 -0500 Subject: [PATCH 01/14] removed useless open --- vulnwhisp/frameworks/qualys_web.py | 98 +++++++++++++++--------------- 1 file changed, 48 insertions(+), 50 deletions(-) diff --git a/vulnwhisp/frameworks/qualys_web.py b/vulnwhisp/frameworks/qualys_web.py index 158c886..057bafd 100644 --- a/vulnwhisp/frameworks/qualys_web.py +++ b/vulnwhisp/frameworks/qualys_web.py @@ -266,7 +266,7 @@ class qualysUtils: ): temp_list = [] max_col_count = 0 - with open(report, 'rb') as csvfile: + with open(report, 'rt') as csvfile: q_report = csv.reader(csvfile, delimiter=',', quotechar='"') for line in q_report: if set(line) == set(section): @@ -345,7 +345,6 @@ class qualysScanReport: if file_stream: self.open_file = file_in.splitlines() elif file_in: - self.open_file = open(file_in, 'rb') self.downloaded_file = None @@ -353,56 +352,55 @@ class qualysScanReport: def grab_sections(self, report): all_dataframes = [] dict_tracker = {} - with open(report, 'rb') as csvfile: - dict_tracker['WEB_SCAN_VULN_BLOCK'] = pd.DataFrame(self.utils.grab_section(report, - self.WEB_SCAN_VULN_BLOCK, - end=[ - self.WEB_SCAN_SENSITIVE_BLOCK, - self.WEB_SCAN_INFO_BLOCK], - pop_last=True), - columns=self.WEB_SCAN_VULN_HEADER) - dict_tracker['WEB_SCAN_SENSITIVE_BLOCK'] = pd.DataFrame(self.utils.grab_section(report, - self.WEB_SCAN_SENSITIVE_BLOCK, - end=[ - self.WEB_SCAN_INFO_BLOCK, - self.WEB_SCAN_SENSITIVE_BLOCK], - pop_last=True), - columns=self.WEB_SCAN_SENSITIVE_HEADER) - dict_tracker['WEB_SCAN_INFO_BLOCK'] = pd.DataFrame(self.utils.grab_section(report, - self.WEB_SCAN_INFO_BLOCK, - end=[self.QID_HEADER], - pop_last=True), - columns=self.WEB_SCAN_INFO_HEADER) - dict_tracker['QID_HEADER'] = pd.DataFrame(self.utils.grab_section(report, - self.QID_HEADER, - end=[self.GROUP_HEADER], - pop_last=True), - columns=self.QID_HEADER) - dict_tracker['GROUP_HEADER'] = pd.DataFrame(self.utils.grab_section(report, - self.GROUP_HEADER, - end=[self.OWASP_HEADER], - pop_last=True), - columns=self.GROUP_HEADER) - dict_tracker['OWASP_HEADER'] = pd.DataFrame(self.utils.grab_section(report, - self.OWASP_HEADER, - end=[self.WASC_HEADER], - pop_last=True), - columns=self.OWASP_HEADER) - dict_tracker['WASC_HEADER'] = pd.DataFrame(self.utils.grab_section(report, - self.WASC_HEADER, end=[['APPENDIX']], - pop_last=True), - columns=self.WASC_HEADER) + dict_tracker['WEB_SCAN_VULN_BLOCK'] = pd.DataFrame(self.utils.grab_section(report, + self.WEB_SCAN_VULN_BLOCK, + end=[ + self.WEB_SCAN_SENSITIVE_BLOCK, + self.WEB_SCAN_INFO_BLOCK], + pop_last=True), + columns=self.WEB_SCAN_VULN_HEADER) + dict_tracker['WEB_SCAN_SENSITIVE_BLOCK'] = pd.DataFrame(self.utils.grab_section(report, + self.WEB_SCAN_SENSITIVE_BLOCK, + end=[ + self.WEB_SCAN_INFO_BLOCK, + self.WEB_SCAN_SENSITIVE_BLOCK], + pop_last=True), + columns=self.WEB_SCAN_SENSITIVE_HEADER) + dict_tracker['WEB_SCAN_INFO_BLOCK'] = pd.DataFrame(self.utils.grab_section(report, + self.WEB_SCAN_INFO_BLOCK, + end=[self.QID_HEADER], + pop_last=True), + columns=self.WEB_SCAN_INFO_HEADER) + dict_tracker['QID_HEADER'] = pd.DataFrame(self.utils.grab_section(report, + self.QID_HEADER, + end=[self.GROUP_HEADER], + pop_last=True), + columns=self.QID_HEADER) + dict_tracker['GROUP_HEADER'] = pd.DataFrame(self.utils.grab_section(report, + self.GROUP_HEADER, + end=[self.OWASP_HEADER], + pop_last=True), + columns=self.GROUP_HEADER) + dict_tracker['OWASP_HEADER'] = pd.DataFrame(self.utils.grab_section(report, + self.OWASP_HEADER, + end=[self.WASC_HEADER], + pop_last=True), + columns=self.OWASP_HEADER) + dict_tracker['WASC_HEADER'] = pd.DataFrame(self.utils.grab_section(report, + self.WASC_HEADER, end=[['APPENDIX']], + pop_last=True), + columns=self.WASC_HEADER) - dict_tracker['SCAN_META'] = pd.DataFrame(self.utils.grab_section(report, - self.SCAN_META, - end=[self.CATEGORY_HEADER], - pop_last=True), - columns=self.SCAN_META) + dict_tracker['SCAN_META'] = pd.DataFrame(self.utils.grab_section(report, + self.SCAN_META, + end=[self.CATEGORY_HEADER], + pop_last=True), + columns=self.SCAN_META) - dict_tracker['CATEGORY_HEADER'] = pd.DataFrame(self.utils.grab_section(report, - self.CATEGORY_HEADER), - columns=self.CATEGORY_HEADER) - all_dataframes.append(dict_tracker) + dict_tracker['CATEGORY_HEADER'] = pd.DataFrame(self.utils.grab_section(report, + self.CATEGORY_HEADER), + columns=self.CATEGORY_HEADER) + all_dataframes.append(dict_tracker) return all_dataframes From 73342fdeb8147ea630095d02851330929b405a86 Mon Sep 17 00:00:00 2001 From: nate Date: Tue, 3 Aug 2021 13:14:51 -0500 Subject: [PATCH 02/14] use get method for downloading report --- vulnwhisp/frameworks/qualys_web.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vulnwhisp/frameworks/qualys_web.py b/vulnwhisp/frameworks/qualys_web.py index 057bafd..71a4f55 100644 --- a/vulnwhisp/frameworks/qualys_web.py +++ b/vulnwhisp/frameworks/qualys_web.py @@ -136,7 +136,8 @@ class qualysWhisperAPI(object): return self.qgc.request(self.REPORT_STATUS.format(report_id=report_id)) def download_report(self, report_id): - return self.qgc.request(self.REPORT_DOWNLOAD.format(report_id=report_id)) + return self.qgc.request( + self.REPORT_DOWNLOAD.format(report_id=report_id), http_method='get') def generate_scan_report_XML(self, scan_id): """Generates a CSV report for an asset based on template defined in .ini file""" From 68519d56482ddc1b2b5aa6e8ed2be56a3a8241dd Mon Sep 17 00:00:00 2001 From: nate Date: Tue, 3 Aug 2021 13:15:14 -0500 Subject: [PATCH 03/14] fixed formatting --- vulnwhisp/frameworks/qualys_web.py | 106 ++++++++++++++++------------- 1 file changed, 58 insertions(+), 48 deletions(-) diff --git a/vulnwhisp/frameworks/qualys_web.py b/vulnwhisp/frameworks/qualys_web.py index 71a4f55..7c54bfb 100644 --- a/vulnwhisp/frameworks/qualys_web.py +++ b/vulnwhisp/frameworks/qualys_web.py @@ -353,54 +353,64 @@ class qualysScanReport: def grab_sections(self, report): all_dataframes = [] dict_tracker = {} - dict_tracker['WEB_SCAN_VULN_BLOCK'] = pd.DataFrame(self.utils.grab_section(report, - self.WEB_SCAN_VULN_BLOCK, - end=[ - self.WEB_SCAN_SENSITIVE_BLOCK, - self.WEB_SCAN_INFO_BLOCK], - pop_last=True), - columns=self.WEB_SCAN_VULN_HEADER) - dict_tracker['WEB_SCAN_SENSITIVE_BLOCK'] = pd.DataFrame(self.utils.grab_section(report, - self.WEB_SCAN_SENSITIVE_BLOCK, - end=[ - self.WEB_SCAN_INFO_BLOCK, - self.WEB_SCAN_SENSITIVE_BLOCK], - pop_last=True), - columns=self.WEB_SCAN_SENSITIVE_HEADER) - dict_tracker['WEB_SCAN_INFO_BLOCK'] = pd.DataFrame(self.utils.grab_section(report, - self.WEB_SCAN_INFO_BLOCK, - end=[self.QID_HEADER], - pop_last=True), - columns=self.WEB_SCAN_INFO_HEADER) - dict_tracker['QID_HEADER'] = pd.DataFrame(self.utils.grab_section(report, - self.QID_HEADER, - end=[self.GROUP_HEADER], - pop_last=True), - columns=self.QID_HEADER) - dict_tracker['GROUP_HEADER'] = pd.DataFrame(self.utils.grab_section(report, - self.GROUP_HEADER, - end=[self.OWASP_HEADER], - pop_last=True), - columns=self.GROUP_HEADER) - dict_tracker['OWASP_HEADER'] = pd.DataFrame(self.utils.grab_section(report, - self.OWASP_HEADER, - end=[self.WASC_HEADER], - pop_last=True), - columns=self.OWASP_HEADER) - dict_tracker['WASC_HEADER'] = pd.DataFrame(self.utils.grab_section(report, - self.WASC_HEADER, end=[['APPENDIX']], - pop_last=True), - columns=self.WASC_HEADER) - - dict_tracker['SCAN_META'] = pd.DataFrame(self.utils.grab_section(report, - self.SCAN_META, - end=[self.CATEGORY_HEADER], - pop_last=True), - columns=self.SCAN_META) - - dict_tracker['CATEGORY_HEADER'] = pd.DataFrame(self.utils.grab_section(report, - self.CATEGORY_HEADER), - columns=self.CATEGORY_HEADER) + dict_tracker['WEB_SCAN_VULN_BLOCK'] = pd.DataFrame( + self.utils.grab_section( + report, + self.WEB_SCAN_VULN_BLOCK, + end=[self.WEB_SCAN_SENSITIVE_BLOCK, self.WEB_SCAN_INFO_BLOCK], + pop_last=True), + columns=self.WEB_SCAN_VULN_HEADER) + dict_tracker['WEB_SCAN_SENSITIVE_BLOCK'] = pd.DataFrame( + self.utils.grab_section(report, + self.WEB_SCAN_SENSITIVE_BLOCK, + end=[self.WEB_SCAN_INFO_BLOCK, self.WEB_SCAN_SENSITIVE_BLOCK], + pop_last=True), + columns=self.WEB_SCAN_SENSITIVE_HEADER) + dict_tracker['WEB_SCAN_INFO_BLOCK'] = pd.DataFrame( + self.utils.grab_section( + report, + self.WEB_SCAN_INFO_BLOCK, + end=[self.QID_HEADER], + pop_last=True), + columns=self.WEB_SCAN_INFO_HEADER) + dict_tracker['QID_HEADER'] = pd.DataFrame( + self.utils.grab_section( + report, + self.QID_HEADER, + end=[self.GROUP_HEADER], + pop_last=True), + columns=self.QID_HEADER) + dict_tracker['GROUP_HEADER'] = pd.DataFrame( + self.utils.grab_section( + report, + self.GROUP_HEADER, + end=[self.OWASP_HEADER], + pop_last=True), + columns=self.GROUP_HEADER) + dict_tracker['OWASP_HEADER'] = pd.DataFrame( + self.utils.grab_section( + report, + self.OWASP_HEADER, + end=[self.WASC_HEADER], + pop_last=True), + columns=self.OWASP_HEADER) + dict_tracker['WASC_HEADER'] = pd.DataFrame( + self.utils.grab_section( + report, + self.WASC_HEADER, + end=[['APPENDIX']], + pop_last=True), + columns=self.WASC_HEADER) + dict_tracker['SCAN_META'] = pd.DataFrame( + self.utils.grab_section(report, + self.SCAN_META, + end=[self.CATEGORY_HEADER], + pop_last=True), + columns=self.SCAN_META) + dict_tracker['CATEGORY_HEADER'] = pd.DataFrame( + self.utils.grab_section(report, + self.CATEGORY_HEADER), + columns=self.CATEGORY_HEADER) all_dataframes.append(dict_tracker) return all_dataframes From 8403b351990ac8d6b3d1c4d31def9dd305fda3ce Mon Sep 17 00:00:00 2001 From: nate Date: Tue, 3 Aug 2021 13:58:24 -0500 Subject: [PATCH 04/14] increased size to sys max size --- vulnwhisp/frameworks/qualys_web.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vulnwhisp/frameworks/qualys_web.py b/vulnwhisp/frameworks/qualys_web.py index 7c54bfb..7b41e0b 100644 --- a/vulnwhisp/frameworks/qualys_web.py +++ b/vulnwhisp/frameworks/qualys_web.py @@ -17,6 +17,7 @@ import os import csv import logging import dateutil.parser as dp +csv.field_size_limit(sys.maxsize) class qualysWhisperAPI(object): From 13a52a3e086348708a71181d483ae1e3d5d21016 Mon Sep 17 00:00:00 2001 From: nate Date: Tue, 3 Aug 2021 13:59:37 -0500 Subject: [PATCH 05/14] formatting --- vulnwhisp/frameworks/qualys_web.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/vulnwhisp/frameworks/qualys_web.py b/vulnwhisp/frameworks/qualys_web.py index 7b41e0b..5b4b563 100644 --- a/vulnwhisp/frameworks/qualys_web.py +++ b/vulnwhisp/frameworks/qualys_web.py @@ -259,13 +259,7 @@ class qualysUtils: def __init__(self): self.logger = logging.getLogger('qualysUtils') - def grab_section( - self, - report, - section, - end=[], - pop_last=False, - ): + def grab_section(self, report, section, end=[], pop_last=False): temp_list = [] max_col_count = 0 with open(report, 'rt') as csvfile: From 649ecd431b954dd2abf9ee7c20b5f32ae142f5cc Mon Sep 17 00:00:00 2001 From: nate Date: Tue, 3 Aug 2021 14:01:38 -0500 Subject: [PATCH 06/14] moved qualysReportFields class into qualysScanReport; it only consists of constants and they are unused outside of qualysScanReport --- vulnwhisp/frameworks/qualys_web.py | 136 ++++++++--------------------- 1 file changed, 38 insertions(+), 98 deletions(-) diff --git a/vulnwhisp/frameworks/qualys_web.py b/vulnwhisp/frameworks/qualys_web.py index 5b4b563..4093174 100644 --- a/vulnwhisp/frameworks/qualys_web.py +++ b/vulnwhisp/frameworks/qualys_web.py @@ -180,81 +180,6 @@ class qualysWhisperAPI(object): def delete_report(self, report_id): return self.qgc.request(self.DELETE_REPORT.format(report_id=report_id)) - -class qualysReportFields: - CATEGORIES = ['VULNERABILITY', - 'SENSITIVECONTENT', - 'INFORMATION_GATHERED'] - - # URL Vulnerability Information - - VULN_BLOCK = [ - CATEGORIES[0], - 'ID', - 'QID', - 'Url', - 'Param', - 'Function', - 'Form Entry Point', - 'Access Path', - 'Authentication', - 'Ajax Request', - 'Ajax Request ID', - 'Ignored', - 'Ignore Reason', - 'Ignore Date', - 'Ignore User', - 'Ignore Comments', - 'First Time Detected', - 'Last Time Detected', - 'Last Time Tested', - 'Times Detected', - 'Payload #1', - 'Request Method #1', - 'Request URL #1', - 'Request Headers #1', - 'Response #1', - 'Evidence #1', - ] - - INFO_HEADER = [ - 'Vulnerability Category', - 'ID', - 'QID', - 'Response #1', - 'Last Time Detected', - ] - INFO_BLOCK = [ - CATEGORIES[2], - 'ID', - 'QID', - 'Results', - 'Detection Date', - ] - - QID_HEADER = [ - 'QID', - 'Id', - 'Title', - 'Category', - 'Severity Level', - 'Groups', - 'OWASP', - 'WASC', - 'CWE', - 'CVSS Base', - 'CVSS Temporal', - 'Description', - 'Impact', - 'Solution', - ] - GROUP_HEADER = ['GROUP', 'Name', 'Category'] - OWASP_HEADER = ['OWASP', 'Code', 'Name'] - WASC_HEADER = ['WASC', 'Code', 'Name'] - SCAN_META = ['Web Application Name', 'URL', 'Owner', 'Scope', 'Operating System'] - CATEGORY_HEADER = ['Category', 'Severity', 'Level', 'Description'] - - class qualysUtils: def __init__(self): self.logger = logging.getLogger('qualysUtils') @@ -288,35 +213,50 @@ class qualysUtils: return _data class qualysScanReport: - # URL Vulnerability Information - WEB_SCAN_VULN_BLOCK = list(qualysReportFields.VULN_BLOCK) - WEB_SCAN_VULN_BLOCK.insert(WEB_SCAN_VULN_BLOCK.index('QID'), 'Detection ID') + CATEGORIES = ['VULNERABILITY', 'SENSITIVECONTENT', 'INFORMATION_GATHERED'] - WEB_SCAN_VULN_HEADER = list(WEB_SCAN_VULN_BLOCK) - WEB_SCAN_VULN_HEADER[WEB_SCAN_VULN_BLOCK.index(qualysReportFields.CATEGORIES[0])] = \ - 'Vulnerability Category' + WEB_SCAN_BLOCK = [ + "ID", "Detection ID", "QID", "Url", "Param/Cookie", "Function", + "Form Entry Point", "Access Path", "Authentication", "Ajax Request", + "Ajax Request ID", "Ignored", "Ignore Reason", "Ignore Date", "Ignore User", + "Ignore Comments", "Detection Date", "Payload #1", "Request Method #1", + "Request URL #1", "Request Headers #1", "Response #1", "Evidence #1", + "Unique ID", "Flags", "Protocol", "Virtual Host", "IP", "Port", "Result", + "Info#1", "CVSS V3 Base", "CVSS V3 Temporal", "CVSS V3 Attack Vector", + "Request Body #1" + ] - WEB_SCAN_SENSITIVE_HEADER = list(WEB_SCAN_VULN_HEADER) - WEB_SCAN_SENSITIVE_HEADER.insert(WEB_SCAN_SENSITIVE_HEADER.index('Url' - ), 'Content') + WEB_SCAN_VULN_BLOCK = [CATEGORIES[0]] + WEB_SCAN_BLOCK + WEB_SCAN_VULN_HEADER = WEB_SCAN_VULN_BLOCK - WEB_SCAN_SENSITIVE_BLOCK = list(WEB_SCAN_SENSITIVE_HEADER) - WEB_SCAN_SENSITIVE_BLOCK.insert(WEB_SCAN_SENSITIVE_BLOCK.index('QID'), 'Detection ID') - WEB_SCAN_SENSITIVE_BLOCK[WEB_SCAN_SENSITIVE_BLOCK.index('Vulnerability Category' - )] = qualysReportFields.CATEGORIES[1] + WEB_SCAN_SENSITIVE_BLOCK = [CATEGORIES[1]] + WEB_SCAN_BLOCK + WEB_SCAN_SENSITIVE_HEADER = WEB_SCAN_SENSITIVE_BLOCK - WEB_SCAN_INFO_HEADER = list(qualysReportFields.INFO_HEADER) - WEB_SCAN_INFO_HEADER.insert(WEB_SCAN_INFO_HEADER.index('QID'), 'Detection ID') + WEB_SCAN_INFO_BLOCK = [ + "INFORMATION_GATHERED", "ID", "Detection ID", "QID", "Results", "Detection Date", + "Unique ID", "Flags", "Protocol", "Virtual Host", "IP", "Port", "Result", + "Info#1" + ] - WEB_SCAN_INFO_BLOCK = list(qualysReportFields.INFO_BLOCK) - WEB_SCAN_INFO_BLOCK.insert(WEB_SCAN_INFO_BLOCK.index('QID'), 'Detection ID') + WEB_SCAN_INFO_HEADER = [ + "Vulnerability Category", "ID", "Detection ID", "QID", "Results", "Detection Date", + "Unique ID", "Flags", "Protocol", "Virtual Host", "IP", "Port", "Result", + "Info#1" + ] - QID_HEADER = list(qualysReportFields.QID_HEADER) - GROUP_HEADER = list(qualysReportFields.GROUP_HEADER) - OWASP_HEADER = list(qualysReportFields.OWASP_HEADER) - WASC_HEADER = list(qualysReportFields.WASC_HEADER) - SCAN_META = list(qualysReportFields.SCAN_META) - CATEGORY_HEADER = list(qualysReportFields.CATEGORY_HEADER) + QID_HEADER = [ + "QID", "Id", "Title", "Category", "Severity Level", "Groups", "OWASP", "WASC", + "CWE", "CVSS Base", "CVSS Temporal", "Description", "Impact", "Solution", + "CVSS V3 Base", "CVSS V3 Temporal", "CVSS V3 Attack Vector" + ] + GROUP_HEADER = ['GROUP', 'Name', 'Category'] + OWASP_HEADER = ['OWASP', 'Code', 'Name'] + WASC_HEADER = ['WASC', 'Code', 'Name'] + SCAN_META = [ + "Web Application Name", "URL", "Owner", "Scope", "ID", "Tags", + "Custom Attributes" + ] + CATEGORY_HEADER = ['Category', 'Severity', 'Level', 'Description'] def __init__( self, From 3db931f3eb8e7ec0ed931cc24847d03fe6447971 Mon Sep 17 00:00:00 2001 From: nate Date: Tue, 3 Aug 2021 14:07:34 -0500 Subject: [PATCH 07/14] removed unused constants --- vulnwhisp/frameworks/qualys_web.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/vulnwhisp/frameworks/qualys_web.py b/vulnwhisp/frameworks/qualys_web.py index 4093174..d74a1a3 100644 --- a/vulnwhisp/frameworks/qualys_web.py +++ b/vulnwhisp/frameworks/qualys_web.py @@ -21,21 +21,12 @@ csv.field_size_limit(sys.maxsize) class qualysWhisperAPI(object): - COUNT_WEBAPP = '/count/was/webapp' COUNT_WASSCAN = '/count/was/wasscan' DELETE_REPORT = '/delete/was/report/{report_id}' - GET_WEBAPP_DETAILS = '/get/was/webapp/{was_id}' - QPS_REST_3 = '/qps/rest/3.0' - REPORT_DETAILS = '/get/was/report/{report_id}' REPORT_STATUS = '/status/was/report/{report_id}' REPORT_CREATE = '/create/was/report' REPORT_DOWNLOAD = '/download/was/report/{report_id}' - SCAN_DETAILS = '/get/was/wasscan/{scan_id}' - SCAN_DOWNLOAD = '/download/was/wasscan/{scan_id}' - SEARCH_REPORTS = '/search/was/report' - SEARCH_WEB_APPS = '/search/was/webapp' SEARCH_WAS_SCAN = '/search/was/wasscan' - VERSION = '/qps/rest/portal/version' def __init__(self, config=None): self.logger = logging.getLogger('qualysWhisperAPI') From 5dad1ceb1087ba1de10867af188156856ad827b1 Mon Sep 17 00:00:00 2001 From: nate Date: Tue, 3 Aug 2021 14:25:15 -0500 Subject: [PATCH 08/14] removed commented code --- vulnwhisp/frameworks/qualys_web.py | 1 - 1 file changed, 1 deletion(-) diff --git a/vulnwhisp/frameworks/qualys_web.py b/vulnwhisp/frameworks/qualys_web.py index d74a1a3..0e92346 100644 --- a/vulnwhisp/frameworks/qualys_web.py +++ b/vulnwhisp/frameworks/qualys_web.py @@ -37,7 +37,6 @@ class qualysWhisperAPI(object): except Exception as e: self.logger.error('Could not connect to Qualys: {}'.format(str(e))) self.headers = { - #"content-type": "text/xml"} "Accept" : "application/json", "Content-Type": "application/json"} self.config_parse = qcconf.QualysConnectConfig(config, 'qualys_web') From 51234a569fdab0931e11976b54d742f113ddc327 Mon Sep 17 00:00:00 2001 From: nate Date: Tue, 3 Aug 2021 14:28:13 -0500 Subject: [PATCH 09/14] cleaned newline formatting --- vulnwhisp/frameworks/qualys_web.py | 36 +++++++++--------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/vulnwhisp/frameworks/qualys_web.py b/vulnwhisp/frameworks/qualys_web.py index 0e92346..78c6386 100644 --- a/vulnwhisp/frameworks/qualys_web.py +++ b/vulnwhisp/frameworks/qualys_web.py @@ -63,14 +63,8 @@ class qualysWhisperAPI(object): def generate_scan_result_XML(self, limit=1000, offset=1, status='FINISHED'): report_xml = E.ServiceRequest( - E.filters( - E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status - ), - ), - E.preferences( - E.startFromOffset(str(offset)), - E.limitResults(str(limit)) - ), + E.filters(E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status)), + E.preferences(E.startFromOffset(str(offset)), E.limitResults(str(limit))), ) return report_xml @@ -109,8 +103,10 @@ class qualysWhisperAPI(object): if i % limit == 0: if (total - i) < limit: qualys_api_limit = total - i - self.logger.info('Making a request with a limit of {} at offset {}'.format((str(qualys_api_limit)), str(i + 1))) - scan_info = self.get_scan_info(limit=qualys_api_limit, offset=i + 1, status=status) + self.logger.info('Making a request with a limit of {} at offset {}' + .format((str(qualys_api_limit)), str(i + 1))) + scan_info = self.get_scan_info( + limit=qualys_api_limit, offset=i + 1, status=status) _records.append(scan_info) self.logger.debug('Converting XML to DataFrame') dataframes = [self.xml_parser(xml) for xml in _records] @@ -140,20 +136,8 @@ class qualysWhisperAPI(object): E.format('CSV'), #type is not needed, as the template already has it E.type('WAS_SCAN_REPORT'), - E.template( - E.id(self.template_id) - ), - E.config( - E.scanReport( - E.target( - E.scans( - E.WasScan( - E.id(scan_id) - ) - ), - ), - ), - ) + E.template(E.id(self.template_id)), + E.config(E.scanReport(E.target(E.scans(E.WasScan(E.id(scan_id)))))) ) ) ) @@ -266,7 +250,9 @@ class qualysScanReport: try: self.qw = qualysWhisperAPI(config=config) except Exception as e: - self.logger.error('Could not load config! Please check settings. Error: {}'.format(str(e))) + self.logger.error( + 'Could not load config! Please check settings. Error: {}'.format( + str(e))) if file_stream: self.open_file = file_in.splitlines() From 742a645190341239918d0daa0799219fd188742b Mon Sep 17 00:00:00 2001 From: nate Date: Tue, 3 Aug 2021 14:29:00 -0500 Subject: [PATCH 10/14] moved dict_tracker assignments into creation --- vulnwhisp/frameworks/qualys_web.py | 39 ++++++++++++++++-------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/vulnwhisp/frameworks/qualys_web.py b/vulnwhisp/frameworks/qualys_web.py index 78c6386..b4105f4 100644 --- a/vulnwhisp/frameworks/qualys_web.py +++ b/vulnwhisp/frameworks/qualys_web.py @@ -263,67 +263,70 @@ class qualysScanReport: def grab_sections(self, report): all_dataframes = [] - dict_tracker = {} - dict_tracker['WEB_SCAN_VULN_BLOCK'] = pd.DataFrame( + dict_tracker = { + 'WEB_SCAN_VULN_BLOCK': pd.DataFrame( self.utils.grab_section( report, self.WEB_SCAN_VULN_BLOCK, end=[self.WEB_SCAN_SENSITIVE_BLOCK, self.WEB_SCAN_INFO_BLOCK], pop_last=True), - columns=self.WEB_SCAN_VULN_HEADER) - dict_tracker['WEB_SCAN_SENSITIVE_BLOCK'] = pd.DataFrame( + columns=self.WEB_SCAN_VULN_HEADER), + 'WEB_SCAN_SENSITIVE_BLOCK': pd.DataFrame( self.utils.grab_section(report, self.WEB_SCAN_SENSITIVE_BLOCK, end=[self.WEB_SCAN_INFO_BLOCK, self.WEB_SCAN_SENSITIVE_BLOCK], pop_last=True), - columns=self.WEB_SCAN_SENSITIVE_HEADER) - dict_tracker['WEB_SCAN_INFO_BLOCK'] = pd.DataFrame( + columns=self.WEB_SCAN_SENSITIVE_HEADER), + 'WEB_SCAN_INFO_BLOCK': pd.DataFrame( self.utils.grab_section( report, self.WEB_SCAN_INFO_BLOCK, end=[self.QID_HEADER], pop_last=True), - columns=self.WEB_SCAN_INFO_HEADER) - dict_tracker['QID_HEADER'] = pd.DataFrame( + columns=self.WEB_SCAN_INFO_HEADER), + + 'QID_HEADER': pd.DataFrame( self.utils.grab_section( report, self.QID_HEADER, end=[self.GROUP_HEADER], pop_last=True), - columns=self.QID_HEADER) - dict_tracker['GROUP_HEADER'] = pd.DataFrame( + columns=self.QID_HEADER), + 'GROUP_HEADER': pd.DataFrame( self.utils.grab_section( report, self.GROUP_HEADER, end=[self.OWASP_HEADER], pop_last=True), - columns=self.GROUP_HEADER) - dict_tracker['OWASP_HEADER'] = pd.DataFrame( + columns=self.GROUP_HEADER), + 'OWASP_HEADER': pd.DataFrame( self.utils.grab_section( report, self.OWASP_HEADER, end=[self.WASC_HEADER], pop_last=True), - columns=self.OWASP_HEADER) - dict_tracker['WASC_HEADER'] = pd.DataFrame( + columns=self.OWASP_HEADER), + 'WASC_HEADER': pd.DataFrame( self.utils.grab_section( report, self.WASC_HEADER, end=[['APPENDIX']], pop_last=True), - columns=self.WASC_HEADER) - dict_tracker['SCAN_META'] = pd.DataFrame( + columns=self.WASC_HEADER), + 'SCAN_META': pd.DataFrame( self.utils.grab_section(report, self.SCAN_META, end=[self.CATEGORY_HEADER], pop_last=True), - columns=self.SCAN_META) - dict_tracker['CATEGORY_HEADER'] = pd.DataFrame( + columns=self.SCAN_META), + 'CATEGORY_HEADER': pd.DataFrame( self.utils.grab_section(report, self.CATEGORY_HEADER), columns=self.CATEGORY_HEADER) + } all_dataframes.append(dict_tracker) + return all_dataframes def data_normalizer(self, dataframes): From 61539afa4d4e17cf6afa9190021adfe53a97a2f0 Mon Sep 17 00:00:00 2001 From: nate Date: Tue, 3 Aug 2021 16:26:37 -0500 Subject: [PATCH 11/14] headers is unused --- vulnwhisp/frameworks/qualys_web.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/vulnwhisp/frameworks/qualys_web.py b/vulnwhisp/frameworks/qualys_web.py index b4105f4..2978656 100644 --- a/vulnwhisp/frameworks/qualys_web.py +++ b/vulnwhisp/frameworks/qualys_web.py @@ -36,9 +36,6 @@ class qualysWhisperAPI(object): self.logger.info('Connected to Qualys at {}'.format(self.qgc.server)) except Exception as e: self.logger.error('Could not connect to Qualys: {}'.format(str(e))) - self.headers = { - "Accept" : "application/json", - "Content-Type": "application/json"} self.config_parse = qcconf.QualysConnectConfig(config, 'qualys_web') try: self.template_id = self.config_parse.get_template_id() From ff5f4cb33179d85343fee08efdfd6efb8341ebd7 Mon Sep 17 00:00:00 2001 From: nate Date: Tue, 3 Aug 2021 16:39:24 -0500 Subject: [PATCH 12/14] renamed and cleaned columns --- vulnwhisp/frameworks/qualys_web.py | 48 +++++++++++++++++------------- vulnwhisp/vulnwhisp.py | 13 +++++++- 2 files changed, 39 insertions(+), 22 deletions(-) diff --git a/vulnwhisp/frameworks/qualys_web.py b/vulnwhisp/frameworks/qualys_web.py index 2978656..1c9d122 100644 --- a/vulnwhisp/frameworks/qualys_web.py +++ b/vulnwhisp/frameworks/qualys_web.py @@ -196,12 +196,11 @@ class qualysScanReport: "Info#1", "CVSS V3 Base", "CVSS V3 Temporal", "CVSS V3 Attack Vector", "Request Body #1" ] - WEB_SCAN_VULN_BLOCK = [CATEGORIES[0]] + WEB_SCAN_BLOCK - WEB_SCAN_VULN_HEADER = WEB_SCAN_VULN_BLOCK - WEB_SCAN_SENSITIVE_BLOCK = [CATEGORIES[1]] + WEB_SCAN_BLOCK - WEB_SCAN_SENSITIVE_HEADER = WEB_SCAN_SENSITIVE_BLOCK + + WEB_SCAN_HEADER = ["Vulnerability Category"] + WEB_SCAN_BLOCK + WEB_SCAN_INFO_BLOCK = [ "INFORMATION_GATHERED", "ID", "Detection ID", "QID", "Results", "Detection Date", @@ -210,7 +209,7 @@ class qualysScanReport: ] WEB_SCAN_INFO_HEADER = [ - "Vulnerability Category", "ID", "Detection ID", "QID", "Results", "Detection Date", + "Vulnerability Category", "ID", "Detection ID", "QID", "Results", "Last Time Detected", "Unique ID", "Flags", "Protocol", "Virtual Host", "IP", "Port", "Result", "Info#1" ] @@ -259,21 +258,20 @@ class qualysScanReport: self.downloaded_file = None def grab_sections(self, report): - all_dataframes = [] - dict_tracker = { + return { 'WEB_SCAN_VULN_BLOCK': pd.DataFrame( self.utils.grab_section( report, self.WEB_SCAN_VULN_BLOCK, end=[self.WEB_SCAN_SENSITIVE_BLOCK, self.WEB_SCAN_INFO_BLOCK], pop_last=True), - columns=self.WEB_SCAN_VULN_HEADER), + columns=self.WEB_SCAN_HEADER), 'WEB_SCAN_SENSITIVE_BLOCK': pd.DataFrame( self.utils.grab_section(report, self.WEB_SCAN_SENSITIVE_BLOCK, end=[self.WEB_SCAN_INFO_BLOCK, self.WEB_SCAN_SENSITIVE_BLOCK], pop_last=True), - columns=self.WEB_SCAN_SENSITIVE_HEADER), + columns=self.WEB_SCAN_HEADER), 'WEB_SCAN_INFO_BLOCK': pd.DataFrame( self.utils.grab_section( report, @@ -321,10 +319,6 @@ class qualysScanReport: self.CATEGORY_HEADER), columns=self.CATEGORY_HEADER) } - all_dataframes.append(dict_tracker) - - - return all_dataframes def data_normalizer(self, dataframes): """ @@ -332,12 +326,21 @@ class qualysScanReport: :param dataframes: :return: """ - df_dict = dataframes[0] - merged_df = pd.concat([df_dict['WEB_SCAN_VULN_BLOCK'], df_dict['WEB_SCAN_SENSITIVE_BLOCK'], - df_dict['WEB_SCAN_INFO_BLOCK']], axis=0, - ignore_index=False) - merged_df = pd.merge(merged_df, df_dict['QID_HEADER'], left_on='QID', - right_on='Id') + df_dict = dataframes + merged_df = pd.concat([ + df_dict['WEB_SCAN_VULN_BLOCK'], + df_dict['WEB_SCAN_SENSITIVE_BLOCK'], + df_dict['WEB_SCAN_INFO_BLOCK'] + ], axis=0, ignore_index=False) + + merged_df = pd.merge( + merged_df, + df_dict['QID_HEADER'].drop( + #these columns always seem to be the same as what we're merging into + ['CVSS V3 Attack Vector', 'CVSS V3 Base', 'CVSS V3 Temporal'], + axis=1), + left_on='QID', right_on='Id' + ) if 'Content' not in merged_df: merged_df['Content'] = '' @@ -354,8 +357,11 @@ class qualysScanReport: merged_df = merged_df.assign(**df_dict['SCAN_META'].to_dict(orient='records')[0]) - merged_df = pd.merge(merged_df, df_dict['CATEGORY_HEADER'], how='left', left_on=['Category', 'Severity Level'], - right_on=['Category', 'Severity'], suffixes=('Severity', 'CatSev')) + merged_df = pd.merge( + merged_df, df_dict['CATEGORY_HEADER'], + how='left', left_on=['Category', 'Severity Level'], + right_on=['Category', 'Severity'], suffixes=('Severity', 'CatSev') + ) merged_df = merged_df.replace('N/A', '').fillna('') diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index 795826b..998763e 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -530,10 +530,14 @@ class vulnWhispererQualys(vulnWhispererBase): 'Ajax Request ID': 'ajax_request_id', 'Authentication': 'authentication', 'CVSS Base': 'cvss', + 'CVSS V3 Attack Vector': 'cvss_v3_attack_vector', + 'CVSS V3 Base': 'cvss_v3_base', + 'CVSS V3 Temporal': 'cvss_v3_temporal', 'CVSS Temporal': 'cvss_temporal', 'CWE': 'cwe', 'Category': 'category', 'Content': 'content', + 'Custom Attributes': 'custom_attributes', 'DescriptionSeverity': 'severity_description', 'DescriptionCatSev': 'category_description', 'Detection ID': 'detection_id', @@ -549,15 +553,19 @@ class vulnWhispererQualys(vulnWhispererBase): 'Ignore User': 'ignore_user', 'Ignored': 'ignored', 'Impact': 'impact', + 'Info#1': 'info_1', 'Last Time Detected': 'last_time_detected', 'Last Time Tested': 'last_time_tested', 'Level': 'level', 'OWASP': 'owasp', 'Operating System': 'operating_system', 'Owner': 'owner', - 'Param': 'param', + 'Param/Cookie': 'param', 'Payload #1': 'payload_1', + 'Port': 'port', + 'Protocol': 'protocol', 'QID': 'plugin_id', + 'Request Body #1': 'request_body_1', 'Request Headers #1': 'request_headers_1', 'Request Method #1': 'request_method_1', 'Request URL #1': 'request_url_1', @@ -566,11 +574,14 @@ class vulnWhispererQualys(vulnWhispererBase): 'Severity': 'risk', 'Severity Level': 'security_level', 'Solution': 'solution', + 'Tags': 'tags', 'Times Detected': 'times_detected', 'Title': 'plugin_name', 'URL': 'url', + 'Unique ID': 'unique_id', 'Url': 'uri', 'Vulnerability Category': 'vulnerability_category', + 'Virtual Host': 'virutal_host', 'WASC': 'wasc', 'Web Application Name': 'web_application_name'} From 273b17009ae1cb3fb95693d0acccfa81284f273a Mon Sep 17 00:00:00 2001 From: nate Date: Tue, 3 Aug 2021 16:39:58 -0500 Subject: [PATCH 13/14] renamed detection date to last time detected --- vulnwhisp/frameworks/qualys_web.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vulnwhisp/frameworks/qualys_web.py b/vulnwhisp/frameworks/qualys_web.py index 1c9d122..a722ebe 100644 --- a/vulnwhisp/frameworks/qualys_web.py +++ b/vulnwhisp/frameworks/qualys_web.py @@ -200,6 +200,7 @@ class qualysScanReport: WEB_SCAN_SENSITIVE_BLOCK = [CATEGORIES[1]] + WEB_SCAN_BLOCK WEB_SCAN_HEADER = ["Vulnerability Category"] + WEB_SCAN_BLOCK + WEB_SCAN_HEADER[WEB_SCAN_HEADER.index("Detection Date")] = "Last Time Detected" WEB_SCAN_INFO_BLOCK = [ From 54fa0ace8a553563d01d1dfb4819bcb5f87d26a2 Mon Sep 17 00:00:00 2001 From: nate Date: Tue, 3 Aug 2021 16:40:14 -0500 Subject: [PATCH 14/14] formatting --- vulnwhisp/frameworks/qualys_web.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/vulnwhisp/frameworks/qualys_web.py b/vulnwhisp/frameworks/qualys_web.py index a722ebe..8ce5f3e 100644 --- a/vulnwhisp/frameworks/qualys_web.py +++ b/vulnwhisp/frameworks/qualys_web.py @@ -229,14 +229,8 @@ class qualysScanReport: ] CATEGORY_HEADER = ['Category', 'Severity', 'Level', 'Description'] - def __init__( - self, - config=None, - file_in=None, - file_stream=False, - delimiter=',', - quotechar='"', - ): + def __init__(self, config=None, file_in=None, + file_stream=False, delimiter=',', quotechar='"'): self.logger = logging.getLogger('qualysScanReport') self.file_in = file_in self.file_stream = file_stream