diff --git a/vulnwhisp/frameworks/qualys.py b/vulnwhisp/frameworks/qualys.py index dcff1af..1d425f2 100644 --- a/vulnwhisp/frameworks/qualys.py +++ b/vulnwhisp/frameworks/qualys.py @@ -358,7 +358,7 @@ class qualysUtils: return dp.parse(dt).strftime('%s') def cleanser(self, _data): - repls = (('\n', '|||'), ('\r', '|||'), (',', ';'), ('\t', '|||'), ('""', "''")) + repls = (('\n', '|||'), ('\r', '|||'), (',', ';'), ('\t', '|||')) if _data: _data = reduce(lambda a, kv: a.replace(*kv), repls, str(_data)) return _data @@ -445,46 +445,40 @@ class qualysWebAppReport: def grab_sections(self, report): all_dataframes = [] + dict_tracker = {} with open(report, 'rb') as csvfile: - all_dataframes.append(pd.DataFrame(self.utils.grab_section(report, + dict_tracker['WEB_APP_VULN_BLOCK'] = pd.DataFrame(self.utils.grab_section(report, self.WEB_APP_VULN_BLOCK, end=[self.WEB_APP_SENSITIVE_BLOCK, self.WEB_APP_INFO_BLOCK], - pop_last=True), - columns=self.WEB_APP_VULN_HEADER)) - all_dataframes.append(pd.DataFrame(self.utils.grab_section(report, + pop_last=True), columns=self.WEB_APP_VULN_HEADER) + dict_tracker['WEB_APP_SENSITIVE_BLOCK'] = pd.DataFrame(self.utils.grab_section(report, self.WEB_APP_SENSITIVE_BLOCK, end=[self.WEB_APP_INFO_BLOCK, self.WEB_APP_SENSITIVE_BLOCK], - pop_last=True), - columns=self.WEB_APP_SENSITIVE_HEADER)) - all_dataframes.append(pd.DataFrame(self.utils.grab_section(report, + pop_last=True), columns=self.WEB_APP_SENSITIVE_HEADER) + dict_tracker['WEB_APP_INFO_BLOCK'] = pd.DataFrame(self.utils.grab_section(report, self.WEB_APP_INFO_BLOCK, end=[self.QID_HEADER], - pop_last=True), - columns=self.WEB_APP_INFO_HEADER)) - all_dataframes.append(pd.DataFrame(self.utils.grab_section(report, + pop_last=True), columns=self.WEB_APP_INFO_HEADER) + dict_tracker['QID_HEADER'] = pd.DataFrame(self.utils.grab_section(report, self.QID_HEADER, end=[self.GROUP_HEADER], - pop_last=True), - columns=self.QID_HEADER)) - all_dataframes.append(pd.DataFrame(self.utils.grab_section(report, + pop_last=True), columns=self.QID_HEADER) + dict_tracker['GROUP_HEADER'] = pd.DataFrame(self.utils.grab_section(report, self.GROUP_HEADER, end=[self.OWASP_HEADER], - pop_last=True), - columns=self.GROUP_HEADER)) - all_dataframes.append(pd.DataFrame(self.utils.grab_section(report, + pop_last=True), columns=self.GROUP_HEADER) + dict_tracker['OWASP_HEADER'] = pd.DataFrame(self.utils.grab_section(report, self.OWASP_HEADER, end=[self.WASC_HEADER], - pop_last=True), - columns=self.OWASP_HEADER)) - all_dataframes.append(pd.DataFrame(self.utils.grab_section(report, + pop_last=True), columns=self.OWASP_HEADER) + dict_tracker['WASC_HEADER'] = pd.DataFrame(self.utils.grab_section(report, self.WASC_HEADER, end=[['APPENDIX']], - pop_last=True), - columns=self.WASC_HEADER)) - all_dataframes.append(pd.DataFrame(self.utils.grab_section(report, - self.CATEGORY_HEADER), - columns=self.CATEGORY_HEADER)) + pop_last=True), columns=self.WASC_HEADER) + dict_tracker['CATEGORY_HEADER'] =pd.DataFrame(self.utils.grab_section(report, + self.CATEGORY_HEADER), columns=self.CATEGORY_HEADER) + all_dataframes.append(dict_tracker) return all_dataframes @@ -494,6 +488,13 @@ class qualysWebAppReport: :param dataframes: :return: """ + df_dict = dataframes[0] + merged_df = pd.concat([df_dict['WEB_APP_VULN_BLOCK'], df_dict['WEB_APP_SENSITIVE_BLOCK'], + df_dict['WEB_APP_INFO_BLOCK']], axis=0, + ignore_index=False) + + merged_df = pd.merge(merged_df, df_dict['QID_HEADER'], left_on='QID', + right_on='Id') merged_df = pd.concat([dataframes[0], dataframes[1], dataframes[2]], axis=0, @@ -508,9 +509,10 @@ class qualysWebAppReport: 'Request Headers #1', 'Response #1', 'Evidence #1', 'Description', 'Impact', 'Solution', 'Url', 'Content'] - #for col in columns_to_cleanse: - # merged_df[col] = merged_df[col].astype(str).apply(self.utils.cleanser) + for col in columns_to_cleanse: + merged_df[col] = merged_df[col].astype(str).apply(self.utils.cleanser) + merged_df = pd.merge(merged_df, df_dict['CATEGORY_HEADER']) merged_df = merged_df.drop(['QID_y', 'QID_x'], axis=1) merged_df = merged_df.rename(columns={'Id': 'QID'}) merged_df = merged_df.replace('N/A','').fillna('') @@ -673,41 +675,41 @@ class qualysScanReport: self.WEB_SCAN_INFO_BLOCK, self.WEB_SCAN_SENSITIVE_BLOCK], pop_last=True), - columns=self.WEB_SCAN_SENSITIVE_HEADER) + columns=self.WEB_SCAN_SENSITIVE_HEADER) dict_tracker['WEB_SCAN_INFO_BLOCK'] = pd.DataFrame(self.utils.grab_section(report, self.WEB_SCAN_INFO_BLOCK, end=[self.QID_HEADER], pop_last=True), - columns=self.WEB_SCAN_INFO_HEADER) + columns=self.WEB_SCAN_INFO_HEADER) dict_tracker['QID_HEADER'] = pd.DataFrame(self.utils.grab_section(report, self.QID_HEADER, end=[self.GROUP_HEADER], pop_last=True), - columns=self.QID_HEADER) + columns=self.QID_HEADER) dict_tracker['GROUP_HEADER'] = pd.DataFrame(self.utils.grab_section(report, self.GROUP_HEADER, end=[self.OWASP_HEADER], pop_last=True), - columns=self.GROUP_HEADER) + columns=self.GROUP_HEADER) dict_tracker['OWASP_HEADER'] = pd.DataFrame(self.utils.grab_section(report, self.OWASP_HEADER, end=[self.WASC_HEADER], pop_last=True), - columns=self.OWASP_HEADER) + columns=self.OWASP_HEADER) dict_tracker['WASC_HEADER'] = pd.DataFrame(self.utils.grab_section(report, self.WASC_HEADER, end=[['APPENDIX']], pop_last=True), - columns=self.WASC_HEADER) + columns=self.WASC_HEADER) dict_tracker['SCAN_META'] = pd.DataFrame(self.utils.grab_section(report, self.SCAN_META, end=[self.CATEGORY_HEADER], pop_last=True), - columns=self.SCAN_META) + columns=self.SCAN_META) dict_tracker['CATEGORY_HEADER'] = pd.DataFrame(self.utils.grab_section(report, self.CATEGORY_HEADER), - columns=self.CATEGORY_HEADER) + columns=self.CATEGORY_HEADER) all_dataframes.append(dict_tracker) return all_dataframes diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index 2da1577..07f6bcd 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -33,6 +33,7 @@ class vulnWhispererBase(object): username=None, password=None, section=None, + develop=False, ): @@ -41,6 +42,7 @@ class vulnWhispererBase(object): self.db_name = db_name self.purge = purge + self.develop = develop if config is not None: self.config = vwConfig(config_in=config) @@ -163,14 +165,13 @@ class vulnWhispererNessus(vulnWhispererBase): ): super(vulnWhispererNessus, self).__init__(config=config) - self.port = int(self.config.get(self.CONFIG_NAME, 'port')) + self.port = int(self.config.get(self.CONFIG_SECTION, 'port')) self.develop = True self.purge = purge if config is not None: try: - #if self.enabled: self.nessus_port = self.config.get(self.CONFIG_SECTION, 'port') self.nessus_trash = self.config.getbool(self.CONFIG_SECTION, @@ -325,7 +326,7 @@ class vulnWhispererNessus(vulnWhispererBase): file_name, time.time(), csv_in.shape[0], - 'nessus', + self.CONFIG_SECTION, uuid, 1, ) @@ -361,7 +362,7 @@ class vulnWhispererNessus(vulnWhispererBase): file_name, time.time(), clean_csv.shape[0], - 'nessus', + self.CONFIG_SECTION, uuid, 1, ) @@ -378,7 +379,7 @@ class vulnWhispererNessus(vulnWhispererBase): file_name, time.time(), clean_csv.shape[0], - 'nessus', + self.CONFIG_SECTION, uuid, 1, ) @@ -456,6 +457,7 @@ class vulnWhispererQualys(vulnWhispererBase): username=None, password=None, ): + super(vulnWhispererQualys, self).__init__(config=config, ) self.qualys_scan = qualysScanReport(config=config) @@ -463,7 +465,6 @@ class vulnWhispererQualys(vulnWhispererBase): self.directory_check() - def directory_check(self): if not os.path.exists(self.write_path): os.makedirs(self.write_path) @@ -474,7 +475,13 @@ class vulnWhispererQualys(vulnWhispererBase): self.vprint('{info} Directory already exist for {scan} - Skipping creation'.format( scan=self.write_path, info=bcolors.INFO)) - def whisper_reports(self, report_id, updated_date, output_format='json', cleanup=True): + def whisper_reports(self, + report_id=None, + launched_date=None, + scan_name=None, + scan_reference=None, + output_format='json', + cleanup=True): """ report_id: App ID updated_date: Last time scan was ran for app_id @@ -482,28 +489,34 @@ class vulnWhispererQualys(vulnWhispererBase): vuln_ready = None try: - if 'Z' in updated_date: - updated_date = self.qualys_scan.utils.iso_to_epoch(updated_date) + if 'Z' in launched_date: + launched_date = self.qualys_scan.utils.iso_to_epoch(launched_date) report_name = 'qualys_web_' + str(report_id) \ - + '_{last_updated}'.format(last_updated=updated_date) \ + + '_{last_updated}'.format(last_updated=launched_date) \ + '.{extension}'.format(output_format) - """ - record_meta = ( - scan_name, - app_id, - norm_time, - report_name, - time.time(), - clean_csv.shape[0], - 'qualys', - uuid, - 1, - ) - """ - #self.record_insert(record_meta) + + relative_path_name = self.path_check(report_name) + if os.path.isfile(self.path_check(report_name)): - print('{action} - File already exist! Skipping...'.format(action=bcolors.ACTION)) - pass + #TODO Possibly make this optional to sync directories + file_length = len(open(report_name).readlines()) + record_meta = ( + scan_name, + scan_reference, + launched_date, + report_name, + time.time(), + file_length, + self.CONFIG_SECTION, + report_id, + 1, + ) + self.record_insert(record_meta) + self.vprint('{info} File {filename} already exist! Updating database'.format(info=bcolors.INFO, filename=relative_path_name)) + #else: + # print('{action} - File already exist! Skipping...'.format(action=bcolors.ACTION)) + # pass + else: print('{action} - Generating report for %s'.format(action=bcolors.ACTION) % report_id) status = self.qualys_scan.qw.create_report(report_id) @@ -519,6 +532,20 @@ class vulnWhispererQualys(vulnWhispererBase): vuln_ready.to_csv(self.path_check(report_name), index=False, header=True) # add when timestamp occured vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True) + + record_meta = ( + scan_name, + scan_reference, + launched_date, + report_name, + time.time(), + vuln_ready.shape[0], + self.CONFIG_SECTION, + report_id, + 1, + ) + self.record_insert(record_meta) + if output_format == 'json': with open(self.path_check(report_name), 'w') as f: f.write(vuln_ready.to_json(orient='records', lines=True)) @@ -543,12 +570,31 @@ class vulnWhispererQualys(vulnWhispererBase): print('{error} - Could not process %s - %s'.format(error=bcolors.FAIL) % (report_id, e)) return vuln_ready + + def identify_scans_to_process(self): + scan_id_list = self.latest_scans.id.tolist() + if self.uuids: + scan_list = [scan for scan in scan_id_list if scan + not in self.uuids] + else: + scan_list = scan_id_list + self.vprint('{info} Identified {new} scans to be processed'.format(info=bcolors.INFO, + new=len(scan_list))) + + if not scan_list: + self.vprint('{info} No new scans to process. Exiting...'.format(info=bcolors.INFO)) + exit(0) + def process_web_assets(self): counter = 0 for app in self.latest_scans.iterrows(): counter += 1 + r = app[1] print('Processing %s/%s' % (counter, len(self.latest_scans))) - self.whisper_reports(app[1]['id'], app[1]['launchedDate']) + self.whisper_reports(report_id=r['id'], + launched_date=r['launchedDate'], + scan_name=r['name'], + scan_reference=r['reference']) @@ -581,6 +627,4 @@ class vulnWhisperer(object): elif self.profile == 'qualys': vw = vulnWhispererQualys(config=self.config) - vw.process_web_assets() - - + vw.process_web_assets() \ No newline at end of file