Cyberdefense/VulnWhisperer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Database tracking for processed Qualys scans

Browse Source
This commit is contained in:
Austin Taylor
2017-12-30 11:20:31 -05:00
parent 469f3fee81
commit 07a99eda54
2 changed files with 111 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
vulnwhisp/frameworks/qualys.py
View File

@ -358,7 +358,7 @@ class qualysUtils:
return dp.parse(dt).strftime('%s')
def cleanser(self, _data):
repls = (('\n', '|||'), ('\r', '|||'), (',', ';'), ('\t', '|||'), ('""', "''"))
repls = (('\n', '|||'), ('\r', '|||'), (',', ';'), ('\t', '|||'))
if _data:
_data = reduce(lambda a, kv: a.replace(*kv), repls, str(_data))
return _data
@ -445,46 +445,40 @@ class qualysWebAppReport:
def grab_sections(self, report):
all_dataframes = []
dict_tracker = {}
with open(report, 'rb') as csvfile:
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
dict_tracker['WEB_APP_VULN_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
self.WEB_APP_VULN_BLOCK,
end=[self.WEB_APP_SENSITIVE_BLOCK,
self.WEB_APP_INFO_BLOCK],
pop_last=True),
columns=self.WEB_APP_VULN_HEADER))
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
pop_last=True), columns=self.WEB_APP_VULN_HEADER)
dict_tracker['WEB_APP_SENSITIVE_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
self.WEB_APP_SENSITIVE_BLOCK,
end=[self.WEB_APP_INFO_BLOCK,
self.WEB_APP_SENSITIVE_BLOCK],
pop_last=True),
columns=self.WEB_APP_SENSITIVE_HEADER))
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
pop_last=True), columns=self.WEB_APP_SENSITIVE_HEADER)
dict_tracker['WEB_APP_INFO_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
self.WEB_APP_INFO_BLOCK,
end=[self.QID_HEADER],
pop_last=True),
columns=self.WEB_APP_INFO_HEADER))
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
pop_last=True), columns=self.WEB_APP_INFO_HEADER)
dict_tracker['QID_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
self.QID_HEADER,
end=[self.GROUP_HEADER],
pop_last=True),
columns=self.QID_HEADER))
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
pop_last=True), columns=self.QID_HEADER)
dict_tracker['GROUP_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
self.GROUP_HEADER,
end=[self.OWASP_HEADER],
pop_last=True),
columns=self.GROUP_HEADER))
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
pop_last=True), columns=self.GROUP_HEADER)
dict_tracker['OWASP_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
self.OWASP_HEADER,
end=[self.WASC_HEADER],
pop_last=True),
columns=self.OWASP_HEADER))
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
pop_last=True), columns=self.OWASP_HEADER)
dict_tracker['WASC_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
self.WASC_HEADER, end=[['APPENDIX']],
pop_last=True),
columns=self.WASC_HEADER))
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
self.CATEGORY_HEADER),
columns=self.CATEGORY_HEADER))
pop_last=True), columns=self.WASC_HEADER)
dict_tracker['CATEGORY_HEADER'] =pd.DataFrame(self.utils.grab_section(report,
self.CATEGORY_HEADER), columns=self.CATEGORY_HEADER)
all_dataframes.append(dict_tracker)
return all_dataframes
@ -494,6 +488,13 @@ class qualysWebAppReport:
:param dataframes:
:return:
"""
df_dict = dataframes[0]
merged_df = pd.concat([df_dict['WEB_APP_VULN_BLOCK'], df_dict['WEB_APP_SENSITIVE_BLOCK'],
df_dict['WEB_APP_INFO_BLOCK']], axis=0,
ignore_index=False)
merged_df = pd.merge(merged_df, df_dict['QID_HEADER'], left_on='QID',
right_on='Id')
merged_df = pd.concat([dataframes[0], dataframes[1],
dataframes[2]], axis=0,
@ -508,9 +509,10 @@ class qualysWebAppReport:
'Request Headers #1', 'Response #1', 'Evidence #1',
'Description', 'Impact', 'Solution', 'Url', 'Content']
#for col in columns_to_cleanse:
# merged_df[col] = merged_df[col].astype(str).apply(self.utils.cleanser)
for col in columns_to_cleanse:
merged_df[col] = merged_df[col].astype(str).apply(self.utils.cleanser)
merged_df = pd.merge(merged_df, df_dict['CATEGORY_HEADER'])
merged_df = merged_df.drop(['QID_y', 'QID_x'], axis=1)
merged_df = merged_df.rename(columns={'Id': 'QID'})
merged_df = merged_df.replace('N/A','').fillna('')
@ -673,41 +675,41 @@ class qualysScanReport:
self.WEB_SCAN_INFO_BLOCK,
self.WEB_SCAN_SENSITIVE_BLOCK],
pop_last=True),
columns=self.WEB_SCAN_SENSITIVE_HEADER)
columns=self.WEB_SCAN_SENSITIVE_HEADER)
dict_tracker['WEB_SCAN_INFO_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
self.WEB_SCAN_INFO_BLOCK,
end=[self.QID_HEADER],
pop_last=True),
columns=self.WEB_SCAN_INFO_HEADER)
columns=self.WEB_SCAN_INFO_HEADER)
dict_tracker['QID_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
self.QID_HEADER,
end=[self.GROUP_HEADER],
pop_last=True),
columns=self.QID_HEADER)
columns=self.QID_HEADER)
dict_tracker['GROUP_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
self.GROUP_HEADER,
end=[self.OWASP_HEADER],
pop_last=True),
columns=self.GROUP_HEADER)
columns=self.GROUP_HEADER)
dict_tracker['OWASP_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
self.OWASP_HEADER,
end=[self.WASC_HEADER],
pop_last=True),
columns=self.OWASP_HEADER)
columns=self.OWASP_HEADER)
dict_tracker['WASC_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
self.WASC_HEADER, end=[['APPENDIX']],
pop_last=True),
columns=self.WASC_HEADER)
columns=self.WASC_HEADER)
dict_tracker['SCAN_META'] = pd.DataFrame(self.utils.grab_section(report,
self.SCAN_META,
end=[self.CATEGORY_HEADER],
pop_last=True),
columns=self.SCAN_META)
columns=self.SCAN_META)
dict_tracker['CATEGORY_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
self.CATEGORY_HEADER),
columns=self.CATEGORY_HEADER)
columns=self.CATEGORY_HEADER)
all_dataframes.append(dict_tracker)
return all_dataframes

102
vulnwhisp/vulnwhisp.py
View File

@ -33,6 +33,7 @@ class vulnWhispererBase(object):
username=None,
password=None,
section=None,
develop=False,
):
@ -41,6 +42,7 @@ class vulnWhispererBase(object):
self.db_name = db_name
self.purge = purge
self.develop = develop
if config is not None:
self.config = vwConfig(config_in=config)
@ -163,14 +165,13 @@ class vulnWhispererNessus(vulnWhispererBase):
):
super(vulnWhispererNessus, self).__init__(config=config)
self.port = int(self.config.get(self.CONFIG_NAME, 'port'))
self.port = int(self.config.get(self.CONFIG_SECTION, 'port'))
self.develop = True
self.purge = purge
if config is not None:
try:
#if self.enabled:
self.nessus_port = self.config.get(self.CONFIG_SECTION, 'port')
self.nessus_trash = self.config.getbool(self.CONFIG_SECTION,
@ -325,7 +326,7 @@ class vulnWhispererNessus(vulnWhispererBase):
file_name,
time.time(),
csv_in.shape[0],
'nessus',
self.CONFIG_SECTION,
uuid,
1,
)
@ -361,7 +362,7 @@ class vulnWhispererNessus(vulnWhispererBase):
file_name,
time.time(),
clean_csv.shape[0],
'nessus',
self.CONFIG_SECTION,
uuid,
1,
)
@ -378,7 +379,7 @@ class vulnWhispererNessus(vulnWhispererBase):
file_name,
time.time(),
clean_csv.shape[0],
'nessus',
self.CONFIG_SECTION,
uuid,
1,
)
@ -456,6 +457,7 @@ class vulnWhispererQualys(vulnWhispererBase):
username=None,
password=None,
):
super(vulnWhispererQualys, self).__init__(config=config, )
self.qualys_scan = qualysScanReport(config=config)
@ -463,7 +465,6 @@ class vulnWhispererQualys(vulnWhispererBase):
self.directory_check()
def directory_check(self):
if not os.path.exists(self.write_path):
os.makedirs(self.write_path)
@ -474,7 +475,13 @@ class vulnWhispererQualys(vulnWhispererBase):
self.vprint('{info} Directory already exist for {scan} - Skipping creation'.format(
scan=self.write_path, info=bcolors.INFO))
def whisper_reports(self, report_id, updated_date, output_format='json', cleanup=True):
def whisper_reports(self,
report_id=None,
launched_date=None,
scan_name=None,
scan_reference=None,
output_format='json',
cleanup=True):
"""
report_id: App ID
updated_date: Last time scan was ran for app_id
@ -482,28 +489,34 @@ class vulnWhispererQualys(vulnWhispererBase):
vuln_ready = None
try:
if 'Z' in updated_date:
updated_date = self.qualys_scan.utils.iso_to_epoch(updated_date)
if 'Z' in launched_date:
launched_date = self.qualys_scan.utils.iso_to_epoch(launched_date)
report_name = 'qualys_web_' + str(report_id) \
+ '_{last_updated}'.format(last_updated=updated_date) \
+ '_{last_updated}'.format(last_updated=launched_date) \
+ '.{extension}'.format(output_format)
"""
record_meta = (
scan_name,
app_id,
norm_time,
report_name,
time.time(),
clean_csv.shape[0],
'qualys',
uuid,
1,
)
"""
#self.record_insert(record_meta)
relative_path_name = self.path_check(report_name)
if os.path.isfile(self.path_check(report_name)):
print('{action} - File already exist! Skipping...'.format(action=bcolors.ACTION))
pass
#TODO Possibly make this optional to sync directories
file_length = len(open(report_name).readlines())
record_meta = (
scan_name,
scan_reference,
launched_date,
report_name,
time.time(),
file_length,
self.CONFIG_SECTION,
report_id,
1,
)
self.record_insert(record_meta)
self.vprint('{info} File {filename} already exist! Updating database'.format(info=bcolors.INFO, filename=relative_path_name))
#else:
# print('{action} - File already exist! Skipping...'.format(action=bcolors.ACTION))
# pass
else:
print('{action} - Generating report for %s'.format(action=bcolors.ACTION) % report_id)
status = self.qualys_scan.qw.create_report(report_id)
@ -519,6 +532,20 @@ class vulnWhispererQualys(vulnWhispererBase):
vuln_ready.to_csv(self.path_check(report_name), index=False, header=True) # add when timestamp occured
vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
record_meta = (
scan_name,
scan_reference,
launched_date,
report_name,
time.time(),
vuln_ready.shape[0],
self.CONFIG_SECTION,
report_id,
1,
)
self.record_insert(record_meta)
if output_format == 'json':
with open(self.path_check(report_name), 'w') as f:
f.write(vuln_ready.to_json(orient='records', lines=True))
@ -543,12 +570,31 @@ class vulnWhispererQualys(vulnWhispererBase):
print('{error} - Could not process %s - %s'.format(error=bcolors.FAIL) % (report_id, e))
return vuln_ready
def identify_scans_to_process(self):
scan_id_list = self.latest_scans.id.tolist()
if self.uuids:
scan_list = [scan for scan in scan_id_list if scan
not in self.uuids]
else:
scan_list = scan_id_list
self.vprint('{info} Identified {new} scans to be processed'.format(info=bcolors.INFO,
new=len(scan_list)))
if not scan_list:
self.vprint('{info} No new scans to process. Exiting...'.format(info=bcolors.INFO))
exit(0)
def process_web_assets(self):
counter = 0
for app in self.latest_scans.iterrows():
counter += 1
r = app[1]
print('Processing %s/%s' % (counter, len(self.latest_scans)))
self.whisper_reports(app[1]['id'], app[1]['launchedDate'])
self.whisper_reports(report_id=r['id'],
launched_date=r['launchedDate'],
scan_name=r['name'],
scan_reference=r['reference'])
@ -582,5 +628,3 @@ class vulnWhisperer(object):
elif self.profile == 'qualys':
vw = vulnWhispererQualys(config=self.config)
vw.process_web_assets()