Cyberdefense/VulnWhisperer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Database tracking for processed Qualys scans

Browse Source
This commit is contained in:
Austin Taylor
2017-12-30 11:20:31 -05:00
parent 469f3fee81
commit 07a99eda54
2 changed files with 111 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
vulnwhisp/frameworks/qualys.py
View File

@ -358,7 +358,7 @@ class qualysUtils:
return dp.parse(dt).strftime('%s') return dp.parse(dt).strftime('%s')
def cleanser(self, _data): def cleanser(self, _data):
repls = (('\n', '|||'), ('\r', '|||'), (',', ';'), ('\t', '|||'), ('""', "''")) repls = (('\n', '|||'), ('\r', '|||'), (',', ';'), ('\t', '|||'))
if _data: if _data:
_data = reduce(lambda a, kv: a.replace(*kv), repls, str(_data)) _data = reduce(lambda a, kv: a.replace(*kv), repls, str(_data))
return _data return _data
@ -445,46 +445,40 @@ class qualysWebAppReport:
def grab_sections(self, report): def grab_sections(self, report):
all_dataframes = [] all_dataframes = []
dict_tracker = {}
with open(report, 'rb') as csvfile: with open(report, 'rb') as csvfile:
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report, dict_tracker['WEB_APP_VULN_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
self.WEB_APP_VULN_BLOCK, self.WEB_APP_VULN_BLOCK,
end=[self.WEB_APP_SENSITIVE_BLOCK, end=[self.WEB_APP_SENSITIVE_BLOCK,
self.WEB_APP_INFO_BLOCK], self.WEB_APP_INFO_BLOCK],
pop_last=True), pop_last=True), columns=self.WEB_APP_VULN_HEADER)
columns=self.WEB_APP_VULN_HEADER)) dict_tracker['WEB_APP_SENSITIVE_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
self.WEB_APP_SENSITIVE_BLOCK, self.WEB_APP_SENSITIVE_BLOCK,
end=[self.WEB_APP_INFO_BLOCK, end=[self.WEB_APP_INFO_BLOCK,
self.WEB_APP_SENSITIVE_BLOCK], self.WEB_APP_SENSITIVE_BLOCK],
pop_last=True), pop_last=True), columns=self.WEB_APP_SENSITIVE_HEADER)
columns=self.WEB_APP_SENSITIVE_HEADER)) dict_tracker['WEB_APP_INFO_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
self.WEB_APP_INFO_BLOCK, self.WEB_APP_INFO_BLOCK,
end=[self.QID_HEADER], end=[self.QID_HEADER],
pop_last=True), pop_last=True), columns=self.WEB_APP_INFO_HEADER)
columns=self.WEB_APP_INFO_HEADER)) dict_tracker['QID_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
self.QID_HEADER, self.QID_HEADER,
end=[self.GROUP_HEADER], end=[self.GROUP_HEADER],
pop_last=True), pop_last=True), columns=self.QID_HEADER)
columns=self.QID_HEADER)) dict_tracker['GROUP_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
self.GROUP_HEADER, self.GROUP_HEADER,
end=[self.OWASP_HEADER], end=[self.OWASP_HEADER],
pop_last=True), pop_last=True), columns=self.GROUP_HEADER)
columns=self.GROUP_HEADER)) dict_tracker['OWASP_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
self.OWASP_HEADER, self.OWASP_HEADER,
end=[self.WASC_HEADER], end=[self.WASC_HEADER],
pop_last=True), pop_last=True), columns=self.OWASP_HEADER)
columns=self.OWASP_HEADER)) dict_tracker['WASC_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
self.WASC_HEADER, end=[['APPENDIX']], self.WASC_HEADER, end=[['APPENDIX']],
pop_last=True), pop_last=True), columns=self.WASC_HEADER)
columns=self.WASC_HEADER)) dict_tracker['CATEGORY_HEADER'] =pd.DataFrame(self.utils.grab_section(report,
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report, self.CATEGORY_HEADER), columns=self.CATEGORY_HEADER)
self.CATEGORY_HEADER), all_dataframes.append(dict_tracker)
columns=self.CATEGORY_HEADER))
return all_dataframes return all_dataframes
@ -494,6 +488,13 @@ class qualysWebAppReport:
:param dataframes: :param dataframes:
:return: :return:
""" """
df_dict = dataframes[0]
merged_df = pd.concat([df_dict['WEB_APP_VULN_BLOCK'], df_dict['WEB_APP_SENSITIVE_BLOCK'],
df_dict['WEB_APP_INFO_BLOCK']], axis=0,
ignore_index=False)
merged_df = pd.merge(merged_df, df_dict['QID_HEADER'], left_on='QID',
right_on='Id')
merged_df = pd.concat([dataframes[0], dataframes[1], merged_df = pd.concat([dataframes[0], dataframes[1],
dataframes[2]], axis=0, dataframes[2]], axis=0,
@ -508,9 +509,10 @@ class qualysWebAppReport:
'Request Headers #1', 'Response #1', 'Evidence #1', 'Request Headers #1', 'Response #1', 'Evidence #1',
'Description', 'Impact', 'Solution', 'Url', 'Content'] 'Description', 'Impact', 'Solution', 'Url', 'Content']
#for col in columns_to_cleanse: for col in columns_to_cleanse:
# merged_df[col] = merged_df[col].astype(str).apply(self.utils.cleanser) merged_df[col] = merged_df[col].astype(str).apply(self.utils.cleanser)
merged_df = pd.merge(merged_df, df_dict['CATEGORY_HEADER'])
merged_df = merged_df.drop(['QID_y', 'QID_x'], axis=1) merged_df = merged_df.drop(['QID_y', 'QID_x'], axis=1)
merged_df = merged_df.rename(columns={'Id': 'QID'}) merged_df = merged_df.rename(columns={'Id': 'QID'})
merged_df = merged_df.replace('N/A','').fillna('') merged_df = merged_df.replace('N/A','').fillna('')

92
vulnwhisp/vulnwhisp.py
View File

@ -33,6 +33,7 @@ class vulnWhispererBase(object):
username=None, username=None,
password=None, password=None,
section=None, section=None,
develop=False,
): ):
@ -41,6 +42,7 @@ class vulnWhispererBase(object):
self.db_name = db_name self.db_name = db_name
self.purge = purge self.purge = purge
self.develop = develop
if config is not None: if config is not None:
self.config = vwConfig(config_in=config) self.config = vwConfig(config_in=config)
@ -163,14 +165,13 @@ class vulnWhispererNessus(vulnWhispererBase):
): ):
super(vulnWhispererNessus, self).__init__(config=config) super(vulnWhispererNessus, self).__init__(config=config)
self.port = int(self.config.get(self.CONFIG_NAME, 'port')) self.port = int(self.config.get(self.CONFIG_SECTION, 'port'))
self.develop = True self.develop = True
self.purge = purge self.purge = purge
if config is not None: if config is not None:
try: try:
#if self.enabled:
self.nessus_port = self.config.get(self.CONFIG_SECTION, 'port') self.nessus_port = self.config.get(self.CONFIG_SECTION, 'port')
self.nessus_trash = self.config.getbool(self.CONFIG_SECTION, self.nessus_trash = self.config.getbool(self.CONFIG_SECTION,
@ -325,7 +326,7 @@ class vulnWhispererNessus(vulnWhispererBase):
file_name, file_name,
time.time(), time.time(),
csv_in.shape[0], csv_in.shape[0],
'nessus', self.CONFIG_SECTION,
uuid, uuid,
1, 1,
) )
@ -361,7 +362,7 @@ class vulnWhispererNessus(vulnWhispererBase):
file_name, file_name,
time.time(), time.time(),
clean_csv.shape[0], clean_csv.shape[0],
'nessus', self.CONFIG_SECTION,
uuid, uuid,
1, 1,
) )
@ -378,7 +379,7 @@ class vulnWhispererNessus(vulnWhispererBase):
file_name, file_name,
time.time(), time.time(),
clean_csv.shape[0], clean_csv.shape[0],
'nessus', self.CONFIG_SECTION,
uuid, uuid,
1, 1,
) )
@ -456,6 +457,7 @@ class vulnWhispererQualys(vulnWhispererBase):
username=None, username=None,
password=None, password=None,
): ):
super(vulnWhispererQualys, self).__init__(config=config, ) super(vulnWhispererQualys, self).__init__(config=config, )
self.qualys_scan = qualysScanReport(config=config) self.qualys_scan = qualysScanReport(config=config)
@ -463,7 +465,6 @@ class vulnWhispererQualys(vulnWhispererBase):
self.directory_check() self.directory_check()
def directory_check(self): def directory_check(self):
if not os.path.exists(self.write_path): if not os.path.exists(self.write_path):
os.makedirs(self.write_path) os.makedirs(self.write_path)
@ -474,7 +475,13 @@ class vulnWhispererQualys(vulnWhispererBase):
self.vprint('{info} Directory already exist for {scan} - Skipping creation'.format( self.vprint('{info} Directory already exist for {scan} - Skipping creation'.format(
scan=self.write_path, info=bcolors.INFO)) scan=self.write_path, info=bcolors.INFO))
def whisper_reports(self, report_id, updated_date, output_format='json', cleanup=True): def whisper_reports(self,
report_id=None,
launched_date=None,
scan_name=None,
scan_reference=None,
output_format='json',
cleanup=True):
""" """
report_id: App ID report_id: App ID
updated_date: Last time scan was ran for app_id updated_date: Last time scan was ran for app_id
@ -482,28 +489,34 @@ class vulnWhispererQualys(vulnWhispererBase):
vuln_ready = None vuln_ready = None
try: try:
if 'Z' in updated_date: if 'Z' in launched_date:
updated_date = self.qualys_scan.utils.iso_to_epoch(updated_date) launched_date = self.qualys_scan.utils.iso_to_epoch(launched_date)
report_name = 'qualys_web_' + str(report_id) \ report_name = 'qualys_web_' + str(report_id) \
+ '_{last_updated}'.format(last_updated=updated_date) \ + '_{last_updated}'.format(last_updated=launched_date) \
+ '.{extension}'.format(output_format) + '.{extension}'.format(output_format)
"""
relative_path_name = self.path_check(report_name)
if os.path.isfile(self.path_check(report_name)):
#TODO Possibly make this optional to sync directories
file_length = len(open(report_name).readlines())
record_meta = ( record_meta = (
scan_name, scan_name,
app_id, scan_reference,
norm_time, launched_date,
report_name, report_name,
time.time(), time.time(),
clean_csv.shape[0], file_length,
'qualys', self.CONFIG_SECTION,
uuid, report_id,
1, 1,
) )
""" self.record_insert(record_meta)
#self.record_insert(record_meta) self.vprint('{info} File {filename} already exist! Updating database'.format(info=bcolors.INFO, filename=relative_path_name))
if os.path.isfile(self.path_check(report_name)): #else:
print('{action} - File already exist! Skipping...'.format(action=bcolors.ACTION)) # print('{action} - File already exist! Skipping...'.format(action=bcolors.ACTION))
pass # pass
else: else:
print('{action} - Generating report for %s'.format(action=bcolors.ACTION) % report_id) print('{action} - Generating report for %s'.format(action=bcolors.ACTION) % report_id)
status = self.qualys_scan.qw.create_report(report_id) status = self.qualys_scan.qw.create_report(report_id)
@ -519,6 +532,20 @@ class vulnWhispererQualys(vulnWhispererBase):
vuln_ready.to_csv(self.path_check(report_name), index=False, header=True) # add when timestamp occured vuln_ready.to_csv(self.path_check(report_name), index=False, header=True) # add when timestamp occured
vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True) vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
record_meta = (
scan_name,
scan_reference,
launched_date,
report_name,
time.time(),
vuln_ready.shape[0],
self.CONFIG_SECTION,
report_id,
1,
)
self.record_insert(record_meta)
if output_format == 'json': if output_format == 'json':
with open(self.path_check(report_name), 'w') as f: with open(self.path_check(report_name), 'w') as f:
f.write(vuln_ready.to_json(orient='records', lines=True)) f.write(vuln_ready.to_json(orient='records', lines=True))
@ -543,12 +570,31 @@ class vulnWhispererQualys(vulnWhispererBase):
print('{error} - Could not process %s - %s'.format(error=bcolors.FAIL) % (report_id, e)) print('{error} - Could not process %s - %s'.format(error=bcolors.FAIL) % (report_id, e))
return vuln_ready return vuln_ready
def identify_scans_to_process(self):
scan_id_list = self.latest_scans.id.tolist()
if self.uuids:
scan_list = [scan for scan in scan_id_list if scan
not in self.uuids]
else:
scan_list = scan_id_list
self.vprint('{info} Identified {new} scans to be processed'.format(info=bcolors.INFO,
new=len(scan_list)))
if not scan_list:
self.vprint('{info} No new scans to process. Exiting...'.format(info=bcolors.INFO))
exit(0)
def process_web_assets(self): def process_web_assets(self):
counter = 0 counter = 0
for app in self.latest_scans.iterrows(): for app in self.latest_scans.iterrows():
counter += 1 counter += 1
r = app[1]
print('Processing %s/%s' % (counter, len(self.latest_scans))) print('Processing %s/%s' % (counter, len(self.latest_scans)))
self.whisper_reports(app[1]['id'], app[1]['launchedDate']) self.whisper_reports(report_id=r['id'],
launched_date=r['launchedDate'],
scan_name=r['name'],
scan_reference=r['reference'])
@ -582,5 +628,3 @@ class vulnWhisperer(object):
elif self.profile == 'qualys': elif self.profile == 'qualys':
vw = vulnWhispererQualys(config=self.config) vw = vulnWhispererQualys(config=self.config)
vw.process_web_assets() vw.process_web_assets()