Cyberdefense/VulnWhisperer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Addition of category class and special class for Qualys Scanning Reports. Also added additional enrichments to reports

Browse Source
This commit is contained in:
Austin Taylor
2017-12-28 21:57:21 -05:00
parent a274341d23
commit d03ba15772
3 changed files with 594 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files

677
vulnwhisp/frameworks/qualys.py
View File

@ -18,8 +18,10 @@ import os
import csv
import dateutil.parser as dp
class qualysWhisper(object):
COUNT = '/count/was/webapp'
class qualysWhisperAPI(object):
COUNT_WEBAPP = '/count/was/webapp'
COUNT_WASSCAN = '/count/was/wasscan'
DELETE_REPORT = '/delete/was/report/{report_id}'
GET_WEBAPP_DETAILS = '/get/was/webapp/{was_id}'
QPS_REST_3 = '/qps/rest/3.0'
@ -28,6 +30,8 @@ class qualysWhisper(object):
REPORT_STATUS = '/status/was/report/{report_id}'
REPORT_CREATE = '/create/was/report'
REPORT_DOWNLOAD = '/download/was/report/{report_id}'
SCAN_DETAILS = '/get/was/wasscan/{scan_id}'
SCAN_DOWNLOAD = '/download/was/wasscan/{scan_id}'
SEARCH_REPORTS = '/search/was/report'
SEARCH_WEB_APPS = '/search/was/webapp'
SEARCH_WAS_SCAN = '/search/was/wasscan'
@ -37,37 +41,42 @@ class qualysWhisper(object):
self.config = config
try:
self.qgc = qualysapi.connect(config)
print('[SUCCESS] - Connected to Qualys at %s' \
% self.qgc.server)
print('[SUCCESS] - Connected to Qualys at %s' % self.qgc.server)
except Exception as e:
print('[ERROR] Could not connect to Qualys - %s' % e)
self.headers = {'content-type': 'text/xml'}
self.headers = {
"content-type": "text/xml"}
self.config_parse = qcconf.QualysConnectConfig(config)
try:
self.template_id = self.config_parse.get_template_id()
except:
print 'ERROR - Could not retrieve template ID'
sys.exit(2)
print('ERROR - Could not retrieve template ID')
def request(
self,
path,
method='get',
data=None,
):
methods = {'get': requests.get, 'post': requests.post}
def request(self, path, method='get', data=None):
methods = {'get': requests.get,
'post': requests.post}
base = 'https://' + self.qgc.server + path
req = methods[method](base, auth=self.qgc.auth, data=data,
headers=self.headers).content
req = methods[method](base, auth=self.qgc.auth, data=data, headers=self.headers).content
return req
def get_version(self):
return self.request(self.VERSION)
def get_scan_count(self, scan_name):
parameters = E.ServiceRequest(E.filters(E.Criteria(scan_name,
field='name', operator='CONTAINS')))
xml_output = self.qgc.request(self.COUNT, parameters)
parameters = (
E.ServiceRequest(
E.filters(
E.Criteria({'field': 'name', 'operator': 'CONTAINS'}, scan_name))))
xml_output = self.qgc.request(self.COUNT_WEBAPP, parameters)
root = objectify.fromstring(xml_output)
return root.count.text
def get_was_scan_count(self, status):
parameters = (
E.ServiceRequest(
E.filters(
E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status))))
xml_output = self.qgc.request(self.COUNT_WASSCAN, parameters)
root = objectify.fromstring(xml_output)
return root.count.text
@ -77,14 +86,16 @@ class qualysWhisper(object):
def xml_parser(self, xml, dupfield=None):
all_records = []
root = ET.XML(xml)
for (i, child) in enumerate(root):
for i, child in enumerate(root):
for subchild in child:
record = {}
dup_tracker = 0
for p in subchild:
record[p.tag] = p.text
for o in p:
if o.tag == 'id':
record[dupfield] = o.text
if o.tag in record:
dup_tracker += 1
record[o.tag + '_%s' % dup_tracker] = o.text
else:
record[o.tag] = o.text
all_records.append(record)
@ -92,28 +103,50 @@ class qualysWhisper(object):
def get_report_list(self):
"""Returns a dataframe of reports"""
return self.xml_parser(self.get_reports(), dupfield='user_id')
def get_web_apps(self):
"""Returns webapps available for account"""
return self.qgc.request(self.SEARCH_WEB_APPS)
def get_web_app_list(self):
"""Returns dataframe of webapps"""
return self.xml_parser(self.get_web_apps(), dupfield='user_id')
def get_web_app_details(self, was_id):
"""Get webapp details - use to retrieve app ID tag"""
return self.qgc.request(self.GET_WEBAPP_DETAILS.format(was_id=was_id))
def get_scans_by_app_id(self, app_id):
data = self.generate_app_id_scan_XML(app_id)
return self.qgc.request(self.SEARCH_WAS_SCAN, data)
def get_scan_info(self, limit=1000, offset=1, status='FINISHED'):
""" Returns XML of ALL WAS Scans"""
data = self.generate_scan_result_XML(limit=limit, offset=offset, status=status)
return self.qgc.request(self.SEARCH_WAS_SCAN, data)
def get_all_scans(self, limit=1000, offset=1, status='FINISHED'):
qualys_api_limit = limit
dataframes = []
_records = []
total = int(self.get_was_scan_count(status=status))
print('Processing %s total scans' % total)
for i in range(0, total):
if i % limit == 0:
if (total - i) < limit:
qualys_api_limit = total - i
print('Making a request with a limit of %s at offset %s' % (str(qualys_api_limit), str(i + 1)))
scan_info = self.get_scan_info(limit=qualys_api_limit, offset=i + 1, status=status)
_records.append(scan_info)
print('Converting XML to DataFrame')
dataframes = [self.xml_parser(xml) for xml in _records]
return pd.concat(dataframes, axis=0).reset_index().drop('index', axis=1)
def get_scan_details(self, scan_id):
return self.qgc.request(self.SCAN_DETAILS.format(scan_id=scan_id))
def get_report_details(self, report_id):
return self.qgc.request(self.REPORT_DETAILS.format(report_id=report_id))
@ -123,43 +156,109 @@ class qualysWhisper(object):
def download_report(self, report_id):
return self.qgc.request(self.REPORT_DOWNLOAD.format(report_id=report_id))
def download_scan_results(self, scan_id):
return self.qgc.request(self.SCAN_DOWNLOAD.format(scan_id=scan_id))
def generate_scan_result_XML(self, limit=1000, offset=1, status='FINISHED'):
report_xml = E.ServiceRequest(
E.filters(
E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status
),
),
E.preferences(
E.startFromOffset(str(offset)),
E.limitResults(str(limit))
),
)
return report_xml
def generate_scan_report_XML(self, scan_id):
"""Generates a CSV report for an asset based on template defined in .ini file"""
report_xml = E.ServiceRequest(
E.data(
E.Report(
E.name('![CDATA[API Scan Report generated by VulnWhisperer]]>'),
E.description('<![CDATA[CSV Scanning report for VulnWhisperer]]>'),
E.format('CSV'),
E.type('WAS_SCAN_REPORT'),
E.template(
E.id(self.template_id)
),
E.config(
E.scanReport(
E.target(
E.scans(
E.WasScan(
E.id(scan_id)
)
),
),
),
)
)
)
)
return report_xml
def generate_webapp_report_XML(self, app_id):
"""Generates a CSV report for an asset based on template defined in .ini file"""
report_xml = \
E.ServiceRequest(E.data(E.Report(E.name('![CDATA[API Web Application Report generated by VulnWhisperer]]>'
),
E.description('<![CDATA[CSV WebApp report for VulnWhisperer]]>'
), E.format('CSV'),
E.template(E.id(self.template_id)),
E.config(E.webAppReport(E.target(E.webapps(E.WebApp(E.id(app_id)))))))))
report_xml = E.ServiceRequest(
E.data(
E.Report(
E.name('![CDATA[API Web Application Report generated by VulnWhisperer]]>'),
E.description('<![CDATA[CSV WebApp report for VulnWhisperer]]>'),
E.format('CSV'),
E.template(
E.id(self.template_id)
),
E.config(
E.webAppReport(
E.target(
E.webapps(
E.WebApp(
E.id(app_id)
)
),
),
),
)
)
)
)
return report_xml
def generate_app_id_scan_XML(self, app_id):
report_xml = \
E.ServiceRequest(E.filters(E.Criteria({'field': 'webApp.id'
, 'operator': 'EQUALS'}, app_id)))
report_xml = E.ServiceRequest(
E.filters(
E.Criteria({'field': 'webApp.id', 'operator': 'EQUALS'}, app_id
),
),
)
return report_xml
def create_report(self, report_id):
data = self.generate_webapp_report_XML(report_id)
return self.qgc.request(self.REPORT_CREATE.format(report_id=report_id),
data)
def create_report(self, report_id, kind='scan'):
mapper = {'scan': self.generate_scan_report_XML,
'webapp': self.generate_webapp_report_XML}
try:
# print lxml.etree.tostring(mapper[kind](report_id), pretty_print=True)
data = mapper[kind](report_id)
except Exception as e:
print(e)
return self.qgc.request(self.REPORT_CREATE, data)
def delete_report(self, report_id):
return self.qgc.request(self.DELETE_REPORT.format(report_id=report_id))
class qualysWebAppReport:
CATEGORIES = ['VULNERABILITY', 'SENSITIVE CONTENT',
'INFORMATION GATHERED']
class qualysReportFields:
CATEGORIES = ['VULNERABILITY',
'SENSITIVECONTENT',
'INFORMATION_GATHERED']
# URL Vulnerability Information
WEB_APP_VULN_BLOCK = [
'Web Application Name',
VULN_BLOCK = [
CATEGORIES[0],
'ID',
'QID',
@ -171,7 +270,6 @@ class qualysWebAppReport:
'Authentication',
'Ajax Request',
'Ajax Request ID',
'Status',
'Ignored',
'Ignore Reason',
'Ignore Date',
@ -189,29 +287,14 @@ class qualysWebAppReport:
'Evidence #1',
]
WEB_APP_VULN_HEADER = list(WEB_APP_VULN_BLOCK)
WEB_APP_VULN_HEADER[WEB_APP_VULN_BLOCK.index(CATEGORIES[0])] = \
'Vulnerability Category'
WEB_APP_SENSITIVE_HEADER = list(WEB_APP_VULN_HEADER)
WEB_APP_SENSITIVE_HEADER.insert(WEB_APP_SENSITIVE_HEADER.index('Url'
), 'Content')
WEB_APP_SENSITIVE_BLOCK = list(WEB_APP_SENSITIVE_HEADER)
WEB_APP_SENSITIVE_BLOCK[WEB_APP_SENSITIVE_BLOCK.index('Vulnerability Category'
)] = CATEGORIES[1]
WEB_APP_INFO_HEADER = [
'Web Application Name',
INFO_HEADER = [
'Vulnerability Category',
'ID',
'QID',
'Response #1',
'Last Time Detected',
]
WEB_APP_INFO_BLOCK = [
'Web Application Name',
INFO_BLOCK = [
CATEGORIES[2],
'ID',
'QID',
@ -238,43 +321,13 @@ class qualysWebAppReport:
GROUP_HEADER = ['GROUP', 'Name', 'Category']
OWASP_HEADER = ['OWASP', 'Code', 'Name']
WASC_HEADER = ['WASC', 'Code', 'Name']
SCAN_META = ['Web Application Name', 'URL', 'Owner', 'Operating System', 'Scope']
CATEGORY_HEADER = ['Category', 'Severity', 'Level', 'Description']
def __init__(
self,
config=None,
file_in=None,
file_stream=False,
delimiter=',',
quotechar='"',
):
self.file_in = file_in
self.file_stream = file_stream
self.report = None
if config:
try:
self.qw = qualysWhisper(config=config)
except Exception as e:
print('Could not load config! Please check settings for %s' \
% e)
if file_stream:
self.open_file = file_in.splitlines()
elif file_in:
self.open_file = open(file_in, 'rb')
self.downloaded_file = None
def get_hostname(self, report):
host = ''
with open(report, 'rb') as csvfile:
q_report = csv.reader(csvfile, delimiter=',', quotechar='"')
for x in q_report:
if 'Web Application Name' in x[0]:
host = q_report.next()[0]
return host
class qualysUtils:
def __init__(self):
pass
def grab_section(
self,
@ -311,47 +364,127 @@ class qualysWebAppReport:
_data = reduce(lambda a, kv: a.replace(*kv), repls, _data)
return _data
class qualysWebAppReport:
# URL Vulnerability Information
WEB_APP_VULN_BLOCK = list(qualysReportFields.VULN_BLOCK)
WEB_APP_VULN_BLOCK.insert(0, 'Web Application Name')
WEB_APP_VULN_BLOCK.insert(WEB_APP_VULN_BLOCK.index('Ignored'), 'Status')
WEB_APP_VULN_HEADER = list(WEB_APP_VULN_BLOCK)
WEB_APP_VULN_HEADER[WEB_APP_VULN_BLOCK.index(qualysReportFields.CATEGORIES[0])] = \
'Vulnerability Category'
WEB_APP_SENSITIVE_HEADER = list(WEB_APP_VULN_HEADER)
WEB_APP_SENSITIVE_HEADER.insert(WEB_APP_SENSITIVE_HEADER.index('Url'
), 'Content')
WEB_APP_SENSITIVE_BLOCK = list(WEB_APP_SENSITIVE_HEADER)
WEB_APP_SENSITIVE_BLOCK[WEB_APP_SENSITIVE_BLOCK.index('Vulnerability Category'
)] = qualysReportFields.CATEGORIES[1]
WEB_APP_INFO_HEADER = list(qualysReportFields.INFO_HEADER)
WEB_APP_INFO_HEADER.insert(0, 'Web Application Name')
WEB_APP_INFO_BLOCK = list(qualysReportFields.INFO_BLOCK)
WEB_APP_INFO_BLOCK.insert(0, 'Web Application Name')
QID_HEADER = list(qualysReportFields.QID_HEADER)
GROUP_HEADER = list(qualysReportFields.GROUP_HEADER)
OWASP_HEADER = list(qualysReportFields.OWASP_HEADER)
WASC_HEADER = list(qualysReportFields.WASC_HEADER)
SCAN_META = list(qualysReportFields.SCAN_META)
CATEGORY_HEADER = list(qualysReportFields.CATEGORY_HEADER)
def __init__(
self,
config=None,
file_in=None,
file_stream=False,
delimiter=',',
quotechar='"',
):
self.file_in = file_in
self.file_stream = file_stream
self.report = None
self.utils = qualysUtils()
if config:
try:
self.qw = qualysWhisperAPI(config=config)
except Exception as e:
print('Could not load config! Please check settings for %s' \
% e)
if file_stream:
self.open_file = file_in.splitlines()
elif file_in:
self.open_file = open(file_in, 'rb')
self.downloaded_file = None
def get_hostname(self, report):
host = ''
with open(report, 'rb') as csvfile:
q_report = csv.reader(csvfile, delimiter=',', quotechar='"')
for x in q_report:
if 'Web Application Name' in x[0]:
host = q_report.next()[0]
return host
def get_scanreport_name(self, report):
scan_name = ''
with open(report, 'rb') as csvfile:
q_report = csv.reader(csvfile, delimiter=',', quotechar='"')
for x in q_report:
if 'Scans' in x[0]:
scan_name = x[1]
return scan_name
def grab_sections(self, report):
all_dataframes = []
with open(report, 'rb') as csvfile:
all_dataframes.append(pd.DataFrame(self.grab_section(report,
self.WEB_APP_VULN_BLOCK,
end=[self.WEB_APP_SENSITIVE_BLOCK,
self.WEB_APP_INFO_BLOCK],
pop_last=True),
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
self.WEB_APP_VULN_BLOCK,
end=[self.WEB_APP_SENSITIVE_BLOCK,
self.WEB_APP_INFO_BLOCK],
pop_last=True),
columns=self.WEB_APP_VULN_HEADER))
all_dataframes.append(pd.DataFrame(self.grab_section(report,
self.WEB_APP_SENSITIVE_BLOCK,
end=[self.WEB_APP_INFO_BLOCK,
self.WEB_APP_SENSITIVE_BLOCK],
pop_last=True),
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
self.WEB_APP_SENSITIVE_BLOCK,
end=[self.WEB_APP_INFO_BLOCK,
self.WEB_APP_SENSITIVE_BLOCK],
pop_last=True),
columns=self.WEB_APP_SENSITIVE_HEADER))
all_dataframes.append(pd.DataFrame(self.grab_section(report,
self.WEB_APP_INFO_BLOCK,
end=[self.QID_HEADER],
pop_last=True),
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
self.WEB_APP_INFO_BLOCK,
end=[self.QID_HEADER],
pop_last=True),
columns=self.WEB_APP_INFO_HEADER))
all_dataframes.append(pd.DataFrame(self.grab_section(report,
self.QID_HEADER,
end=[self.GROUP_HEADER],
pop_last=True),
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
self.QID_HEADER,
end=[self.GROUP_HEADER],
pop_last=True),
columns=self.QID_HEADER))
all_dataframes.append(pd.DataFrame(self.grab_section(report,
self.GROUP_HEADER,
end=[self.OWASP_HEADER],
pop_last=True),
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
self.GROUP_HEADER,
end=[self.OWASP_HEADER],
pop_last=True),
columns=self.GROUP_HEADER))
all_dataframes.append(pd.DataFrame(self.grab_section(report,
self.OWASP_HEADER,
end=[self.WASC_HEADER],
pop_last=True),
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
self.OWASP_HEADER,
end=[self.WASC_HEADER],
pop_last=True),
columns=self.OWASP_HEADER))
all_dataframes.append(pd.DataFrame(self.grab_section(report,
self.WASC_HEADER, end=[['APPENDIX']],
pop_last=True),
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
self.WASC_HEADER, end=[['APPENDIX']],
pop_last=True),
columns=self.WASC_HEADER))
all_dataframes.append(pd.DataFrame(self.grab_section(report,
self.CATEGORY_HEADER, end=''),
all_dataframes.append(pd.DataFrame(self.utils.grab_section(report,
self.CATEGORY_HEADER),
columns=self.CATEGORY_HEADER))
return all_dataframes
@ -372,13 +505,12 @@ class qualysWebAppReport:
if 'Content' not in merged_df:
merged_df['Content'] = ''
columns_to_cleanse = ['Payload #1','Request Method #1','Request URL #1',
'Request Headers #1','Response #1','Evidence #1',
'Description','Impact','Solution','Url','Content']
columns_to_cleanse = ['Payload #1', 'Request Method #1', 'Request URL #1',
'Request Headers #1', 'Response #1', 'Evidence #1',
'Description', 'Impact', 'Solution', 'Url', 'Content']
for col in columns_to_cleanse:
merged_df[col] = merged_df[col].apply(self.cleanser)
merged_df[col] = merged_df[col].apply(self.utils.cleanser)
merged_df = merged_df.drop(['QID_y', 'QID_x'], axis=1)
merged_df = merged_df.rename(columns={'Id': 'QID'})
@ -404,6 +536,233 @@ class qualysWebAppReport:
def remove_file(self, filename):
os.remove(filename)
def process_data(self, file_id, scan=True, cleanup=True):
"""Downloads a file from qualys and normalizes it"""
download_file = self.download_file(file_id)
print('[ACTION] - Downloading file ID: %s' % file_id)
report_data = self.grab_sections(download_file)
merged_data = self.data_normalizer(report_data)
if scan:
scan_name = self.get_scanreport_name(download_file)
merged_data['ScanName'] = scan_name
# TODO cleanup old data (delete)
return merged_data
def whisper_reports(self, report_id, updated_date, cleanup=False):
"""
report_id: App ID
updated_date: Last time scan was ran for app_id
"""
vuln_ready = None
try:
if 'Z' in updated_date:
updated_date = self.utils.iso_to_epoch(updated_date)
report_name = 'qualys_web_' + str(report_id) \
+ '_{last_updated}'.format(last_updated=updated_date) \
+ '.csv'
if os.path.isfile(report_name):
print('[ACTION] - File already exist! Skipping...')
pass
else:
print('[ACTION] - Generating report for %s' % report_id)
status = self.qw.create_report(report_id)
root = objectify.fromstring(status)
if root.responseCode == 'SUCCESS':
print('[INFO] - Successfully generated report for webapp: %s' \
% report_id)
generated_report_id = root.data.Report.id
print ('[INFO] - New Report ID: %s' \
% generated_report_id)
vuln_ready = self.process_data(generated_report_id)
vuln_ready.to_csv(report_name, index=False, header=True) # add when timestamp occured
print('[SUCCESS] - Report written to %s' \
% report_name)
if cleanup:
print('[ACTION] - Removing report %s' \
% generated_report_id)
cleaning_up = \
self.qw.delete_report(generated_report_id)
self.remove_file(str(generated_report_id) + '.csv')
print('[ACTION] - Deleted report: %s' \
% generated_report_id)
else:
print('Could not process report ID: %s' % status)
except Exception as e:
print('[ERROR] - Could not process %s - %s' % (report_id, e))
return vuln_ready
class qualysScanReport:
# URL Vulnerability Information
WEB_SCAN_VULN_BLOCK = list(qualysReportFields.VULN_BLOCK)
WEB_SCAN_VULN_BLOCK.insert(WEB_SCAN_VULN_BLOCK.index('QID'), 'Detection ID')
WEB_SCAN_VULN_HEADER = list(WEB_SCAN_VULN_BLOCK)
WEB_SCAN_VULN_HEADER[WEB_SCAN_VULN_BLOCK.index(qualysReportFields.CATEGORIES[0])] = \
'Vulnerability Category'
WEB_SCAN_SENSITIVE_HEADER = list(WEB_SCAN_VULN_HEADER)
WEB_SCAN_SENSITIVE_HEADER.insert(WEB_SCAN_SENSITIVE_HEADER.index('Url'
), 'Content')
WEB_SCAN_SENSITIVE_BLOCK = list(WEB_SCAN_SENSITIVE_HEADER)
WEB_SCAN_SENSITIVE_BLOCK.insert(WEB_SCAN_SENSITIVE_BLOCK.index('QID'), 'Detection ID')
WEB_SCAN_SENSITIVE_BLOCK[WEB_SCAN_SENSITIVE_BLOCK.index('Vulnerability Category'
)] = qualysReportFields.CATEGORIES[1]
WEB_SCAN_INFO_HEADER = list(qualysReportFields.INFO_HEADER)
WEB_SCAN_INFO_HEADER.insert(WEB_SCAN_INFO_HEADER.index('QID'), 'Detection ID')
WEB_SCAN_INFO_BLOCK = list(qualysReportFields.INFO_BLOCK)
WEB_SCAN_INFO_BLOCK.insert(WEB_SCAN_INFO_BLOCK.index('QID'), 'Detection ID')
QID_HEADER = list(qualysReportFields.QID_HEADER)
GROUP_HEADER = list(qualysReportFields.GROUP_HEADER)
OWASP_HEADER = list(qualysReportFields.OWASP_HEADER)
WASC_HEADER = list(qualysReportFields.WASC_HEADER)
SCAN_META = list(qualysReportFields.SCAN_META)
CATEGORY_HEADER = list(qualysReportFields.CATEGORY_HEADER)
def __init__(
self,
config=None,
file_in=None,
file_stream=False,
delimiter=',',
quotechar='"',
):
self.file_in = file_in
self.file_stream = file_stream
self.report = None
self.utils = qualysUtils()
if config:
try:
self.qw = qualysWhisperAPI(config=config)
except Exception as e:
print('Could not load config! Please check settings for %s' \
% e)
if file_stream:
self.open_file = file_in.splitlines()
elif file_in:
self.open_file = open(file_in, 'rb')
self.downloaded_file = None
def grab_sections(self, report):
all_dataframes = []
dict_tracker = {}
with open(report, 'rb') as csvfile:
dict_tracker['WEB_SCAN_VULN_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
self.WEB_SCAN_VULN_BLOCK,
end=[
self.WEB_SCAN_SENSITIVE_BLOCK,
self.WEB_SCAN_INFO_BLOCK],
pop_last=True),
columns=self.WEB_SCAN_VULN_HEADER)
dict_tracker['WEB_SCAN_SENSITIVE_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
self.WEB_SCAN_SENSITIVE_BLOCK,
end=[
self.WEB_SCAN_INFO_BLOCK,
self.WEB_SCAN_SENSITIVE_BLOCK],
pop_last=True),
columns=self.WEB_SCAN_SENSITIVE_HEADER)
dict_tracker['WEB_SCAN_INFO_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
self.WEB_SCAN_INFO_BLOCK,
end=[self.QID_HEADER],
pop_last=True),
columns=self.WEB_SCAN_INFO_HEADER)
dict_tracker['QID_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
self.QID_HEADER,
end=[self.GROUP_HEADER],
pop_last=True),
columns=self.QID_HEADER)
dict_tracker['GROUP_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
self.GROUP_HEADER,
end=[self.OWASP_HEADER],
pop_last=True),
columns=self.GROUP_HEADER)
dict_tracker['OWASP_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
self.OWASP_HEADER,
end=[self.WASC_HEADER],
pop_last=True),
columns=self.OWASP_HEADER)
dict_tracker['WASC_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
self.WASC_HEADER, end=[['APPENDIX']],
pop_last=True),
columns=self.WASC_HEADER)
dict_tracker['SCAN_META'] = pd.DataFrame(self.utils.grab_section(report,
self.SCAN_META,
end=[self.CATEGORY_HEADER],
pop_last=True),
columns=self.SCAN_META)
dict_tracker['CATEGORY_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
self.CATEGORY_HEADER),
columns=self.CATEGORY_HEADER)
all_dataframes.append(dict_tracker)
return all_dataframes
def data_normalizer(self, dataframes):
"""
Merge and clean data
:param dataframes:
:return:
"""
df_dict = dataframes[0]
merged_df = pd.concat([df_dict['WEB_SCAN_VULN_BLOCK'], df_dict['WEB_SCAN_SENSITIVE_BLOCK'],
df_dict['WEB_SCAN_INFO_BLOCK']], axis=0,
ignore_index=False).fillna('')
merged_df = pd.merge(merged_df, df_dict['QID_HEADER'], left_on='QID',
right_on='Id')
if 'Content' not in merged_df:
merged_df['Content'] = ''
columns_to_cleanse = ['Payload #1', 'Request Method #1', 'Request URL #1',
'Request Headers #1', 'Response #1', 'Evidence #1',
'Description', 'Impact', 'Solution', 'Url', 'Content']
for col in columns_to_cleanse:
merged_df[col] = merged_df[col].apply(self.utils.cleanser)
merged_df = merged_df.drop(['QID_y', 'QID_x'], axis=1)
merged_df = merged_df.rename(columns={'Id': 'QID'})
merged_df = merged_df.assign(**df_dict['SCAN_META'].to_dict(orient='records')[0])
merged_df = pd.merge(merged_df, df_dict['CATEGORY_HEADER'], how='left', left_on=['Category', 'Severity Level'],
right_on=['Category', 'Severity'], suffixes=('Severity', 'CatSev'))
try:
merged_df = \
merged_df[~merged_df.Title.str.contains('Links Crawled|External Links Discovered'
)]
except Exception as e:
print(e)
return merged_df
def download_file(self, file_id):
report = self.qw.download_report(file_id)
filename = str(file_id) + '.csv'
file_out = open(filename, 'w')
for line in report.splitlines():
file_out.write(line + '\n')
file_out.close()
print('[ACTION] - File written to %s' % filename)
return filename
def remove_file(self, filename):
os.remove(filename)
def process_data(self, file_id, cleanup=True):
"""Downloads a file from qualys and normalizes it"""
@ -411,11 +770,55 @@ class qualysWebAppReport:
print('[ACTION] - Downloading file ID: %s' % file_id)
report_data = self.grab_sections(download_file)
merged_data = self.data_normalizer(report_data)
merged_data.sort_index(axis=1, inplace=True)
# TODO cleanup old data (delete)
return merged_data
def whisper_reports(self, report_id, updated_date, cleanup=False):
"""
report_id: App ID
updated_date: Last time scan was ran for app_id
"""
vuln_ready = None
try:
if 'Z' in updated_date:
updated_date = self.utils.iso_to_epoch(updated_date)
report_name = 'qualys_web_' + str(report_id) \
+ '_{last_updated}'.format(last_updated=updated_date) \
+ '.csv'
if os.path.isfile(report_name):
print('[ACTION] - File already exist! Skipping...')
pass
else:
print('[ACTION] - Generating report for %s' % report_id)
status = self.qw.create_report(report_id)
root = objectify.fromstring(status)
if root.responseCode == 'SUCCESS':
print('[INFO] - Successfully generated report for webapp: %s' \
% report_id)
generated_report_id = root.data.Report.id
print ('[INFO] - New Report ID: %s' \
% generated_report_id)
vuln_ready = self.process_data(generated_report_id)
vuln_ready.to_csv(report_name, index=False, header=True) # add when timestamp occured
print('[SUCCESS] - Report written to %s' \
% report_name)
if cleanup:
print('[ACTION] - Removing report %s' \
% generated_report_id)
cleaning_up = \
self.qw.delete_report(generated_report_id)
self.remove_file(str(generated_report_id) + '.csv')
print('[ACTION] - Deleted report: %s' \
% generated_report_id)
else:
print('Could not process report ID: %s' % status)
except Exception as e:
print('[ERROR] - Could not process %s - %s' % (report_id, e))
return vuln_ready
maxInt = sys.maxsize

62
vulnwhisp/vulnwhisp.py
View File

@ -4,7 +4,7 @@ __author__ = 'Austin Taylor'
from base.config import vwConfig
from frameworks.nessus import NessusAPI
from frameworks.qualys import qualysWebAppReport
from frameworks.qualys import qualysScanReport
from utils.cli import bcolors
import pandas as pd
from lxml import objectify
@ -411,11 +411,22 @@ class vulnWhispererQualys(vulnWhispererBase):
):
super(vulnWhispererQualys, self).__init__(config=config, )
self.qualys_web = qualysWebAppReport(config=config)
self.latest_scans = self.qualys_web.qw.get_web_app_list()
self.qualys_scan = qualysScanReport(config=config)
self.latest_scans = self.qualys_scan.qw.get_all_scans()
self.directory_check()
def whisper_webapp(self, report_id, updated_date):
def directory_check(self):
if not os.path.exists(self.write_path):
os.makedirs(self.write_path)
self.vprint('{info} Directory created at {scan} - Skipping creation'.format(
scan=self.write_path, info=bcolors.INFO))
else:
os.path.exists(self.write_path)
self.vprint('{info} Directory already exist for {scan} - Skipping creation'.format(
scan=self.write_path, info=bcolors.INFO))
def whisper_reports(self, report_id, updated_date, cleanup=True):
"""
report_id: App ID
updated_date: Last time scan was ran for app_id
@ -424,16 +435,30 @@ class vulnWhispererQualys(vulnWhispererBase):
try:
if 'Z' in updated_date:
updated_date = self.qualys_web.iso_to_epoch(updated_date)
updated_date = self.qualys_scan.utils.iso_to_epoch(updated_date)
report_name = 'qualys_web_' + str(report_id) \
+ '_{last_updated}'.format(last_updated=updated_date) \
+ '.csv'
if os.path.isfile(report_name):
"""
record_meta = (
scan_name,
app_id,
norm_time,
report_name,
time.time(),
clean_csv.shape[0],
'qualys',
uuid,
1,
)
"""
#self.record_insert(record_meta)
if os.path.isfile(self.path_check(report_name)):
print('{action} - File already exist! Skipping...'.format(action=bcolors.ACTION))
pass
else:
print('{action} - Generating report for %s'.format(action=bcolors.ACTION) % report_id)
status = self.qualys_web.qw.create_report(report_id)
status = self.qualys_scan.qw.create_report(report_id)
root = objectify.fromstring(status)
if root.responseCode == 'SUCCESS':
print('{info} - Successfully generated report for webapp: %s'.format(info=bcolors.INFO) \
@ -441,18 +466,19 @@ class vulnWhispererQualys(vulnWhispererBase):
generated_report_id = root.data.Report.id
print('{info} - New Report ID: %s'.format(info=bcolors.INFO) \
% generated_report_id)
vuln_ready = self.qualys_web.process_data(generated_report_id)
vuln_ready = self.qualys_scan.process_data(generated_report_id)
vuln_ready.to_csv(report_name, index=False, header=True) # add when timestamp occured
vuln_ready.to_csv(self.path_check(report_name), index=False, header=True) # add when timestamp occured
print('{success} - Report written to %s'.format(success=bcolors.SUCCESS) \
% report_name)
print('{action} - Removing report %s'.format(action=bcolors.ACTION) \
% generated_report_id)
cleaning_up = \
self.qualys_web.qw.delete_report(generated_report_id)
os.remove(str(generated_report_id) + '.csv')
print('{action} - Deleted report: %s'.format(action=bcolors.ACTION) \
% generated_report_id)
if cleanup:
cleaning_up = \
self.qualys_scan.qw.delete_report(generated_report_id)
os.remove(self.path_check(str(generated_report_id) + '.csv'))
print('{action} - Deleted report: %s'.format(action=bcolors.ACTION) \
% generated_report_id)
else:
print('{error} Could not process report ID: %s'.format(error=bcolors.FAIL) % status)
except Exception as e:
@ -464,7 +490,7 @@ class vulnWhispererQualys(vulnWhispererBase):
for app in self.latest_scans.iterrows():
counter += 1
print('Processing %s/%s' % (counter, len(self.latest_scans)))
self.whisper_webapp(app[1]['id'], app[1]['createdDate'])
self.whisper_reports(app[1]['id'], app[1]['launchedDate'])
@ -489,10 +515,14 @@ class vulnWhisperer(object):
def whisper_vulnerabilities(self):
if self.profile == 'nessus':
vw = vulnWhispererNessus(config=self.config, username=self.username, password=self.password, verbose=self.verbose)
vw = vulnWhispererNessus(config=self.config,
username=self.username,
password=self.password,
verbose=self.verbose)
vw.whisper_nessus()
elif self.profile == 'qualys':
vw = vulnWhispererQualys(config=self.config)
vw.process_web_assets()