Fixed multiple bugs, cleaned up formatting, produces solid csv output for Qualys Web App scans
This commit is contained in:
@ -1,4 +1,5 @@
|
|||||||
[info]
|
[info]
|
||||||
|
#Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
|
||||||
hostname = qualysapi.qg2.apps.qualys.com
|
hostname = qualysapi.qg2.apps.qualys.com
|
||||||
username = exampleuser
|
username = exampleuser
|
||||||
password = examplepass
|
password = examplepass
|
||||||
@ -19,3 +20,7 @@ max_retries = 10
|
|||||||
; proxy authentication
|
; proxy authentication
|
||||||
#proxy_username = proxyuser
|
#proxy_username = proxyuser
|
||||||
#proxy_password = proxypass
|
#proxy_password = proxypass
|
||||||
|
|
||||||
|
[report]
|
||||||
|
# Default template ID for CSVs
|
||||||
|
template_id = 126024
|
||||||
|
@ -1,64 +1,81 @@
|
|||||||
|
#!/usr/bin/python
|
||||||
|
# -*- coding: utf-8 -*-
|
||||||
__author__ = 'Austin Taylor'
|
__author__ = 'Austin Taylor'
|
||||||
|
|
||||||
import qualysapi
|
|
||||||
from lxml import objectify
|
from lxml import objectify
|
||||||
from lxml.builder import E
|
from lxml.builder import E
|
||||||
import xml.etree.ElementTree as ET
|
import xml.etree.ElementTree as ET
|
||||||
import pandas as pd
|
import pandas as pd
|
||||||
|
import qualysapi.config as qcconf
|
||||||
import requests
|
import requests
|
||||||
from requests.packages.urllib3.exceptions import InsecureRequestWarning
|
from requests.packages.urllib3.exceptions import InsecureRequestWarning
|
||||||
|
|
||||||
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
|
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
|
||||||
|
|
||||||
import sys
|
import sys
|
||||||
|
import os
|
||||||
import csv
|
import csv
|
||||||
|
import dateutil.parser as dp
|
||||||
|
|
||||||
class qualysWhisper(object):
|
class qualysWhisper(object):
|
||||||
COUNT = '/count/was/webapp'
|
COUNT = '/count/was/webapp'
|
||||||
VERSION = '/qps/rest/portal/version'
|
DELETE_REPORT = '/delete/was/report/{report_id}'
|
||||||
|
GET_WEBAPP_DETAILS = '/get/was/webapp/{was_id}'
|
||||||
QPS_REST_3 = '/qps/rest/3.0'
|
QPS_REST_3 = '/qps/rest/3.0'
|
||||||
SEARCH_REPORTS = QPS_REST_3 + '/search/was/report'
|
|
||||||
SEARCH_WEB_APPS = QPS_REST_3 + '/search/was/webapp'
|
REPORT_DETAILS = '/get/was/report/{report_id}'
|
||||||
REPORT_DETAILS = QPS_REST_3 + '/get/was/report/{report_id}'
|
REPORT_STATUS = '/status/was/report/{report_id}'
|
||||||
REPORT_STATUS = QPS_REST_3 + '/status/was/report/{report_id}'
|
REPORT_CREATE = '/create/was/report'
|
||||||
REPORT_DOWNLOAD = QPS_REST_3 + '/download/was/report/{report_id}'
|
REPORT_DOWNLOAD = '/download/was/report/{report_id}'
|
||||||
|
SEARCH_REPORTS = '/search/was/report'
|
||||||
|
SEARCH_WEB_APPS = '/search/was/webapp'
|
||||||
|
SEARCH_WAS_SCAN = '/search/was/wasscan'
|
||||||
|
VERSION = '/qps/rest/portal/version'
|
||||||
|
|
||||||
def __init__(self, config=None):
|
def __init__(self, config=None):
|
||||||
self.config = config
|
self.config = config
|
||||||
try:
|
try:
|
||||||
self.qgc = qualysapi.connect(config)
|
self.qgc = qualysapi.connect(config)
|
||||||
print('[SUCCESS] - Connected to Qualys at %s' % self.qgc.server)
|
print('[SUCCESS] - Connected to Qualys at %s' \
|
||||||
|
% self.qgc.server)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print('[ERROR] Could not connect to Qualys - %s' % e)
|
print('[ERROR] Could not connect to Qualys - %s' % e)
|
||||||
self.headers = {
|
self.headers = {'content-type': 'text/xml'}
|
||||||
"content-type": "text/xml"}
|
self.config_parse = qcconf.QualysConnectConfig(config)
|
||||||
|
try:
|
||||||
|
self.template_id = self.config_parse.get_template_id()
|
||||||
|
except:
|
||||||
|
print 'ERROR - Could not retrieve template ID'
|
||||||
|
|
||||||
def request(self, path, method='get'):
|
def request(
|
||||||
methods = {'get': requests.get,
|
self,
|
||||||
'post': requests.post}
|
path,
|
||||||
|
method='get',
|
||||||
|
data=None,
|
||||||
|
):
|
||||||
|
methods = {'get': requests.get, 'post': requests.post}
|
||||||
base = 'https://' + self.qgc.server + path
|
base = 'https://' + self.qgc.server + path
|
||||||
req = methods[method](base, auth=self.qgc.auth, headers=self.headers).content
|
req = methods[method](base, auth=self.qgc.auth, data=data,
|
||||||
|
headers=self.headers).content
|
||||||
return req
|
return req
|
||||||
|
|
||||||
def get_version(self):
|
def get_version(self):
|
||||||
return self.request(self.VERSION)
|
return self.request(self.VERSION)
|
||||||
|
|
||||||
def get_scan_count(self, scan_name):
|
def get_scan_count(self, scan_name):
|
||||||
parameters = (
|
parameters = E.ServiceRequest(E.filters(E.Criteria(scan_name,
|
||||||
E.ServiceRequest(
|
field='name', operator='CONTAINS')))
|
||||||
E.filters(
|
|
||||||
E.Criteria(scan_name, field='name', operator='CONTAINS'))))
|
|
||||||
xml_output = self.qgc.request(self.COUNT, parameters)
|
xml_output = self.qgc.request(self.COUNT, parameters)
|
||||||
root = objectify.fromstring(xml_output)
|
root = objectify.fromstring(xml_output)
|
||||||
return root.count.text
|
return root.count.text
|
||||||
|
|
||||||
def get_reports(self):
|
def get_reports(self):
|
||||||
return self.request(self.SEARCH_REPORTS, method='post')
|
return self.qgc.request(self.SEARCH_REPORTS)
|
||||||
|
|
||||||
def xml_parser(self, xml, dupfield=None):
|
def xml_parser(self, xml, dupfield=None):
|
||||||
all_records = []
|
all_records = []
|
||||||
root = ET.XML(xml)
|
root = ET.XML(xml)
|
||||||
for i, child in enumerate(root):
|
for (i, child) in enumerate(root):
|
||||||
for subchild in child:
|
for subchild in child:
|
||||||
record = {}
|
record = {}
|
||||||
for p in subchild:
|
for p in subchild:
|
||||||
@ -73,140 +90,302 @@ class qualysWhisper(object):
|
|||||||
|
|
||||||
def get_report_list(self):
|
def get_report_list(self):
|
||||||
"""Returns a dataframe of reports"""
|
"""Returns a dataframe of reports"""
|
||||||
|
|
||||||
return self.xml_parser(self.get_reports(), dupfield='user_id')
|
return self.xml_parser(self.get_reports(), dupfield='user_id')
|
||||||
|
|
||||||
def get_web_apps(self):
|
def get_web_apps(self):
|
||||||
"""Returns webapps available for account"""
|
"""Returns webapps available for account"""
|
||||||
return self.request(self.SEARCH_WEB_APPS, method='post')
|
|
||||||
|
return self.qgc.request(self.SEARCH_WEB_APPS)
|
||||||
|
|
||||||
def get_web_app_list(self):
|
def get_web_app_list(self):
|
||||||
"""Returns dataframe of webapps"""
|
"""Returns dataframe of webapps"""
|
||||||
return self.xml_parser(self.get_web_apps(), dupfield='app_id')
|
|
||||||
|
return self.xml_parser(self.get_web_apps(), dupfield='user_id')
|
||||||
|
|
||||||
|
def get_web_app_details(self, was_id):
|
||||||
|
"""Get webapp details - use to retrieve app ID tag"""
|
||||||
|
|
||||||
|
return self.qgc.request(self.GET_WEBAPP_DETAILS.format(was_id=was_id))
|
||||||
|
|
||||||
|
def get_scans_by_app_id(self, app_id):
|
||||||
|
data = self.generate_app_id_scan_XML(app_id)
|
||||||
|
return self.qgc.request(self.SEARCH_WAS_SCAN, data)
|
||||||
|
|
||||||
def get_report_details(self, report_id):
|
def get_report_details(self, report_id):
|
||||||
r = self.REPORT_DETAILS.format(report_id=report_id)
|
return self.qgc.request(self.REPORT_DETAILS.format(report_id=report_id))
|
||||||
return self.request(r)
|
|
||||||
|
|
||||||
def get_report_status(self, report_id):
|
def get_report_status(self, report_id):
|
||||||
r = self.REPORT_STATUS.format(report_id=report_id)
|
return self.qgc.request(self.REPORT_STATUS.format(report_id=report_id))
|
||||||
return self.request(r)
|
|
||||||
|
|
||||||
def download_report(self, report_id):
|
def download_report(self, report_id):
|
||||||
r = self.REPORT_DOWNLOAD.format(report_id=report_id)
|
return self.qgc.request(self.REPORT_DOWNLOAD.format(report_id=report_id))
|
||||||
return self.request(r)
|
|
||||||
|
def generate_webapp_report_XML(self, app_id):
|
||||||
|
"""Generates a CSV report for an asset based on template defined in .ini file"""
|
||||||
|
|
||||||
|
report_xml = \
|
||||||
|
E.ServiceRequest(E.data(E.Report(E.name('![CDATA[API Web Application Report generated by VulnWhisperer]]>'
|
||||||
|
),
|
||||||
|
E.description('<![CDATA[CSV WebApp report for VulnWhisperer]]>'
|
||||||
|
), E.format('CSV'),
|
||||||
|
E.template(E.id(self.template_id)),
|
||||||
|
E.config(E.webAppReport(E.target(E.webapps(E.WebApp(E.id(app_id)))))))))
|
||||||
|
|
||||||
|
return report_xml
|
||||||
|
|
||||||
|
def generate_app_id_scan_XML(self, app_id):
|
||||||
|
report_xml = \
|
||||||
|
E.ServiceRequest(E.filters(E.Criteria({'field': 'webApp.id'
|
||||||
|
, 'operator': 'EQUALS'}, app_id)))
|
||||||
|
|
||||||
|
return report_xml
|
||||||
|
|
||||||
|
def create_report(self, report_id):
|
||||||
|
data = self.generate_webapp_report_XML(report_id)
|
||||||
|
return self.qgc.request(self.REPORT_CREATE.format(report_id=report_id),
|
||||||
|
data)
|
||||||
|
|
||||||
|
def delete_report(self, report_id):
|
||||||
|
return self.qgc.request(self.DELETE_REPORT.format(report_id=report_id))
|
||||||
|
|
||||||
|
|
||||||
class qualysWebAppReport:
|
class qualysWebAppReport:
|
||||||
WEB_APP_VULN_HEADER = ["Web Application Name", "VULNERABILITY", "ID", "QID", "Url", "Param", "Function",
|
CATEGORIES = ['VULNERABILITY', 'SENSITIVE CONTENT',
|
||||||
"Form Entry Point",
|
'INFORMATION GATHERED']
|
||||||
"Access Path", "Authentication", "Ajax Request", "Ajax Request ID", "Status", "Ignored",
|
|
||||||
"Ignore Reason",
|
|
||||||
"Ignore Date", "Ignore User", "Ignore Comments", "First Time Detected",
|
|
||||||
"Last Time Detected", "Last Time Tested",
|
|
||||||
"Times Detected", "Payload #1", "Request Method #1", "Request URL #1",
|
|
||||||
"Request Headers #1", "Response #1", "Evidence #1"]
|
|
||||||
WEB_APP_INFO_HEADER = ["Web Application Name", "INFORMATION GATHERED", "ID", "QID", "Results", "Detection Date"]
|
|
||||||
QID_HEADER = ["QID", "Id", "Title", "Category", "Severity Level", "Groups", "OWASP", "WASC", "CWE", "CVSS Base",
|
|
||||||
"CVSS Temporal", "Description", "Impact", "Solution"]
|
|
||||||
GROUP_HEADER = ["GROUP", "Name", "Category"]
|
|
||||||
OWASP_HEADER = ["OWASP", "Code", "Name"]
|
|
||||||
WASC_HEADER = ["WASC", "Code", "Name"]
|
|
||||||
CATEGORY_HEADER = ["Category", "Severity", "Level", "Description"]
|
|
||||||
|
|
||||||
def __init__(self, config=None, file_in=None, file_stream=False, delimiter=',', quotechar='"'):
|
# URL Vulnerability Information
|
||||||
|
|
||||||
|
WEB_APP_VULN_BLOCK = [
|
||||||
|
'Web Application Name',
|
||||||
|
CATEGORIES[0],
|
||||||
|
'ID',
|
||||||
|
'QID',
|
||||||
|
'Url',
|
||||||
|
'Param',
|
||||||
|
'Function',
|
||||||
|
'Form Entry Point',
|
||||||
|
'Access Path',
|
||||||
|
'Authentication',
|
||||||
|
'Ajax Request',
|
||||||
|
'Ajax Request ID',
|
||||||
|
'Status',
|
||||||
|
'Ignored',
|
||||||
|
'Ignore Reason',
|
||||||
|
'Ignore Date',
|
||||||
|
'Ignore User',
|
||||||
|
'Ignore Comments',
|
||||||
|
'First Time Detected',
|
||||||
|
'Last Time Detected',
|
||||||
|
'Last Time Tested',
|
||||||
|
'Times Detected',
|
||||||
|
'Payload #1',
|
||||||
|
'Request Method #1',
|
||||||
|
'Request URL #1',
|
||||||
|
'Request Headers #1',
|
||||||
|
'Response #1',
|
||||||
|
'Evidence #1',
|
||||||
|
]
|
||||||
|
|
||||||
|
WEB_APP_VULN_HEADER = [
|
||||||
|
'Web Application Name',
|
||||||
|
'Vulnerability Category',
|
||||||
|
'ID',
|
||||||
|
'QID',
|
||||||
|
'Url',
|
||||||
|
'Param',
|
||||||
|
'Function',
|
||||||
|
'Form Entry Point',
|
||||||
|
'Access Path',
|
||||||
|
'Authentication',
|
||||||
|
'Ajax Request',
|
||||||
|
'Ajax Request ID',
|
||||||
|
'Status',
|
||||||
|
'Ignored',
|
||||||
|
'Ignore Reason',
|
||||||
|
'Ignore Date',
|
||||||
|
'Ignore User',
|
||||||
|
'Ignore Comments',
|
||||||
|
'First Time Detected',
|
||||||
|
'Last Time Detected',
|
||||||
|
'Last Time Tested',
|
||||||
|
'Times Detected',
|
||||||
|
'Payload #1',
|
||||||
|
'Request Method #1',
|
||||||
|
'Request URL #1',
|
||||||
|
'Request Headers #1',
|
||||||
|
'Response #1',
|
||||||
|
'Evidence #1',
|
||||||
|
'Content',
|
||||||
|
]
|
||||||
|
|
||||||
|
WEB_APP_VULN_HEADER[WEB_APP_VULN_BLOCK.index(CATEGORIES[0])] = \
|
||||||
|
'Vulnerability Category'
|
||||||
|
|
||||||
|
|
||||||
|
WEB_APP_SENSITIVE_HEADER = list(WEB_APP_VULN_HEADER)
|
||||||
|
WEB_APP_SENSITIVE_HEADER.insert(WEB_APP_SENSITIVE_HEADER.index('Url'
|
||||||
|
), 'Content')
|
||||||
|
|
||||||
|
WEB_APP_SENSITIVE_BLOCK = list(WEB_APP_SENSITIVE_HEADER)
|
||||||
|
WEB_APP_SENSITIVE_BLOCK[WEB_APP_SENSITIVE_BLOCK.index('Vulnerability Category'
|
||||||
|
)] = CATEGORIES[1]
|
||||||
|
|
||||||
|
|
||||||
|
WEB_APP_INFO_HEADER = [
|
||||||
|
'Web Application Name',
|
||||||
|
'Vulnerability Category',
|
||||||
|
'ID',
|
||||||
|
'QID',
|
||||||
|
'Response #1',
|
||||||
|
'Last Time Detected',
|
||||||
|
]
|
||||||
|
WEB_APP_INFO_BLOCK = [
|
||||||
|
'Web Application Name',
|
||||||
|
CATEGORIES[2],
|
||||||
|
'ID',
|
||||||
|
'QID',
|
||||||
|
'Results',
|
||||||
|
'Detection Date',
|
||||||
|
]
|
||||||
|
|
||||||
|
QID_HEADER = [
|
||||||
|
'QID',
|
||||||
|
'Id',
|
||||||
|
'Title',
|
||||||
|
'Category',
|
||||||
|
'Severity Level',
|
||||||
|
'Groups',
|
||||||
|
'OWASP',
|
||||||
|
'WASC',
|
||||||
|
'CWE',
|
||||||
|
'CVSS Base',
|
||||||
|
'CVSS Temporal',
|
||||||
|
'Description',
|
||||||
|
'Impact',
|
||||||
|
'Solution',
|
||||||
|
]
|
||||||
|
GROUP_HEADER = ['GROUP', 'Name', 'Category']
|
||||||
|
OWASP_HEADER = ['OWASP', 'Code', 'Name']
|
||||||
|
WASC_HEADER = ['WASC', 'Code', 'Name']
|
||||||
|
CATEGORY_HEADER = ['Category', 'Severity', 'Level', 'Description']
|
||||||
|
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
config=None,
|
||||||
|
file_in=None,
|
||||||
|
file_stream=False,
|
||||||
|
delimiter=',',
|
||||||
|
quotechar='"',
|
||||||
|
):
|
||||||
self.file_in = file_in
|
self.file_in = file_in
|
||||||
self.file_stream = file_stream
|
self.file_stream = file_stream
|
||||||
self.report = None
|
self.report = None
|
||||||
self.get_sys_max()
|
|
||||||
|
|
||||||
if config:
|
if config:
|
||||||
try:
|
try:
|
||||||
self.qw = qualysWhisper(config=config)
|
self.qw = qualysWhisper(config=config)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print('Could not load config! Please check settings for %s' % config)
|
print('Could not load config! Please check settings for %s' \
|
||||||
|
% e)
|
||||||
|
|
||||||
if file_stream:
|
if file_stream:
|
||||||
self.open_file = file_in.splitlines()
|
self.open_file = file_in.splitlines()
|
||||||
|
|
||||||
elif file_in:
|
elif file_in:
|
||||||
|
|
||||||
self.open_file = open(file_in, 'rb')
|
self.open_file = open(file_in, 'rb')
|
||||||
|
|
||||||
# self.report = csv.reader(self.open_file, delimiter=delimiter, quotechar=quotechar)
|
|
||||||
# self.hostname = self.get_hostname(file_in)
|
|
||||||
self.downloaded_file = None
|
self.downloaded_file = None
|
||||||
|
|
||||||
def get_sys_max(self):
|
|
||||||
maxInt = sys.maxsize
|
|
||||||
decrement = True
|
|
||||||
|
|
||||||
while decrement:
|
|
||||||
# decrease the maxInt value by factor 10
|
|
||||||
# as long as the OverflowError occurs.
|
|
||||||
|
|
||||||
decrement = False
|
|
||||||
try:
|
|
||||||
csv.field_size_limit(maxInt)
|
|
||||||
except OverflowError:
|
|
||||||
maxInt = int(maxInt / 10)
|
|
||||||
decrement = True
|
|
||||||
|
|
||||||
def get_hostname(self, report):
|
def get_hostname(self, report):
|
||||||
host = ''
|
host = ''
|
||||||
with open(report, 'rb') as csvfile:
|
with open(report, 'rb') as csvfile:
|
||||||
q_report = csv.reader(csvfile, delimiter=',', quotechar='"')
|
q_report = csv.reader(csvfile, delimiter=',', quotechar='"')
|
||||||
for x in q_report:
|
for x in q_report:
|
||||||
|
|
||||||
if 'Web Application Name' in x[0]:
|
if 'Web Application Name' in x[0]:
|
||||||
host = q_report.next()[0]
|
host = q_report.next()[0]
|
||||||
return host
|
return host
|
||||||
|
|
||||||
|
def grab_section(
|
||||||
def grab_section(self, report, section, end='', pop_last=False):
|
self,
|
||||||
|
report,
|
||||||
|
section,
|
||||||
|
end=[],
|
||||||
|
pop_last=False,
|
||||||
|
):
|
||||||
temp_list = []
|
temp_list = []
|
||||||
|
max_col_count = 0
|
||||||
with open(report, 'rb') as csvfile:
|
with open(report, 'rb') as csvfile:
|
||||||
# q_report = csv.reader(self., delimiter=',', quotechar='"')
|
|
||||||
q_report = csv.reader(csvfile, delimiter=',', quotechar='"')
|
q_report = csv.reader(csvfile, delimiter=',', quotechar='"')
|
||||||
for line in q_report:
|
for line in q_report:
|
||||||
if set(line) == set(section): # Or whatever test is needed
|
if set(line) == set(section):
|
||||||
break
|
break
|
||||||
|
|
||||||
# Reads text until the end of the block:
|
# Reads text until the end of the block:
|
||||||
for line in q_report: # This keeps reading the file
|
for line in q_report: # This keeps reading the file
|
||||||
temp_list.append(line)
|
temp_list.append(line)
|
||||||
if set(line) == end:
|
|
||||||
|
if line in end:
|
||||||
break
|
break
|
||||||
if pop_last and len(temp_list) > 1:
|
if pop_last and len(temp_list) > 1:
|
||||||
last_line = temp_list.pop(-1)
|
temp_list.pop(-1)
|
||||||
return temp_list
|
return temp_list
|
||||||
|
|
||||||
|
def iso_to_epoch(self, dt):
|
||||||
|
return dp.parse(dt).strftime('%s')
|
||||||
|
|
||||||
def cleanser(self, _data):
|
def cleanser(self, _data):
|
||||||
repls = ('\n', '|||'), ('\r', '|||'), (',', ';'), ('\t', '|||')
|
repls = (('\n', '|||'), ('\r', '|||'), (',', ';'), ('\t', '|||'
|
||||||
data = reduce(lambda a, kv: a.replace(*kv), repls, _data)
|
))
|
||||||
return data
|
if _data:
|
||||||
|
_data = reduce(lambda a, kv: a.replace(*kv), repls, _data)
|
||||||
|
return _data
|
||||||
|
|
||||||
def grab_sections(self, report):
|
def grab_sections(self, report):
|
||||||
all_dataframes = []
|
all_dataframes = []
|
||||||
category_list = []
|
category_list = []
|
||||||
with open(report, 'rb') as csvfile:
|
with open(report, 'rb') as csvfile:
|
||||||
q_report = csv.reader(csvfile, delimiter=',', quotechar='"')
|
q_report = csv.reader(csvfile, delimiter=',', quotechar='"')
|
||||||
all_dataframes.append(pd.DataFrame(
|
all_dataframes.append(pd.DataFrame(self.grab_section(report,
|
||||||
self.grab_section(report, self.WEB_APP_VULN_HEADER, end=set(self.WEB_APP_INFO_HEADER), pop_last=True),
|
self.WEB_APP_VULN_BLOCK,
|
||||||
|
end=[self.WEB_APP_SENSITIVE_BLOCK,
|
||||||
|
self.WEB_APP_INFO_BLOCK],
|
||||||
|
pop_last=True),
|
||||||
columns=self.WEB_APP_VULN_HEADER))
|
columns=self.WEB_APP_VULN_HEADER))
|
||||||
all_dataframes.append(pd.DataFrame(
|
all_dataframes.append(pd.DataFrame(self.grab_section(report,
|
||||||
self.grab_section(report, self.WEB_APP_INFO_HEADER, end=set(self.QID_HEADER), pop_last=True),
|
self.WEB_APP_SENSITIVE_BLOCK,
|
||||||
|
end=[self.WEB_APP_INFO_BLOCK,
|
||||||
|
self.WEB_APP_SENSITIVE_BLOCK],
|
||||||
|
pop_last=True),
|
||||||
|
columns=self.WEB_APP_SENSITIVE_HEADER))
|
||||||
|
all_dataframes.append(pd.DataFrame(self.grab_section(report,
|
||||||
|
self.WEB_APP_INFO_BLOCK,
|
||||||
|
end=[self.QID_HEADER],
|
||||||
|
pop_last=True),
|
||||||
columns=self.WEB_APP_INFO_HEADER))
|
columns=self.WEB_APP_INFO_HEADER))
|
||||||
all_dataframes.append(
|
all_dataframes.append(pd.DataFrame(self.grab_section(report,
|
||||||
pd.DataFrame(self.grab_section(report, self.QID_HEADER, end=set(self.GROUP_HEADER), pop_last=True),
|
self.QID_HEADER,
|
||||||
|
end=[self.GROUP_HEADER],
|
||||||
|
pop_last=True),
|
||||||
columns=self.QID_HEADER))
|
columns=self.QID_HEADER))
|
||||||
all_dataframes.append(
|
all_dataframes.append(pd.DataFrame(self.grab_section(report,
|
||||||
pd.DataFrame(self.grab_section(report, self.GROUP_HEADER, end=set(self.OWASP_HEADER), pop_last=True),
|
self.GROUP_HEADER,
|
||||||
|
end=[self.OWASP_HEADER],
|
||||||
|
pop_last=True),
|
||||||
columns=self.GROUP_HEADER))
|
columns=self.GROUP_HEADER))
|
||||||
all_dataframes.append(
|
all_dataframes.append(pd.DataFrame(self.grab_section(report,
|
||||||
pd.DataFrame(self.grab_section(report, self.OWASP_HEADER, end=set(self.WASC_HEADER), pop_last=True),
|
self.OWASP_HEADER,
|
||||||
|
end=[self.WASC_HEADER],
|
||||||
|
pop_last=True),
|
||||||
columns=self.OWASP_HEADER))
|
columns=self.OWASP_HEADER))
|
||||||
all_dataframes.append(
|
all_dataframes.append(pd.DataFrame(self.grab_section(report,
|
||||||
pd.DataFrame(self.grab_section(report, self.WASC_HEADER, end=set(['APPENDIX']), pop_last=True),
|
self.WASC_HEADER, end=[['APPENDIX']],
|
||||||
|
pop_last=True),
|
||||||
columns=self.WASC_HEADER))
|
columns=self.WASC_HEADER))
|
||||||
all_dataframes.append(
|
all_dataframes.append(pd.DataFrame(self.grab_section(report,
|
||||||
pd.DataFrame(self.grab_section(report, self.CATEGORY_HEADER, end=''), columns=self.CATEGORY_HEADER))
|
self.CATEGORY_HEADER, end=''),
|
||||||
|
columns=self.CATEGORY_HEADER))
|
||||||
|
|
||||||
return all_dataframes
|
return all_dataframes
|
||||||
|
|
||||||
def data_normalizer(self, dataframes):
|
def data_normalizer(self, dataframes):
|
||||||
@ -215,45 +394,116 @@ class qualysWebAppReport:
|
|||||||
:param dataframes:
|
:param dataframes:
|
||||||
:return:
|
:return:
|
||||||
"""
|
"""
|
||||||
merged_df = pd.merge(dataframes[0], dataframes[2], left_on='QID', right_on='Id')
|
|
||||||
merged_df['Payload #1'] = merged_df['Payload #1'].apply(self.cleanser)
|
merged_df = pd.concat([dataframes[0], dataframes[1],
|
||||||
merged_df['Request Method #1'] = merged_df['Request Method #1'].apply(self.cleanser)
|
dataframes[2]], axis=0,
|
||||||
merged_df['Request URL #1'] = merged_df['Request URL #1'].apply(self.cleanser)
|
ignore_index=False).fillna('N/A')
|
||||||
merged_df['Request Headers #1'] = merged_df['Request Headers #1'].apply(self.cleanser)
|
merged_df = pd.merge(merged_df, dataframes[3], left_on='QID',
|
||||||
merged_df['Response #1'] = merged_df['Response #1'].apply(self.cleanser)
|
right_on='Id')
|
||||||
merged_df['Evidence #1'] = merged_df['Evidence #1'].apply(self.cleanser)
|
|
||||||
merged_df['QID_y'] = merged_df['QID_y'].apply(self.cleanser)
|
if 'Content' not in merged_df:
|
||||||
merged_df['Id'] = merged_df['Id'].apply(self.cleanser)
|
merged_df['Content'] = ''
|
||||||
merged_df['Title'] = merged_df['Title'].apply(self.cleanser)
|
|
||||||
merged_df['Category'] = merged_df['Category'].apply(self.cleanser)
|
merged_df['Payload #1'] = merged_df['Payload #1'
|
||||||
merged_df['Severity Level'] = merged_df['Severity Level'].apply(self.cleanser)
|
].apply(self.cleanser)
|
||||||
merged_df['Groups'] = merged_df['Groups'].apply(self.cleanser)
|
merged_df['Request Method #1'] = merged_df['Request Method #1'
|
||||||
merged_df['OWASP'] = merged_df['OWASP'].apply(self.cleanser)
|
].apply(self.cleanser)
|
||||||
merged_df['WASC'] = merged_df['WASC'].apply(self.cleanser)
|
merged_df['Request URL #1'] = merged_df['Request URL #1'
|
||||||
merged_df['CWE'] = merged_df['CWE'].apply(self.cleanser)
|
].apply(self.cleanser)
|
||||||
merged_df['CVSS Base'] = merged_df['CVSS Base'].apply(self.cleanser)
|
merged_df['Request Headers #1'] = merged_df['Request Headers #1'
|
||||||
merged_df['CVSS Temporal'] = merged_df['CVSS Temporal'].apply(self.cleanser)
|
].apply(self.cleanser)
|
||||||
merged_df['Description'] = merged_df['Description'].apply(self.cleanser)
|
merged_df['Response #1'] = merged_df['Response #1'
|
||||||
|
].apply(self.cleanser)
|
||||||
|
merged_df['Evidence #1'] = merged_df['Evidence #1'
|
||||||
|
].apply(self.cleanser)
|
||||||
|
|
||||||
|
merged_df['Description'] = merged_df['Description'
|
||||||
|
].apply(self.cleanser)
|
||||||
merged_df['Impact'] = merged_df['Impact'].apply(self.cleanser)
|
merged_df['Impact'] = merged_df['Impact'].apply(self.cleanser)
|
||||||
merged_df['Solution'] = merged_df['Solution'].apply(self.cleanser)
|
merged_df['Solution'] = merged_df['Solution'
|
||||||
|
].apply(self.cleanser)
|
||||||
|
merged_df['Url'] = merged_df['Url'].apply(self.cleanser)
|
||||||
|
merged_df['Content'] = merged_df['Content'].apply(self.cleanser)
|
||||||
merged_df = merged_df.drop(['QID_y', 'QID_x'], axis=1)
|
merged_df = merged_df.drop(['QID_y', 'QID_x'], axis=1)
|
||||||
|
|
||||||
merged_df = merged_df.rename(columns={'Id': 'QID'})
|
merged_df = merged_df.rename(columns={'Id': 'QID'})
|
||||||
|
|
||||||
|
try:
|
||||||
|
merged_df = \
|
||||||
|
merged_df[~merged_df.Title.str.contains('Links Crawled|External Links Discovered'
|
||||||
|
)]
|
||||||
|
except Exception as e:
|
||||||
|
print(e)
|
||||||
return merged_df
|
return merged_df
|
||||||
|
|
||||||
def download_file(self, file_id):
|
def download_file(self, file_id):
|
||||||
report = self.qw.download_report(file_id)
|
report = self.qw.download_report(file_id)
|
||||||
filename = file_id + '.csv'
|
filename = str(file_id) + '.csv'
|
||||||
file_out = open(filename, 'w')
|
file_out = open(filename, 'w')
|
||||||
for line in report.splitlines():
|
for line in report.splitlines():
|
||||||
file_out.write(line + '\n')
|
file_out.write(line + '\n')
|
||||||
file_out.close()
|
file_out.close()
|
||||||
print('File written to %s' % filename)
|
print('[ACTION] - File written to %s' % filename)
|
||||||
return filename
|
return filename
|
||||||
|
|
||||||
def process_data(self, file_id):
|
def remove_file(self, filename):
|
||||||
|
os.remove(filename)
|
||||||
|
|
||||||
|
def process_data(self, file_id, cleanup=True):
|
||||||
"""Downloads a file from qualys and normalizes it"""
|
"""Downloads a file from qualys and normalizes it"""
|
||||||
|
|
||||||
download_file = self.download_file(file_id)
|
download_file = self.download_file(file_id)
|
||||||
print('Downloading file ID: %s' % file_id)
|
print('[ACTION] - Downloading file ID: %s' % file_id)
|
||||||
report_data = self.grab_sections(download_file)
|
report_data = self.grab_sections(download_file)
|
||||||
merged_data = self.data_normalizer(report_data)
|
merged_data = self.data_normalizer(report_data)
|
||||||
|
|
||||||
|
# TODO cleanup old data (delete)
|
||||||
|
|
||||||
return merged_data
|
return merged_data
|
||||||
|
|
||||||
|
def whisper_webapp(self, report_id, updated_date):
|
||||||
|
"""
|
||||||
|
report_id: App ID
|
||||||
|
updated_date: Last time scan was ran for app_id
|
||||||
|
"""
|
||||||
|
|
||||||
|
try:
|
||||||
|
vuln_ready = None
|
||||||
|
if 'Z' in updated_date:
|
||||||
|
updated_date = self.iso_to_epoch(updated_date)
|
||||||
|
report_name = 'qualys_web_' + str(report_id) \
|
||||||
|
+ '_{last_updated}'.format(last_updated=updated_date) \
|
||||||
|
+ '.csv'
|
||||||
|
if os.path.isfile(report_name):
|
||||||
|
print('[ACTION] - File already exist! Skipping...')
|
||||||
|
pass
|
||||||
|
else:
|
||||||
|
print('[ACTION] - Generating report for %s' % report_id)
|
||||||
|
status = self.qw.create_report(report_id)
|
||||||
|
root = objectify.fromstring(status)
|
||||||
|
if root.responseCode == 'SUCCESS':
|
||||||
|
print('[INFO] - Successfully generated report for webapp: %s' \
|
||||||
|
% report_id)
|
||||||
|
generated_report_id = root.data.Report.id
|
||||||
|
print ('[INFO] - New Report ID: %s' \
|
||||||
|
% generated_report_id)
|
||||||
|
vuln_ready = self.process_data(generated_report_id)
|
||||||
|
|
||||||
|
vuln_ready.to_csv(report_name, index=False) # add when timestamp occured
|
||||||
|
print('[SUCCESS] - Report written to %s' \
|
||||||
|
% report_name)
|
||||||
|
print('[ACTION] - Removing report %s' \
|
||||||
|
% generated_report_id)
|
||||||
|
cleaning_up = \
|
||||||
|
self.qw.delete_report(generated_report_id)
|
||||||
|
os.remove(str(generated_report_id) + '.csv')
|
||||||
|
print('[ACTION] - Deleted report: %s' \
|
||||||
|
% generated_report_id)
|
||||||
|
else:
|
||||||
|
print('Could not process report ID: %s' % status)
|
||||||
|
except Exception as e:
|
||||||
|
print('[ERROR] - Could not process %s - %s' % (report_id, e))
|
||||||
|
return vuln_ready
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
Reference in New Issue
Block a user