Refactored classes to be more modular, update to ini file and submodules

This commit is contained in:
Austin Taylor
2017-12-27 10:38:44 -05:00
parent abe8925ebc
commit 2997e2d2b6
12 changed files with 285 additions and 335 deletions

View File

@ -1 +1 @@
from utils.cli import *
from utils.cli import bcolors

View File

@ -35,7 +35,7 @@ class NessusAPI(object):
'Origin': self.base,
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'en-US,en;q=0.8',
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.96 Safari/537.36',
'User-Agent': 'VulnWhisperer for Nessus',
'Content-Type': 'application/json',
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Referer': self.base,

View File

@ -6,6 +6,7 @@ from lxml import objectify
from lxml.builder import E
import xml.etree.ElementTree as ET
import pandas as pd
import qualysapi
import qualysapi.config as qcconf
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
@ -46,6 +47,7 @@ class qualysWhisper(object):
self.template_id = self.config_parse.get_template_id()
except:
print 'ERROR - Could not retrieve template ID'
sys.exit(2)
def request(
self,
@ -370,28 +372,15 @@ class qualysWebAppReport:
if 'Content' not in merged_df:
merged_df['Content'] = ''
merged_df['Payload #1'] = merged_df['Payload #1'
].apply(self.cleanser)
merged_df['Request Method #1'] = merged_df['Request Method #1'
].apply(self.cleanser)
merged_df['Request URL #1'] = merged_df['Request URL #1'
].apply(self.cleanser)
merged_df['Request Headers #1'] = merged_df['Request Headers #1'
].apply(self.cleanser)
merged_df['Response #1'] = merged_df['Response #1'
].apply(self.cleanser)
merged_df['Evidence #1'] = merged_df['Evidence #1'
].apply(self.cleanser)
columns_to_cleanse = ['Payload #1','Request Method #1','Request URL #1',
'Request Headers #1','Response #1','Evidence #1',
'Description','Impact','Solution','Url','Content']
for col in columns_to_cleanse:
merged_df[col] = merged_df[col].apply(self.cleanser)
merged_df['Description'] = merged_df['Description'
].apply(self.cleanser)
merged_df['Impact'] = merged_df['Impact'].apply(self.cleanser)
merged_df['Solution'] = merged_df['Solution'
].apply(self.cleanser)
merged_df['Url'] = merged_df['Url'].apply(self.cleanser)
merged_df['Content'] = merged_df['Content'].apply(self.cleanser)
merged_df = merged_df.drop(['QID_y', 'QID_x'], axis=1)
merged_df = merged_df.rename(columns={'Id': 'QID'})
try:
@ -427,49 +416,15 @@ class qualysWebAppReport:
return merged_data
def whisper_webapp(self, report_id, updated_date):
"""
report_id: App ID
updated_date: Last time scan was ran for app_id
"""
vuln_ready = None
try:
if 'Z' in updated_date:
updated_date = self.iso_to_epoch(updated_date)
report_name = 'qualys_web_' + str(report_id) \
+ '_{last_updated}'.format(last_updated=updated_date) \
+ '.csv'
if os.path.isfile(report_name):
print('[ACTION] - File already exist! Skipping...')
pass
else:
print('[ACTION] - Generating report for %s' % report_id)
status = self.qw.create_report(report_id)
root = objectify.fromstring(status)
if root.responseCode == 'SUCCESS':
print('[INFO] - Successfully generated report for webapp: %s' \
% report_id)
generated_report_id = root.data.Report.id
print ('[INFO] - New Report ID: %s' \
% generated_report_id)
vuln_ready = self.process_data(generated_report_id)
vuln_ready.to_csv(report_name, index=False, header=True) # add when timestamp occured
print('[SUCCESS] - Report written to %s' \
% report_name)
print('[ACTION] - Removing report %s' \
% generated_report_id)
cleaning_up = \
self.qw.delete_report(generated_report_id)
os.remove(str(generated_report_id) + '.csv')
print('[ACTION] - Deleted report: %s' \
% generated_report_id)
else:
print('Could not process report ID: %s' % status)
except Exception as e:
print('[ERROR] - Could not process %s - %s' % (report_id, e))
return vuln_ready
maxInt = sys.maxsize
decrement = True
while decrement:
decrement = False
try:
csv.field_size_limit(maxInt)
except OverflowError:
maxInt = int(maxInt/10)
decrement = True

View File

@ -12,5 +12,6 @@ class bcolors:
UNDERLINE = '\033[4m'
INFO = '{info}[INFO]{endc}'.format(info=OKBLUE, endc=ENDC)
ACTION = '{info}[ACTION]{endc}'.format(info=OKBLUE, endc=ENDC)
SUCCESS = '{green}[SUCCESS]{endc}'.format(green=OKGREEN, endc=ENDC)
FAIL = '{red}[FAIL]{endc}'.format(red=FAIL, endc=ENDC)

View File

@ -4,8 +4,10 @@ __author__ = 'Austin Taylor'
from base.config import vwConfig
from frameworks.nessus import NessusAPI
from frameworks.qualys import qualysWebAppReport
from utils.cli import bcolors
import pandas as pd
from lxml import objectify
import sys
import os
import io
@ -18,6 +20,9 @@ import logging
class vulnWhispererBase(object):
CONFIG_SECTION = None
def __init__(
self,
config=None,
@ -27,84 +32,31 @@ class vulnWhispererBase(object):
debug=False,
username=None,
password=None,
):
pass
section=None,
):
class vulnWhisperer(object):
def __init__(
self,
config=None,
db_name='report_tracker.db',
purge=False,
verbose=None,
debug=False,
username=None,
password=None,
):
self.verbose = verbose
self.nessus_connect = False
self.develop = True
if self.CONFIG_SECTION is None:
raise Exception('Implementing class must define CONFIG_SECTION')
self.db_name = db_name
self.purge = purge
if config is not None:
try:
self.config = vwConfig(config_in=config)
self.nessus_enabled = self.config.getbool('nessus',
'enabled')
self.config = vwConfig(config_in=config)
self.enabled = self.config.get(self.CONFIG_SECTION, 'enabled')
self.hostname = self.config.get(self.CONFIG_SECTION, 'hostname')
self.username = self.config.get(self.CONFIG_SECTION, 'username')
self.password = self.config.get(self.CONFIG_SECTION, 'password')
self.write_path = self.config.get(self.CONFIG_SECTION, 'write_path')
self.db_path = self.config.get(self.CONFIG_SECTION, 'db_path')
self.verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
if self.nessus_enabled:
self.nessus_hostname = self.config.get('nessus',
'hostname')
self.nessus_port = self.config.get('nessus', 'port')
if password:
self.nessus_password = password
else:
self.nessus_password = self.config.get('nessus'
, 'password')
if username:
self.nessus_username = username
else:
self.nessus_username = self.config.get('nessus'
, 'username')
self.nessus_writepath = self.config.get('nessus',
'write_path')
self.nessus_dbpath = self.config.get('nessus',
'db_path')
self.nessus_trash = self.config.getbool('nessus',
'trash')
self.verbose = self.config.getbool('nessus',
'verbose')
try:
self.vprint('{info} Attempting to connect to nessus...'.format(info=bcolors.INFO))
self.nessus = \
NessusAPI(hostname=self.nessus_hostname,
port=self.nessus_port,
username=self.nessus_username,
password=self.nessus_password)
self.nessus_connect = True
self.vprint('{success} Connected to nessus on {host}:{port}'.format(success=bcolors.SUCCESS,
host=self.nessus_hostname,
port=str(self.nessus_port)))
except Exception as e:
self.vprint(e)
raise Exception(
'{fail} Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format(
config=self.config,
fail=bcolors.FAIL, e=e))
except Exception as e:
self.vprint('{fail} Could not properly load your config!\nReason: {e}'.format(fail=bcolors.FAIL,
e=e))
sys.exit(0)
if db_name is not None:
if self.nessus_dbpath:
self.database = os.path.join(self.nessus_dbpath,
if self.db_name is not None:
if self.db_path:
self.database = os.path.join(self.db_path,
db_name)
else:
self.database = \
@ -137,6 +89,7 @@ class vulnWhisperer(object):
'uuid',
'processed',
]
self.init()
self.uuids = self.retrieve_uuids()
self.processed = 0
@ -145,11 +98,14 @@ class vulnWhisperer(object):
def vprint(self, msg):
if self.verbose:
print msg
print(msg)
def create_table(self):
self.cur.execute(
'CREATE TABLE IF NOT EXISTS scan_history (id INTEGER PRIMARY KEY, scan_name TEXT, scan_id INTEGER, last_modified DATE, filename TEXT, download_time DATE, record_count INTEGER, source TEXT, uuid TEXT, processed INTEGER)'
'CREATE TABLE IF NOT EXISTS scan_history (id INTEGER PRIMARY KEY,'
' scan_name TEXT, scan_id INTEGER, last_modified DATE, filename TEXT,'
' download_time DATE, record_count INTEGER, source TEXT,'
' uuid TEXT, processed INTEGER)'
)
self.conn.commit()
@ -168,10 +124,83 @@ class vulnWhisperer(object):
return data
def path_check(self, _data):
if self.nessus_writepath:
data = self.nessus_writepath + '/' + _data
if self.write_path:
data = self.write_path + '/' + _data
return data
def record_insert(self, record):
self.cur.execute('insert into scan_history({table_columns}) values (?,?,?,?,?,?,?,?,?)'.format(
table_columns=', '.join(self.table_columns)),
record)
self.conn.commit()
def retrieve_uuids(self):
"""
Retrieves UUIDs from database and checks list to determine which files need to be processed.
:return:
"""
try:
self.conn.text_factory = str
self.cur.execute('SELECT uuid FROM scan_history where source = {config_section}'.format(config_section=self.CONFIG_SECTION))
results = frozenset([r[0] for r in self.cur.fetchall()])
except:
results = []
return results
class vulnWhispererNessus(vulnWhispererBase):
CONFIG_SECTION = 'nessus'
def __init__(
self,
config=None,
db_name='report_tracker.db',
purge=False,
verbose=None,
debug=False,
username=None,
password=None,
):
super(vulnWhispererNessus, self).__init__(config=config)
self.port = int(self.config.get(self.CONFIG_NAME, 'port'))
self.develop = True
self.purge = purge
if config is not None:
try:
#if self.enabled:
self.nessus_port = self.config.get(self.CONFIG_SECTION, 'port')
self.nessus_trash = self.config.getbool(self.CONFIG_SECTION,
'trash')
try:
self.vprint('{info} Attempting to connect to nessus...'.format(info=bcolors.INFO))
self.nessus = \
NessusAPI(hostname=self.hostname,
port=self.nessus_port,
username=self.username,
password=self.password)
self.nessus_connect = True
self.vprint('{success} Connected to nessus on {host}:{port}'.format(success=bcolors.SUCCESS,
host=self.hostname,
port=str(self.nessus_port)))
except Exception as e:
self.vprint(e)
raise Exception(
'{fail} Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format(
config=self.config,
fail=bcolors.FAIL, e=e))
except Exception as e:
self.vprint('{fail} Could not properly load your config!\nReason: {e}'.format(fail=bcolors.FAIL,
e=e))
sys.exit(0)
def scan_count(self, scans, completed=False):
"""
@ -206,32 +235,15 @@ class vulnWhisperer(object):
]))
scan_records.append(record.copy())
except Exception as e:
# Generates error each time nonetype is encountered.
# print(e)
pass
if completed:
scan_records = [s for s in scan_records if s['status']
== 'completed']
scan_records = [s for s in scan_records if s['status'] == 'completed']
return scan_records
def record_insert(self, record):
self.cur.execute('insert into scan_history({table_columns}) values (?,?,?,?,?,?,?,?,?)'.format(
table_columns=', '.join(self.table_columns)),
record)
self.conn.commit()
def retrieve_uuids(self):
"""
Retrieves UUIDs from database and checks list to determine which files need to be processed.
:return:
"""
self.conn.text_factory = str
self.cur.execute('SELECT uuid FROM scan_history')
results = frozenset([r[0] for r in self.cur.fetchall()])
return results
def whisper_nessus(self):
if self.nessus_connect:
@ -299,12 +311,9 @@ class vulnWhisperer(object):
if status == 'completed':
file_name = '%s_%s_%s_%s.%s' % (scan_name, scan_id,
history_id, norm_time, 'csv')
repls = (('\\', '_'), ('/', '_'), ('/', '_'), (' ',
'_'))
file_name = reduce(lambda a, kv: a.replace(*kv),
repls, file_name)
relative_path_name = self.path_check(folder_name
+ '/' + file_name)
repls = (('\\', '_'), ('/', '_'), ('/', '_'), (' ', '_'))
file_name = reduce(lambda a, kv: a.replace(*kv), repls, file_name)
relative_path_name = self.path_check(folder_name + '/' + file_name)
if os.path.isfile(relative_path_name):
if self.develop:
@ -335,23 +344,14 @@ class vulnWhisperer(object):
self.vprint('Processing %s/%s for scan: %s'
% (scan_count, len(scan_history),
scan_name))
clean_csv['CVSS'] = clean_csv['CVSS'
].astype(str).apply(self.cleanser)
clean_csv['CVE'] = clean_csv['CVE'
].astype(str).apply(self.cleanser)
clean_csv['Description'] = \
clean_csv['Description'
].astype(str).apply(self.cleanser)
columns_to_cleanse = ['CVSS','CVE','Description','Synopsis','Solution','See Also','Plugin Output']
for col in columns_to_cleanse:
clean_csv[col] = clean_csv[col].astype(str).apply(self.cleanser)
clean_csv['Synopsis'] = \
clean_csv['Description'
].astype(str).apply(self.cleanser)
clean_csv['Solution'] = clean_csv['Solution'
].astype(str).apply(self.cleanser)
clean_csv['See Also'] = clean_csv['See Also'
].astype(str).apply(self.cleanser)
clean_csv['Plugin Output'] = \
clean_csv['Plugin Output'
].astype(str).apply(self.cleanser)
clean_csv.to_csv(relative_path_name,
index=False)
record_meta = (
@ -391,8 +391,129 @@ class vulnWhisperer(object):
else:
self.vprint('{fail} Failed to use scanner at {host}'.format(fail=bcolors.FAIL,
host=self.nessus_hostname + ':'
host=self.hostname + ':'
+ self.nessus_port))
class vulnWhispererQualys(vulnWhispererBase):
CONFIG_SECTION = 'qualys'
def __init__(
self,
config=None,
db_name='report_tracker.db',
purge=False,
verbose=None,
debug=False,
username=None,
password=None,
):
super(vulnWhispererQualys, self).__init__(config=config, )
self.qualys_web = qualysWebAppReport(config=config)
self.latest_scans = self.qualys_web.qw.get_web_app_list()
def whisper_webapp(self, report_id, updated_date):
"""
report_id: App ID
updated_date: Last time scan was ran for app_id
"""
vuln_ready = None
try:
if 'Z' in updated_date:
updated_date = self.qualys_web.iso_to_epoch(updated_date)
report_name = 'qualys_web_' + str(report_id) \
+ '_{last_updated}'.format(last_updated=updated_date) \
+ '.csv'
if os.path.isfile(report_name):
print('{action} - File already exist! Skipping...'.format(action=bcolors.ACTION))
pass
else:
print('{action} - Generating report for %s'.format(action=bcolors.ACTION) % report_id)
status = self.qualys_web.qw.create_report(report_id)
root = objectify.fromstring(status)
if root.responseCode == 'SUCCESS':
print('{info} - Successfully generated report for webapp: %s'.format(info=bcolors.INFO) \
% report_id)
generated_report_id = root.data.Report.id
print('{info} - New Report ID: %s'.format(info=bcolors.INFO) \
% generated_report_id)
vuln_ready = self.qualys_web.process_data(generated_report_id)
vuln_ready.to_csv(report_name, index=False, header=True) # add when timestamp occured
print('{success} - Report written to %s'.format(success=bcolors.SUCCESS) \
% report_name)
print('{action} - Removing report %s'.format(action=bcolors.ACTION) \
% generated_report_id)
cleaning_up = \
self.qualys_web.qw.delete_report(generated_report_id)
os.remove(str(generated_report_id) + '.csv')
print('{action} - Deleted report: %s'.format(action=bcolors.ACTION) \
% generated_report_id)
else:
print('{error} Could not process report ID: %s'.format(error=bcolors.FAIL) % status)
except Exception as e:
print('{error} - Could not process %s - %s'.format(error=bcolors.FAIL) % (report_id, e))
return vuln_ready
def process_web_assets(self):
counter = 0
for app in self.latest_scans.iterrows():
counter += 1
print('Processing %s/%s' % (counter, len(self.latest_scans)))
self.whisper_webapp(app[1]['id'], app[1]['createdDate'])
class vulnWhisperer(object):
def __init__(self,
profile=None,
verbose=None,
username=None,
password=None,
config=None):
self.profile = profile
self.config = config
self.username = username
self.password = password
self.verbose = verbose
def whisper_vulnerabilities(self):
if self.profile == 'nessus':
vw = vulnWhispererNessus(config=self.config, username=self.username, password=self.password, verbose=self.verbose)
vw.whisper_nessus()
elif self.profile == 'qualys':
vw = vulnWhispererQualys(config=self.config)
vw.process_web_assets()
'''
for f in folders:
if not os.path.exists(self.path_check(f['name'])):
if f['name'] == 'Trash' and self.nessus_trash:
os.makedirs(self.path_check(f['name']))
elif f['name'] != 'Trash':
os.makedirs(self.path_check(f['name']))
else:
os.path.exists(self.path_check(f['name']))
self.vprint('{info} Directory already exist for {scan} - Skipping creation'.format(
scan=self.path_check(f['name'
]), info=bcolors.INFO))
'''