Cyberdefense/VulnWhisperer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Addition of OpenVas -- ready for alpha

Browse Source
This commit is contained in:
Austin Taylor
2018-03-03 15:53:23 -05:00
parent d8e813ff5a
commit 53b0b27cb2
3 changed files with 161 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
configs/frameworks_example.ini
View File

@ -22,16 +22,19 @@ verbose=true
# Set the maximum number of retries each connection should attempt.
#Note, this applies only to failed connections and timeouts, never to requests where the server returns a response.
max_retries = 10
# Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID.
template_id = 126024
[openvas]
enabled = true
hostname = localhost
port = 4000
username = exampleuser
password = examplepass
write_path=/opt/vulnwhisp/openvas/
db_path=/opt/vulnwhisp/database
verbose=true
report_format_id=c1645568-627a-11e3-a660-406186ea4fc5
#[proxy]
; This section is optional. Leave it out if you're not using a proxy.

18
vulnwhisp/frameworks/openvas.py
View File

@ -7,7 +7,6 @@ import io
import json
import pandas as pd
import requests
import requests
from bs4 import BeautifulSoup
from requests.packages.urllib3.exceptions import InsecureRequestWarning
@ -47,7 +46,7 @@ class OpenVAS_API(object):
self.login()
self.open_vas_reports = self.get_reports()
self.openvas_reports = self.get_reports()
def vprint(self, msg):
if self.verbose:
@ -113,7 +112,7 @@ class OpenVAS_API(object):
return token
def get_reports(self, complete=True):
print('Retreiving OpenVAS report data...')
print('[INFO] Retreiving OpenVAS report data...')
params = (('cmd', 'get_reports'), ('token', self.token))
reports = self.request(self.OMP, params=params, method='GET')
soup = BeautifulSoup(reports.text, 'lxml')
@ -128,27 +127,28 @@ class OpenVAS_API(object):
links.extend([a['href'] for a in row.find_all('a', href=True) if 'get_report' in str(a)])
cols = [ele.text.strip() for ele in cols]
data.append([ele for ele in cols if ele])
report = pd.DataFrame(data, columns=['date', 'status', 'task', 'severity', 'high', 'medium', 'low', 'log',
report = pd.DataFrame(data, columns=['date', 'status', 'task', 'scan_severity', 'high', 'medium', 'low', 'log',
'false_pos'])
if report.shape[0] != 0:
report['links'] = links
report['report_ids'] = report.links.str.extract('.*report_id=([a-z-0-9]*)')
report['report_ids'] = report.links.str.extract('.*report_id=([a-z-0-9]*)', expand=False)
report['epoch'] = (pd.to_datetime(report['date']) - dt.datetime(1970, 1, 1)).dt.total_seconds().astype(int)
else:
raise Exception("Could not retrieve OpenVAS Reports - Please check your settings and try again")
report['links'] = links
report['report_ids'] = report.links.str.extract('.*report_id=([a-z-0-9]*)')
report['report_ids'] = report.links.str.extract('.*report_id=([a-z-0-9]*)', expand=False)
report['epoch'] = (pd.to_datetime(report['date']) - dt.datetime(1970, 1, 1)).dt.total_seconds().astype(int)
if complete:
report = report[report.status == 'Done']
severity_extraction = report.severity.str.extract('([0-9.]*) \(([\w]+)\)')
severity_extraction.columns = ['severity', 'severity_rate']
severity_extraction = report.scan_severity.str.extract('([0-9.]*) \(([\w]+)\)', expand=False)
severity_extraction.columns = ['scan_highest_severity', 'severity_rate']
report_with_severity = pd.concat([report, severity_extraction], axis=1)
return report_with_severity
def process_report(self, report_id):
params = (
('token', self.token),
('cmd', 'get_report'),
@ -163,5 +163,5 @@ class OpenVAS_API(object):
report_df = pd.read_csv(io.BytesIO(req.text.encode('utf-8')))
report_df['report_ids'] = report_id
self.processed_reports += 1
merged_df = pd.merge(report_df, self.open_vas_reports, on='report_ids').drop('index', axis=1)
merged_df = pd.merge(report_df, self.openvas_reports, on='report_ids').reset_index().drop('index', axis=1)
return merged_df

160
vulnwhisp/vulnwhisp.py
View File

@ -5,6 +5,7 @@ __author__ = 'Austin Taylor'
from base.config import vwConfig
from frameworks.nessus import NessusAPI
from frameworks.qualys import qualysScanReport
from frameworks.openvas import OpenVAS_API
from utils.cli import bcolors
import pandas as pd
from lxml import objectify
@ -44,6 +45,7 @@ class vulnWhispererBase(object):
self.purge = purge
self.develop = develop
if config is not None:
self.config = vwConfig(config_in=config)
self.enabled = self.config.get(self.CONFIG_SECTION, 'enabled')
@ -160,6 +162,18 @@ class vulnWhispererBase(object):
results = []
return results
def directory_check(self):
if not os.path.exists(self.write_path):
os.makedirs(self.write_path)
self.vprint('{info} Directory created at {scan} - Skipping creation'.format(
scan=self.write_path, info=bcolors.INFO))
else:
os.path.exists(self.write_path)
self.vprint('{info} Directory already exist for {scan} - Skipping creation'.format(
scan=self.write_path, info=bcolors.INFO))
class vulnWhispererNessus(vulnWhispererBase):
CONFIG_SECTION = 'nessus'
@ -469,24 +483,13 @@ class vulnWhispererQualys(vulnWhispererBase):
password=None,
):
super(vulnWhispererQualys, self).__init__(config=config, )
super(vulnWhispererQualys, self).__init__(config=config)
self.qualys_scan = qualysScanReport(config=config)
self.latest_scans = self.qualys_scan.qw.get_all_scans()
self.directory_check()
self.scans_to_process = None
def directory_check(self):
if not os.path.exists(self.write_path):
os.makedirs(self.write_path)
self.vprint('{info} Directory created at {scan} - Skipping creation'.format(
scan=self.write_path, info=bcolors.INFO))
else:
os.path.exists(self.write_path)
self.vprint('{info} Directory already exist for {scan} - Skipping creation'.format(
scan=self.write_path, info=bcolors.INFO))
def whisper_reports(self,
report_id=None,
launched_date=None,
@ -609,6 +612,135 @@ class vulnWhispererQualys(vulnWhispererBase):
exit(0)
class vulnWhispererOpenVAS(vulnWhispererBase):
CONFIG_SECTION = 'openvas'
COLUMN_MAPPING = {'IP': 'asset',
'Hostname': 'hostname',
'Port': 'port',
'Port Protocol': 'protocol',
'CVSS': 'cvss',
'Severity': 'severity',
'Solution Type': 'category',
'NVT Name': 'plugin_name',
'Summary': 'synopsis',
'Specific Result': 'plugin_output',
'NVT OID': 'nvt_oid',
'Task ID': 'task_id',
'Task Name': 'task_name',
'Timestamp': 'timestamp',
'Result ID': 'result_id',
'Impact': 'description',
'Solution': 'solution',
'Affected Software/OS': 'affected_software',
'Vulnerability Insight': 'vulnerability_insight',
'Vulnerability Detection Method': 'vulnerability_detection_method',
'Product Detection Result': 'product_detection_result',
'BIDs': 'bids',
'CERTs': 'certs',
'Other References': 'see_also'
}
def __init__(
self,
config=None,
db_name='report_tracker.db',
purge=False,
verbose=None,
debug=False,
username=None,
password=None,
):
super(vulnWhispererOpenVAS, self).__init__(config=config)
self.port = int(self.config.get(self.CONFIG_SECTION, 'port'))
self.template_id = self.config.get(self.CONFIG_SECTION, 'report_format_id')
self.develop = True
self.purge = purge
self.scans_to_process = None
self.openvas_api = OpenVAS_API(hostname=self.hostname, port=self.port, username=self.username,
password=self.password)
def whisper_reports(self, output_format='json', launched_date=None, report_id=None, cleanup=True):
report = None
if report_id:
print('Processing report ID: %s' % report_id)
vuln_ready = self.openvas_api.process_report(report_id=report_id)
scan_name = report_id.replace('-', '')
vuln_ready['scan_name'] = scan_name
vuln_ready['scan_reference'] = report_id
vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
report_name = 'openvas_scan_{scan_name}_{last_updated}.{extension}'.format(scan_name=scan_name,
last_updated=launched_date,
extension=output_format)
relative_path_name = self.path_check(report_name)
scan_reference = report_id
print relative_path_name
if os.path.isfile(relative_path_name):
# TODO Possibly make this optional to sync directories
file_length = len(open(relative_path_name).readlines())
record_meta = (
scan_name,
scan_reference,
launched_date,
report_name,
time.time(),
file_length,
self.CONFIG_SECTION,
report_id,
1,
)
self.record_insert(record_meta)
self.vprint('{info} File {filename} already exist! Updating database'.format(info=bcolors.INFO,
filename=relative_path_name))
record_meta = (
scan_name,
scan_reference,
launched_date,
report_name,
time.time(),
vuln_ready.shape[0],
self.CONFIG_SECTION,
report_id,
1,
)
vuln_ready.port = vuln_ready.port.fillna(0).astype(int)
if output_format == 'json':
with open(relative_path_name, 'w') as f:
f.write(vuln_ready.to_json(orient='records', lines=True))
print('{success} - Report written to %s'.format(success=bcolors.SUCCESS) \
% report_name)
return report
def identify_scans_to_process(self):
if self.uuids:
self.scans_to_process = self.openvas_api.openvas_reports[
~self.openvas_api.openvas_reports.report_ids.isin(self.uuids)]
else:
self.scans_to_process = self.openvas_api.openvas_reports
self.vprint('{info} Identified {new} scans to be processed'.format(info=bcolors.INFO,
new=len(self.scans_to_process)))
def process_openvas_scans(self):
counter = 0
self.identify_scans_to_process()
if self.scans_to_process.shape[0]:
for scan in self.scans_to_process.iterrows():
counter += 1
info = scan[1]
print(
'[INFO] Processing %s/%s - Report ID: %s' % (counter, len(self.scans_to_process), info['report_ids']))
self.whisper_reports(report_id=info['report_ids'],
launched_date=info['epoch'])
self.vprint('{info} Processing complete!'.format(info=bcolors.INFO))
else:
self.vprint('{info} No new scans to process. Exiting...'.format(info=bcolors.INFO))
self.conn.close()
exit(0)
@ -640,3 +772,7 @@ class vulnWhisperer(object):
elif self.profile == 'qualys':
vw = vulnWhispererQualys(config=self.config)
vw.process_web_assets()
elif self.profile == 'openvas':
vw_openvas = vulnWhispererOpenVAS(config=self.config)
vw_openvas.process_openvas_scans()