Addition of OpenVas -- ready for alpha

configs/frameworks_example.ini

@@ -22,16 +22,19 @@ verbose=true
 # Set the maximum number of retries each connection should attempt.
 #Note, this applies only to failed connections and timeouts, never to requests where the server returns a response.
 max_retries = 10
+# Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID.
 template_id = 126024
 
 [openvas]
 enabled = true
 hostname = localhost
+port = 4000
 username = exampleuser
 password = examplepass
 write_path=/opt/vulnwhisp/openvas/
 db_path=/opt/vulnwhisp/database
 verbose=true
+report_format_id=c1645568-627a-11e3-a660-406186ea4fc5
 
 #[proxy]
 ; This section is optional. Leave it out if you're not using a proxy.

vulnwhisp/frameworks/openvas.py

@@ -7,7 +7,6 @@ import io
 import json
 import pandas as pd
 import requests
-import requests
 from bs4 import BeautifulSoup
 from requests.packages.urllib3.exceptions import InsecureRequestWarning
 
@@ -47,7 +46,7 @@ class OpenVAS_API(object):
 
         self.login()
 
-        self.open_vas_reports = self.get_reports()
+        self.openvas_reports = self.get_reports()
 
     def vprint(self, msg):
         if self.verbose:
@@ -113,7 +112,7 @@ class OpenVAS_API(object):
         return token
 
     def get_reports(self, complete=True):
-        print('Retreiving OpenVAS report data...')
+        print('[INFO] Retreiving OpenVAS report data...')
         params = (('cmd', 'get_reports'), ('token', self.token))
         reports = self.request(self.OMP, params=params, method='GET')
         soup = BeautifulSoup(reports.text, 'lxml')
@@ -128,27 +127,28 @@ class OpenVAS_API(object):
             links.extend([a['href'] for a in row.find_all('a', href=True) if 'get_report' in str(a)])
             cols = [ele.text.strip() for ele in cols]
             data.append([ele for ele in cols if ele])
-            report = pd.DataFrame(data, columns=['date', 'status', 'task', 'severity', 'high', 'medium', 'low', 'log',
+            report = pd.DataFrame(data, columns=['date', 'status', 'task', 'scan_severity', 'high', 'medium', 'low', 'log',
                                                  'false_pos'])
 
         if report.shape[0] != 0:
             report['links'] = links
-            report['report_ids'] = report.links.str.extract('.*report_id=([a-z-0-9]*)')
+            report['report_ids'] = report.links.str.extract('.*report_id=([a-z-0-9]*)', expand=False)
             report['epoch'] = (pd.to_datetime(report['date']) - dt.datetime(1970, 1, 1)).dt.total_seconds().astype(int)
         else:
             raise Exception("Could not retrieve OpenVAS Reports - Please check your settings and try again")
 
         report['links'] = links
-        report['report_ids'] = report.links.str.extract('.*report_id=([a-z-0-9]*)')
+        report['report_ids'] = report.links.str.extract('.*report_id=([a-z-0-9]*)', expand=False)
         report['epoch'] = (pd.to_datetime(report['date']) - dt.datetime(1970, 1, 1)).dt.total_seconds().astype(int)
         if complete:
             report = report[report.status == 'Done']
-        severity_extraction = report.severity.str.extract('([0-9.]*) \(([\w]+)\)')
+        severity_extraction = report.scan_severity.str.extract('([0-9.]*) \(([\w]+)\)', expand=False)
-        severity_extraction.columns = ['severity', 'severity_rate']
+        severity_extraction.columns = ['scan_highest_severity', 'severity_rate']
         report_with_severity = pd.concat([report, severity_extraction], axis=1)
         return report_with_severity
 
     def process_report(self, report_id):
+
         params = (
             ('token', self.token),
             ('cmd', 'get_report'),
@@ -163,5 +163,5 @@ class OpenVAS_API(object):
         report_df = pd.read_csv(io.BytesIO(req.text.encode('utf-8')))
         report_df['report_ids'] = report_id
         self.processed_reports += 1
-        merged_df = pd.merge(report_df, self.open_vas_reports, on='report_ids').drop('index', axis=1)
+        merged_df = pd.merge(report_df, self.openvas_reports, on='report_ids').reset_index().drop('index', axis=1)
         return merged_df

vulnwhisp/vulnwhisp.py

@@ -5,6 +5,7 @@ __author__ = 'Austin Taylor'
 from base.config import vwConfig
 from frameworks.nessus import NessusAPI
 from frameworks.qualys import qualysScanReport
+from frameworks.openvas import OpenVAS_API
 from utils.cli import bcolors
 import pandas as pd
 from lxml import objectify
@@ -44,6 +45,7 @@ class vulnWhispererBase(object):
         self.purge = purge
         self.develop = develop
 
+
         if config is not None:
             self.config = vwConfig(config_in=config)
             self.enabled = self.config.get(self.CONFIG_SECTION, 'enabled')
@@ -160,6 +162,18 @@ class vulnWhispererBase(object):
             results = []
         return results
 
+    def directory_check(self):
+        if not os.path.exists(self.write_path):
+            os.makedirs(self.write_path)
+            self.vprint('{info} Directory created at {scan} - Skipping creation'.format(
+                scan=self.write_path, info=bcolors.INFO))
+        else:
+            os.path.exists(self.write_path)
+            self.vprint('{info} Directory already exist for {scan} - Skipping creation'.format(
+                scan=self.write_path, info=bcolors.INFO))
+
+
+
 class vulnWhispererNessus(vulnWhispererBase):
 
     CONFIG_SECTION = 'nessus'
@@ -469,24 +483,13 @@ class vulnWhispererQualys(vulnWhispererBase):
             password=None,
         ):
 
-        super(vulnWhispererQualys, self).__init__(config=config, )
+        super(vulnWhispererQualys, self).__init__(config=config)
 
         self.qualys_scan = qualysScanReport(config=config)
         self.latest_scans = self.qualys_scan.qw.get_all_scans()
         self.directory_check()
         self.scans_to_process = None
 
-
-    def directory_check(self):
-        if not os.path.exists(self.write_path):
-            os.makedirs(self.write_path)
-            self.vprint('{info} Directory created at {scan} - Skipping creation'.format(
-                scan=self.write_path, info=bcolors.INFO))
-        else:
-            os.path.exists(self.write_path)
-            self.vprint('{info} Directory already exist for {scan} - Skipping creation'.format(
-                scan=self.write_path, info=bcolors.INFO))
-
     def whisper_reports(self,
                         report_id=None,
                         launched_date=None,
@@ -609,6 +612,135 @@ class vulnWhispererQualys(vulnWhispererBase):
         exit(0)
 
 
+class vulnWhispererOpenVAS(vulnWhispererBase):
+    CONFIG_SECTION = 'openvas'
+    COLUMN_MAPPING = {'IP': 'asset',
+                      'Hostname': 'hostname',
+                      'Port': 'port',
+                      'Port Protocol': 'protocol',
+                      'CVSS': 'cvss',
+                      'Severity': 'severity',
+                      'Solution Type': 'category',
+                      'NVT Name': 'plugin_name',
+                      'Summary': 'synopsis',
+                      'Specific Result': 'plugin_output',
+                      'NVT OID': 'nvt_oid',
+                      'Task ID': 'task_id',
+                      'Task Name': 'task_name',
+                      'Timestamp': 'timestamp',
+                      'Result ID': 'result_id',
+                      'Impact': 'description',
+                      'Solution': 'solution',
+                      'Affected Software/OS': 'affected_software',
+                      'Vulnerability Insight': 'vulnerability_insight',
+                      'Vulnerability Detection Method': 'vulnerability_detection_method',
+                      'Product Detection Result': 'product_detection_result',
+                      'BIDs': 'bids',
+                      'CERTs': 'certs',
+                      'Other References': 'see_also'
+                      }
+
+    def __init__(
+            self,
+            config=None,
+            db_name='report_tracker.db',
+            purge=False,
+            verbose=None,
+            debug=False,
+            username=None,
+            password=None,
+    ):
+        super(vulnWhispererOpenVAS, self).__init__(config=config)
+
+        self.port = int(self.config.get(self.CONFIG_SECTION, 'port'))
+        self.template_id = self.config.get(self.CONFIG_SECTION, 'report_format_id')
+        self.develop = True
+        self.purge = purge
+        self.scans_to_process = None
+        self.openvas_api = OpenVAS_API(hostname=self.hostname, port=self.port, username=self.username,
+                                       password=self.password)
+
+    def whisper_reports(self, output_format='json', launched_date=None, report_id=None, cleanup=True):
+        report = None
+        if report_id:
+            print('Processing report ID: %s' % report_id)
+
+            vuln_ready = self.openvas_api.process_report(report_id=report_id)
+            scan_name = report_id.replace('-', '')
+            vuln_ready['scan_name'] = scan_name
+            vuln_ready['scan_reference'] = report_id
+            vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
+            report_name = 'openvas_scan_{scan_name}_{last_updated}.{extension}'.format(scan_name=scan_name,
+                                                                                       last_updated=launched_date,
+                                                                                       extension=output_format)
+            relative_path_name = self.path_check(report_name)
+            scan_reference = report_id
+            print relative_path_name
+
+            if os.path.isfile(relative_path_name):
+                # TODO Possibly make this optional to sync directories
+                file_length = len(open(relative_path_name).readlines())
+                record_meta = (
+                    scan_name,
+                    scan_reference,
+                    launched_date,
+                    report_name,
+                    time.time(),
+                    file_length,
+                    self.CONFIG_SECTION,
+                    report_id,
+                    1,
+                )
+                self.record_insert(record_meta)
+                self.vprint('{info} File {filename} already exist! Updating database'.format(info=bcolors.INFO,
+                                                                                             filename=relative_path_name))
+
+                record_meta = (
+                    scan_name,
+                    scan_reference,
+                    launched_date,
+                    report_name,
+                    time.time(),
+                    vuln_ready.shape[0],
+                    self.CONFIG_SECTION,
+                    report_id,
+                    1,
+                )
+
+            vuln_ready.port = vuln_ready.port.fillna(0).astype(int)
+            if output_format == 'json':
+                with open(relative_path_name, 'w') as f:
+                    f.write(vuln_ready.to_json(orient='records', lines=True))
+            print('{success} - Report written to %s'.format(success=bcolors.SUCCESS) \
+                  % report_name)
+
+        return report
+
+    def identify_scans_to_process(self):
+        if self.uuids:
+            self.scans_to_process = self.openvas_api.openvas_reports[
+                ~self.openvas_api.openvas_reports.report_ids.isin(self.uuids)]
+        else:
+            self.scans_to_process = self.openvas_api.openvas_reports
+        self.vprint('{info} Identified {new} scans to be processed'.format(info=bcolors.INFO,
+                                                                           new=len(self.scans_to_process)))
+
+    def process_openvas_scans(self):
+        counter = 0
+        self.identify_scans_to_process()
+        if self.scans_to_process.shape[0]:
+            for scan in self.scans_to_process.iterrows():
+                counter += 1
+                info = scan[1]
+                print(
+                '[INFO] Processing %s/%s - Report ID: %s' % (counter, len(self.scans_to_process), info['report_ids']))
+                self.whisper_reports(report_id=info['report_ids'],
+                                     launched_date=info['epoch'])
+            self.vprint('{info} Processing complete!'.format(info=bcolors.INFO))
+        else:
+            self.vprint('{info} No new scans to process. Exiting...'.format(info=bcolors.INFO))
+        self.conn.close()
+        exit(0)
 
 
 
@@ -639,4 +771,8 @@ class vulnWhisperer(object):
 
         elif self.profile == 'qualys':
             vw = vulnWhispererQualys(config=self.config)
-            vw.process_web_assets()
+            vw.process_web_assets()
+
+        elif self.profile == 'openvas':
+            vw_openvas = vulnWhispererOpenVAS(config=self.config)
+            vw_openvas.process_openvas_scans()