add filter for scan name and days to look back

bin/vuln_whisperer

@@ -28,6 +28,10 @@ def main():
                         help='Path of config file', type=lambda x: isFileValid(parser, x.strip()))
     parser.add_argument('-s', '--section', dest='section', required=False,
                         help='Section in config')
+    parser.add_argument('-f', '--filter', dest='scan_filter', required=False,
+                        help='Regex filter to limit to matching scan names')
+    parser.add_argument('--days', dest='days', type=int, required=False,
+                        help='Only import scans in the last X days')
     parser.add_argument('--source', dest='source', required=False,
                         help='JIRA required only! Source scanner to report')
     parser.add_argument('-n', '--scanname', dest='scanname', required=False,
@@ -87,6 +91,8 @@ def main():
                                    verbose=args.verbose,
                                    debug=args.debug,
                                    source=args.source,
+                                   scan_filter=args.scan_filter,
+                                   days=args.days,
                                    scanname=args.scanname)
                 exit_code += vw.whisper_vulnerabilities()
         else:
@@ -96,6 +102,8 @@ def main():
                                verbose=args.verbose,
                                debug=args.debug,
                                source=args.source,
+                               scan_filter=args.scan_filter,
+                               days=args.days,
                                scanname=args.scanname)
             exit_code += vw.whisper_vulnerabilities()
 

configs/frameworks_example.ini

@@ -10,6 +10,7 @@ write_path=/opt/VulnWhisperer/data/nessus/
 db_path=/opt/VulnWhisperer/data/database
 trash=false
 verbose=false
+scan_filter=
 
 [tenable]
 enabled=true
@@ -23,6 +24,7 @@ write_path=/opt/VulnWhisperer/data/tenable/
 db_path=/opt/VulnWhisperer/data/database
 trash=false
 verbose=false
+scan_filter=
 
 [qualys_web]
 #Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
@@ -33,6 +35,7 @@ password = examplepass
 write_path=/opt/VulnWhisperer/data/qualys_web/
 db_path=/opt/VulnWhisperer/data/database
 verbose=true
+scan_filter=
 
 # Set the maximum number of retries each connection should attempt.
 #Note, this applies only to failed connections and timeouts, never to requests where the server returns a response.
@@ -49,6 +52,7 @@ password = examplepass
 write_path=/opt/VulnWhisperer/data/qualys_vuln/
 db_path=/opt/VulnWhisperer/data/database
 verbose=false
+scan_filter=
 
 [detectify]
 #Reference https://developer.detectify.com/
@@ -61,6 +65,7 @@ password = examplepass
 write_path =/opt/VulnWhisperer/data/detectify/
 db_path = /opt/VulnWhisperer/data/database
 verbose = true
+scan_filter=
 
 [openvas]
 enabled = false
@@ -71,6 +76,7 @@ password = examplepass
 write_path=/opt/VulnWhisperer/data/openvas/
 db_path=/opt/VulnWhisperer/data/database
 verbose=false
+scan_filter=
 
 [jira]
 enabled = false

vulnwhisp/frameworks/nessus.py

@@ -2,7 +2,7 @@ import json
 import logging
 import sys
 import time
-from datetime import datetime
+from datetime import datetime, timedelta
 
 import pytz
 import requests
@@ -81,9 +81,6 @@ class NessusAPI(object):
         else:
             self.login()
 
-        self.scans = self.get_scans()
-        self.scan_ids = self.get_scan_ids()
-
     def login(self):
         auth = '{"username":"%s", "password":"%s"}' % (self.user, self.password)
         resp = self.request(self.SESSION, data=auth, json_output=False)
@@ -92,7 +89,7 @@ class NessusAPI(object):
         else:
             raise Exception('[FAIL] Could not login to Nessus')
 
-    def request(self, url, data=None, headers=None, method='POST', download=False, json_output=False):
+    def request(self, url, data=None, headers=None, method='POST', download=False, json_output=False, params=None):
         timeout = 0
         success = False
 
@@ -101,7 +98,7 @@ class NessusAPI(object):
         self.logger.debug('Requesting to url {}'.format(url))
 
         while (timeout <= 10) and (not success):
-            response = getattr(self.session, method)(url, data=data)
+            response = getattr(self.session, method)(url, data=data, params=params)
             if response.status_code == 401:
                 if url == self.base + self.SESSION:
                     break
@@ -130,12 +127,15 @@ class NessusAPI(object):
             return response_data
         return response
 
-    def get_scans(self):
+    def get_scans(self, days=None):
-        scans = self.request(self.SCANS, method='GET', json_output=True)
+        if days:
+            parameters = {
+                "last_modification_date": (datetime.now() - timedelta(days=days)).strftime("%s")
+            }
+        scans = self.request(self.SCANS, method="GET", params=parameters, json_output=True)
         return scans
 
-    def get_scan_ids(self):
+    def get_scan_ids(self, scans):
-        scans = self.scans
         scan_ids = [scan_id['id'] for scan_id in scans['scans']] if scans['scans'] else []
         self.logger.debug('Found {} scan_ids'.format(len(scan_ids)))
         return scan_ids

vulnwhisp/frameworks/qualys_vm.py

@@ -5,6 +5,7 @@ __author__ = 'Nathan Young'
 import logging
 import sys
 import xml.etree.ElementTree as ET
+from datetime import datetime, timedelta
 
 import dateutil.parser as dp
 import pandas as pd
@@ -29,7 +30,7 @@ class qualysWhisperAPI(object):
     def scan_xml_parser(self, xml):
         all_records = []
         root = ET.XML(xml.encode('utf-8'))
-        if not root.find('.//SCAN_LIST'):
+        if len(root.find('.//SCAN_LIST')) == 0:
             return pd.DataFrame(columns=['id', 'status'])
         for child in root.find('.//SCAN_LIST'):
             all_records.append({
@@ -42,12 +43,17 @@ class qualysWhisperAPI(object):
             })
         return pd.DataFrame(all_records)
 
-    def get_all_scans(self):
+    def get_all_scans(self, days=None):
+        if not days:
+            self.launched_date = '0001-01-01'
+        else:
+            self.launched_date = (datetime.now() - timedelta(days=days)).strftime('%Y-%m-%d')
         parameters = {
             'action': 'list',
             'echo_request': 0,
             'show_op': 0,
-            'launched_after_datetime': '0001-01-01'
+            'state': 'Finished',
+            'launched_after_datetime': self.launched_date
         }
         scans_xml = self.qgc.request(self.SCANS, parameters)
         return self.scan_xml_parser(scans_xml)

vulnwhisp/frameworks/qualys_was.py

@@ -7,6 +7,7 @@ import logging
 import os
 import sys
 import xml.etree.ElementTree as ET
+from datetime import datetime, timedelta
 
 import dateutil.parser as dp
 import pandas as pd
@@ -60,10 +61,12 @@ class qualysWhisperAPI(object):
         """
         Checks number of scans, used to control the api limits
         """
-        parameters = (
+        parameters = E.ServiceRequest(
-            E.ServiceRequest(
                 E.filters(
-                    E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status))))
+                    E.Criteria({"field": "status", "operator": "EQUALS"}, status),
+                    E.Criteria({"field": "launchedDate", "operator": "GREATER"}, self.launched_date)
+                )
+            )
         xml_output = self.qgc.request(self.COUNT_WASSCAN, parameters)
         root = objectify.fromstring(xml_output.encode('utf-8'))
         return root.count.text
@@ -71,8 +74,8 @@ class qualysWhisperAPI(object):
     def generate_scan_result_XML(self, limit=1000, offset=1, status='FINISHED'):
         report_xml = E.ServiceRequest(
             E.filters(
-                E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status
+                E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status),
-                           ),
+                E.Criteria({"field": "launchedDate", "operator": "GREATER"}, self.launched_date)
             ),
             E.preferences(
                 E.startFromOffset(str(offset)),
@@ -104,7 +107,12 @@ class qualysWhisperAPI(object):
                 all_records.append(record)
         return pd.DataFrame(all_records)
 
-    def get_all_scans(self, limit=1000, offset=1, status='FINISHED'):
+
+    def get_all_scans(self, limit=1000, offset=1, status='FINISHED', days=None):
+        if not days:
+            self.launched_date = '0001-01-01'
+        else:
+            self.launched_date = (datetime.now() - timedelta(days=days)).strftime('%Y-%m-%d')
         qualys_api_limit = limit
         dataframes = []
         _records = []

vulnwhisp/vulnwhisp.py

@@ -10,6 +10,7 @@ import socket
 import sqlite3
 import sys
 import time
+import re
 
 import numpy as np
 import pandas as pd
@@ -37,6 +38,8 @@ class vulnWhispererBase(object):
             verbose=False,
             debug=False,
             section=None,
+            scan_filter=None,
+            days=None,
             develop=False,
         ):
 
@@ -47,6 +50,7 @@ class vulnWhispererBase(object):
         self.db_name = db_name
         self.purge = purge
         self.develop = develop
+        self.days = days
 
         if config is not None:
             self.config = vwConfig(config_in=config)
@@ -61,12 +65,29 @@ class vulnWhispererBase(object):
             except:
                 self.username = None
                 self.password = None
+            try:
+                self.scan_filter = self.config.get(self.CONFIG_SECTION, 'scan_filter')
+            except:
+                self.scan_filter = scan_filter
             self.write_path = self.config.get(self.CONFIG_SECTION, 'write_path')
             self.db_path = self.config.get(self.CONFIG_SECTION, 'db_path')
 
         self.logger = logging.getLogger('vulnWhispererBase')
         self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)
 
+        # Preference command line argument over config file
+        if scan_filter:
+            self.scan_filter = scan_filter
+
+        if self.scan_filter:
+            self.logger.info('Filtering for scan names matching "{}"'.format(self.scan_filter))
+            # self.scan_filter = re.compile(scan_filter)
+
+        if self.days:
+            self.logger.info('Searching for scans within {} days'.format(self.days))
+            # self.days = dp.parse(days)
+            # self.logger.info('Searching for scans after {}'.format(self.days))
+
         if self.db_name is not None:
             if self.db_path:
                 self.database = os.path.join(self.db_path,
@@ -321,11 +342,13 @@ class vulnWhispererNessus(vulnWhispererBase):
             purge=False,
             verbose=False,
             debug=False,
-            profile='nessus'
+            profile='nessus',
+            scan_filter=None,
+            days=None,
     ):
         self.CONFIG_SECTION=profile
 
-        super(vulnWhispererNessus, self).__init__(config=config, verbose=verbose, debug=debug)
+        super(vulnWhispererNessus, self).__init__(config=config, verbose=verbose, debug=debug, scan_filter=scan_filter, days=days)
 
         self.logger = logging.getLogger('vulnWhisperer{}'.format(self.CONFIG_SECTION))
         if not verbose:
@@ -422,7 +445,7 @@ class vulnWhispererNessus(vulnWhispererBase):
             self.exit_code += 1
             return self.exit_code
 
-        scan_data = self.nessus.scans
+        scan_data = self.nessus.get_scans(self.days)
         folders = scan_data['folders']
         scans = scan_data['scans'] if scan_data['scans'] else []
         all_scans = self.scan_count(scans)
@@ -434,6 +457,12 @@ class vulnWhispererNessus(vulnWhispererBase):
             ]
         else:
             scan_list = all_scans
+        if self.scan_filter:
+            self.logger.info('Filtering scans that match "{}"'.format(self.scan_filter))
+            scan_list = [
+                x for x in scan_list
+                if re.match(self.scan_filter, x["scan_name"], re.IGNORECASE)
+            ]
         self.logger.info(
             "Identified {new} scans to be processed".format(new=len(scan_list))
         )
@@ -569,16 +598,18 @@ class vulnWhispererQualysWAS(vulnWhispererBase):
             purge=False,
             verbose=False,
             debug=False,
+            scan_filter=None,
+            days=None,
         ):
 
-        super(vulnWhispererQualysWAS, self).__init__(config=config, verbose=verbose, debug=debug)
+        super(vulnWhispererQualysWAS, self).__init__(config=config, verbose=verbose, debug=debug, scan_filter=scan_filter, days=days)
         self.logger = logging.getLogger('vulnWhispererQualysWAS')
         if not verbose:
             verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
         self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)
 
         self.qualys_scan = qualysScanReport(config=config)
-        self.latest_scans = self.qualys_scan.qw.get_all_scans()
+        self.latest_scans = self.qualys_scan.qw.get_all_scans(days=self.days)
         self.directory_check()
         self.scans_to_process = None
 
@@ -683,6 +714,11 @@ class vulnWhispererQualysWAS(vulnWhispererBase):
 
 
     def identify_scans_to_process(self):
+        if self.scan_filter:
+            self.logger.info('Filtering scans that match "{}"'.format(self.scan_filter))
+            self.latest_scans = self.latest_scans.loc[
+                self.latest_scans["name"].str.contains(self.scan_filter, case=False)
+            ]
         if self.uuids:
             self.scans_to_process = self.latest_scans[~self.latest_scans['id'].isin(self.uuids)]
         else:
@@ -718,8 +754,10 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
             purge=False,
             verbose=False,
             debug=False,
+            scan_filter=None,
+            days=None,
     ):
-        super(vulnWhispererOpenVAS, self).__init__(config=config, verbose=verbose, debug=debug)
+        super(vulnWhispererOpenVAS, self).__init__(config=config, verbose=verbose, debug=debug, scan_filter=scan_filter, days=days)
         self.logger = logging.getLogger('vulnWhispererOpenVAS')
         if not verbose:
             verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
@@ -838,9 +876,11 @@ class vulnWhispererQualysVM(vulnWhispererBase):
             purge=False,
             verbose=False,
             debug=False,
+            scan_filter=None,
+            days=None,
         ):
 
-        super(vulnWhispererQualysVM, self).__init__(config=config, verbose=verbose, debug=debug)
+        super(vulnWhispererQualysVM, self).__init__(config=config, verbose=verbose, debug=debug, scan_filter=scan_filter, days=days)
         self.logger = logging.getLogger('vulnWhispererQualysVM')
         if not verbose:
             verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
@@ -929,9 +969,13 @@ class vulnWhispererQualysVM(vulnWhispererBase):
 
             return self.exit_code
 
-
     def identify_scans_to_process(self):
-        self.latest_scans = self.qualys_scan.qw.get_all_scans()
+        self.latest_scans = self.qualys_scan.qw.get_all_scans(days=self.days)
+        if self.scan_filter:
+            self.logger.info('Filtering scans that match "{}"'.format(self.scan_filter))
+            self.latest_scans = self.latest_scans.loc[
+                self.latest_scans["name"].str.contains(self.scan_filter, case=False)
+            ]
         if self.uuids:
             self.scans_to_process = self.latest_scans.loc[
                 (~self.latest_scans['id'].isin(self.uuids))
@@ -1251,6 +1295,8 @@ class vulnWhisperer(object):
                  debug=False,
                  config=None,
                  source=None,
+                 scan_filter=None,
+                 days=None,
                  scanname=None):
 
         self.logger = logging.getLogger('vulnWhisperer')
@@ -1260,6 +1306,8 @@ class vulnWhisperer(object):
         self.debug = debug
         self.config = config
         self.source = source
+        self.scan_filter = scan_filter
+        self.days = days
         self.scanname = scanname
         self.exit_code = 0
 
@@ -1269,18 +1317,24 @@ class vulnWhisperer(object):
         if self.profile == 'nessus':
             vw = vulnWhispererNessus(config=self.config,
                                      profile=self.profile,
+                                     scan_filter=self.scan_filter,
+                                     days=self.days,
                                      verbose=self.verbose,
                                      debug=self.debug)
             self.exit_code += vw.whisper_nessus()
 
         elif self.profile == 'qualys_was':
             vw = vulnWhispererQualysWAS(config=self.config,
-                                     verbose=self.verbose,
+                                        scan_filter=self.scan_filter,
-                                     debug=self.debug)
+                                        days=self.days,
+                                        verbose=self.verbose,
+                                        debug=self.debug)
             self.exit_code += vw.process_web_assets()
 
         elif self.profile == 'openvas':
             vw_openvas = vulnWhispererOpenVAS(config=self.config,
+                                              scan_filter=self.scan_filter,
+                                              days=self.days,
                                               verbose=self.verbose,
                                               debug=self.debug)
             self.exit_code += vw_openvas.process_openvas_scans()
@@ -1288,14 +1342,18 @@ class vulnWhisperer(object):
         elif self.profile == 'tenable':
             vw = vulnWhispererNessus(config=self.config,
                                      profile=self.profile,
+                                     scan_filter=self.scan_filter,
+                                     days=self.days,
                                      verbose=self.verbose,
                                      debug=self.debug)
             self.exit_code += vw.whisper_nessus()
 
         elif self.profile == 'qualys_vm':
             vw = vulnWhispererQualysVM(config=self.config,
-                                         verbose=self.verbose,
+                                       scan_filter=self.scan_filter,
-                                         debug=self.debug)
+                                       days=self.days,
+                                       verbose=self.verbose,
+                                       debug=self.debug)
             self.exit_code += vw.process_vuln_scans()
 
         elif self.profile == 'jira':