From aa9fa5b6528c9724a9425eb46902b7436484373e Mon Sep 17 00:00:00 2001 From: pemontto Date: Fri, 10 May 2019 12:19:53 +0100 Subject: [PATCH] add filter for scan name and days to look back --- bin/vuln_whisperer | 8 +++ configs/frameworks_example.ini | 6 +++ vulnwhisp/frameworks/nessus.py | 20 +++---- vulnwhisp/frameworks/qualys_vm.py | 12 +++-- vulnwhisp/frameworks/qualys_was.py | 20 ++++--- vulnwhisp/vulnwhisp.py | 84 +++++++++++++++++++++++++----- 6 files changed, 118 insertions(+), 32 deletions(-) diff --git a/bin/vuln_whisperer b/bin/vuln_whisperer index 1b8c3a9..e9a1c20 100644 --- a/bin/vuln_whisperer +++ b/bin/vuln_whisperer @@ -28,6 +28,10 @@ def main(): help='Path of config file', type=lambda x: isFileValid(parser, x.strip())) parser.add_argument('-s', '--section', dest='section', required=False, help='Section in config') + parser.add_argument('-f', '--filter', dest='scan_filter', required=False, + help='Regex filter to limit to matching scan names') + parser.add_argument('--days', dest='days', type=int, required=False, + help='Only import scans in the last X days') parser.add_argument('--source', dest='source', required=False, help='JIRA required only! Source scanner to report') parser.add_argument('-n', '--scanname', dest='scanname', required=False, @@ -87,6 +91,8 @@ def main(): verbose=args.verbose, debug=args.debug, source=args.source, + scan_filter=args.scan_filter, + days=args.days, scanname=args.scanname) exit_code += vw.whisper_vulnerabilities() else: @@ -96,6 +102,8 @@ def main(): verbose=args.verbose, debug=args.debug, source=args.source, + scan_filter=args.scan_filter, + days=args.days, scanname=args.scanname) exit_code += vw.whisper_vulnerabilities() diff --git a/configs/frameworks_example.ini b/configs/frameworks_example.ini index e1a2eb1..7aef515 100755 --- a/configs/frameworks_example.ini +++ b/configs/frameworks_example.ini @@ -10,6 +10,7 @@ write_path=/opt/VulnWhisperer/data/nessus/ db_path=/opt/VulnWhisperer/data/database trash=false verbose=false +scan_filter= [tenable] enabled=true @@ -23,6 +24,7 @@ write_path=/opt/VulnWhisperer/data/tenable/ db_path=/opt/VulnWhisperer/data/database trash=false verbose=false +scan_filter= [qualys_web] #Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API @@ -33,6 +35,7 @@ password = examplepass write_path=/opt/VulnWhisperer/data/qualys_web/ db_path=/opt/VulnWhisperer/data/database verbose=true +scan_filter= # Set the maximum number of retries each connection should attempt. #Note, this applies only to failed connections and timeouts, never to requests where the server returns a response. @@ -49,6 +52,7 @@ password = examplepass write_path=/opt/VulnWhisperer/data/qualys_vuln/ db_path=/opt/VulnWhisperer/data/database verbose=false +scan_filter= [detectify] #Reference https://developer.detectify.com/ @@ -61,6 +65,7 @@ password = examplepass write_path =/opt/VulnWhisperer/data/detectify/ db_path = /opt/VulnWhisperer/data/database verbose = true +scan_filter= [openvas] enabled = false @@ -71,6 +76,7 @@ password = examplepass write_path=/opt/VulnWhisperer/data/openvas/ db_path=/opt/VulnWhisperer/data/database verbose=false +scan_filter= [jira] enabled = false diff --git a/vulnwhisp/frameworks/nessus.py b/vulnwhisp/frameworks/nessus.py index eb628b8..25b6748 100755 --- a/vulnwhisp/frameworks/nessus.py +++ b/vulnwhisp/frameworks/nessus.py @@ -2,7 +2,7 @@ import json import logging import sys import time -from datetime import datetime +from datetime import datetime, timedelta import pytz import requests @@ -81,9 +81,6 @@ class NessusAPI(object): else: self.login() - self.scans = self.get_scans() - self.scan_ids = self.get_scan_ids() - def login(self): auth = '{"username":"%s", "password":"%s"}' % (self.user, self.password) resp = self.request(self.SESSION, data=auth, json_output=False) @@ -92,7 +89,7 @@ class NessusAPI(object): else: raise Exception('[FAIL] Could not login to Nessus') - def request(self, url, data=None, headers=None, method='POST', download=False, json_output=False): + def request(self, url, data=None, headers=None, method='POST', download=False, json_output=False, params=None): timeout = 0 success = False @@ -101,7 +98,7 @@ class NessusAPI(object): self.logger.debug('Requesting to url {}'.format(url)) while (timeout <= 10) and (not success): - response = getattr(self.session, method)(url, data=data) + response = getattr(self.session, method)(url, data=data, params=params) if response.status_code == 401: if url == self.base + self.SESSION: break @@ -130,12 +127,15 @@ class NessusAPI(object): return response_data return response - def get_scans(self): - scans = self.request(self.SCANS, method='GET', json_output=True) + def get_scans(self, days=None): + if days: + parameters = { + "last_modification_date": (datetime.now() - timedelta(days=days)).strftime("%s") + } + scans = self.request(self.SCANS, method="GET", params=parameters, json_output=True) return scans - def get_scan_ids(self): - scans = self.scans + def get_scan_ids(self, scans): scan_ids = [scan_id['id'] for scan_id in scans['scans']] if scans['scans'] else [] self.logger.debug('Found {} scan_ids'.format(len(scan_ids))) return scan_ids diff --git a/vulnwhisp/frameworks/qualys_vm.py b/vulnwhisp/frameworks/qualys_vm.py index e22cc39..66c302d 100644 --- a/vulnwhisp/frameworks/qualys_vm.py +++ b/vulnwhisp/frameworks/qualys_vm.py @@ -5,6 +5,7 @@ __author__ = 'Nathan Young' import logging import sys import xml.etree.ElementTree as ET +from datetime import datetime, timedelta import dateutil.parser as dp import pandas as pd @@ -29,7 +30,7 @@ class qualysWhisperAPI(object): def scan_xml_parser(self, xml): all_records = [] root = ET.XML(xml.encode('utf-8')) - if not root.find('.//SCAN_LIST'): + if len(root.find('.//SCAN_LIST')) == 0: return pd.DataFrame(columns=['id', 'status']) for child in root.find('.//SCAN_LIST'): all_records.append({ @@ -42,12 +43,17 @@ class qualysWhisperAPI(object): }) return pd.DataFrame(all_records) - def get_all_scans(self): + def get_all_scans(self, days=None): + if not days: + self.launched_date = '0001-01-01' + else: + self.launched_date = (datetime.now() - timedelta(days=days)).strftime('%Y-%m-%d') parameters = { 'action': 'list', 'echo_request': 0, 'show_op': 0, - 'launched_after_datetime': '0001-01-01' + 'state': 'Finished', + 'launched_after_datetime': self.launched_date } scans_xml = self.qgc.request(self.SCANS, parameters) return self.scan_xml_parser(scans_xml) diff --git a/vulnwhisp/frameworks/qualys_was.py b/vulnwhisp/frameworks/qualys_was.py index c0f000a..f980653 100644 --- a/vulnwhisp/frameworks/qualys_was.py +++ b/vulnwhisp/frameworks/qualys_was.py @@ -7,6 +7,7 @@ import logging import os import sys import xml.etree.ElementTree as ET +from datetime import datetime, timedelta import dateutil.parser as dp import pandas as pd @@ -60,10 +61,12 @@ class qualysWhisperAPI(object): """ Checks number of scans, used to control the api limits """ - parameters = ( - E.ServiceRequest( + parameters = E.ServiceRequest( E.filters( - E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status)))) + E.Criteria({"field": "status", "operator": "EQUALS"}, status), + E.Criteria({"field": "launchedDate", "operator": "GREATER"}, self.launched_date) + ) + ) xml_output = self.qgc.request(self.COUNT_WASSCAN, parameters) root = objectify.fromstring(xml_output.encode('utf-8')) return root.count.text @@ -71,8 +74,8 @@ class qualysWhisperAPI(object): def generate_scan_result_XML(self, limit=1000, offset=1, status='FINISHED'): report_xml = E.ServiceRequest( E.filters( - E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status - ), + E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status), + E.Criteria({"field": "launchedDate", "operator": "GREATER"}, self.launched_date) ), E.preferences( E.startFromOffset(str(offset)), @@ -104,7 +107,12 @@ class qualysWhisperAPI(object): all_records.append(record) return pd.DataFrame(all_records) - def get_all_scans(self, limit=1000, offset=1, status='FINISHED'): + + def get_all_scans(self, limit=1000, offset=1, status='FINISHED', days=None): + if not days: + self.launched_date = '0001-01-01' + else: + self.launched_date = (datetime.now() - timedelta(days=days)).strftime('%Y-%m-%d') qualys_api_limit = limit dataframes = [] _records = [] diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index dc9ae45..6f37b3a 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -10,6 +10,7 @@ import socket import sqlite3 import sys import time +import re import numpy as np import pandas as pd @@ -37,6 +38,8 @@ class vulnWhispererBase(object): verbose=False, debug=False, section=None, + scan_filter=None, + days=None, develop=False, ): @@ -47,6 +50,7 @@ class vulnWhispererBase(object): self.db_name = db_name self.purge = purge self.develop = develop + self.days = days if config is not None: self.config = vwConfig(config_in=config) @@ -61,12 +65,29 @@ class vulnWhispererBase(object): except: self.username = None self.password = None + try: + self.scan_filter = self.config.get(self.CONFIG_SECTION, 'scan_filter') + except: + self.scan_filter = scan_filter self.write_path = self.config.get(self.CONFIG_SECTION, 'write_path') self.db_path = self.config.get(self.CONFIG_SECTION, 'db_path') self.logger = logging.getLogger('vulnWhispererBase') self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING) + # Preference command line argument over config file + if scan_filter: + self.scan_filter = scan_filter + + if self.scan_filter: + self.logger.info('Filtering for scan names matching "{}"'.format(self.scan_filter)) + # self.scan_filter = re.compile(scan_filter) + + if self.days: + self.logger.info('Searching for scans within {} days'.format(self.days)) + # self.days = dp.parse(days) + # self.logger.info('Searching for scans after {}'.format(self.days)) + if self.db_name is not None: if self.db_path: self.database = os.path.join(self.db_path, @@ -321,11 +342,13 @@ class vulnWhispererNessus(vulnWhispererBase): purge=False, verbose=False, debug=False, - profile='nessus' + profile='nessus', + scan_filter=None, + days=None, ): self.CONFIG_SECTION=profile - super(vulnWhispererNessus, self).__init__(config=config, verbose=verbose, debug=debug) + super(vulnWhispererNessus, self).__init__(config=config, verbose=verbose, debug=debug, scan_filter=scan_filter, days=days) self.logger = logging.getLogger('vulnWhisperer{}'.format(self.CONFIG_SECTION)) if not verbose: @@ -422,7 +445,7 @@ class vulnWhispererNessus(vulnWhispererBase): self.exit_code += 1 return self.exit_code - scan_data = self.nessus.scans + scan_data = self.nessus.get_scans(self.days) folders = scan_data['folders'] scans = scan_data['scans'] if scan_data['scans'] else [] all_scans = self.scan_count(scans) @@ -434,6 +457,12 @@ class vulnWhispererNessus(vulnWhispererBase): ] else: scan_list = all_scans + if self.scan_filter: + self.logger.info('Filtering scans that match "{}"'.format(self.scan_filter)) + scan_list = [ + x for x in scan_list + if re.match(self.scan_filter, x["scan_name"], re.IGNORECASE) + ] self.logger.info( "Identified {new} scans to be processed".format(new=len(scan_list)) ) @@ -569,16 +598,18 @@ class vulnWhispererQualysWAS(vulnWhispererBase): purge=False, verbose=False, debug=False, + scan_filter=None, + days=None, ): - super(vulnWhispererQualysWAS, self).__init__(config=config, verbose=verbose, debug=debug) + super(vulnWhispererQualysWAS, self).__init__(config=config, verbose=verbose, debug=debug, scan_filter=scan_filter, days=days) self.logger = logging.getLogger('vulnWhispererQualysWAS') if not verbose: verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose') self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING) self.qualys_scan = qualysScanReport(config=config) - self.latest_scans = self.qualys_scan.qw.get_all_scans() + self.latest_scans = self.qualys_scan.qw.get_all_scans(days=self.days) self.directory_check() self.scans_to_process = None @@ -683,6 +714,11 @@ class vulnWhispererQualysWAS(vulnWhispererBase): def identify_scans_to_process(self): + if self.scan_filter: + self.logger.info('Filtering scans that match "{}"'.format(self.scan_filter)) + self.latest_scans = self.latest_scans.loc[ + self.latest_scans["name"].str.contains(self.scan_filter, case=False) + ] if self.uuids: self.scans_to_process = self.latest_scans[~self.latest_scans['id'].isin(self.uuids)] else: @@ -718,8 +754,10 @@ class vulnWhispererOpenVAS(vulnWhispererBase): purge=False, verbose=False, debug=False, + scan_filter=None, + days=None, ): - super(vulnWhispererOpenVAS, self).__init__(config=config, verbose=verbose, debug=debug) + super(vulnWhispererOpenVAS, self).__init__(config=config, verbose=verbose, debug=debug, scan_filter=scan_filter, days=days) self.logger = logging.getLogger('vulnWhispererOpenVAS') if not verbose: verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose') @@ -838,9 +876,11 @@ class vulnWhispererQualysVM(vulnWhispererBase): purge=False, verbose=False, debug=False, + scan_filter=None, + days=None, ): - super(vulnWhispererQualysVM, self).__init__(config=config, verbose=verbose, debug=debug) + super(vulnWhispererQualysVM, self).__init__(config=config, verbose=verbose, debug=debug, scan_filter=scan_filter, days=days) self.logger = logging.getLogger('vulnWhispererQualysVM') if not verbose: verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose') @@ -929,9 +969,13 @@ class vulnWhispererQualysVM(vulnWhispererBase): return self.exit_code - def identify_scans_to_process(self): - self.latest_scans = self.qualys_scan.qw.get_all_scans() + self.latest_scans = self.qualys_scan.qw.get_all_scans(days=self.days) + if self.scan_filter: + self.logger.info('Filtering scans that match "{}"'.format(self.scan_filter)) + self.latest_scans = self.latest_scans.loc[ + self.latest_scans["name"].str.contains(self.scan_filter, case=False) + ] if self.uuids: self.scans_to_process = self.latest_scans.loc[ (~self.latest_scans['id'].isin(self.uuids)) @@ -1251,6 +1295,8 @@ class vulnWhisperer(object): debug=False, config=None, source=None, + scan_filter=None, + days=None, scanname=None): self.logger = logging.getLogger('vulnWhisperer') @@ -1260,6 +1306,8 @@ class vulnWhisperer(object): self.debug = debug self.config = config self.source = source + self.scan_filter = scan_filter + self.days = days self.scanname = scanname self.exit_code = 0 @@ -1269,18 +1317,24 @@ class vulnWhisperer(object): if self.profile == 'nessus': vw = vulnWhispererNessus(config=self.config, profile=self.profile, + scan_filter=self.scan_filter, + days=self.days, verbose=self.verbose, debug=self.debug) self.exit_code += vw.whisper_nessus() elif self.profile == 'qualys_was': vw = vulnWhispererQualysWAS(config=self.config, - verbose=self.verbose, - debug=self.debug) + scan_filter=self.scan_filter, + days=self.days, + verbose=self.verbose, + debug=self.debug) self.exit_code += vw.process_web_assets() elif self.profile == 'openvas': vw_openvas = vulnWhispererOpenVAS(config=self.config, + scan_filter=self.scan_filter, + days=self.days, verbose=self.verbose, debug=self.debug) self.exit_code += vw_openvas.process_openvas_scans() @@ -1288,14 +1342,18 @@ class vulnWhisperer(object): elif self.profile == 'tenable': vw = vulnWhispererNessus(config=self.config, profile=self.profile, + scan_filter=self.scan_filter, + days=self.days, verbose=self.verbose, debug=self.debug) self.exit_code += vw.whisper_nessus() elif self.profile == 'qualys_vm': vw = vulnWhispererQualysVM(config=self.config, - verbose=self.verbose, - debug=self.debug) + scan_filter=self.scan_filter, + days=self.days, + verbose=self.verbose, + debug=self.debug) self.exit_code += vw.process_vuln_scans() elif self.profile == 'jira':