diff --git a/.travis.yml b/.travis.yml index f11e590..c806fb0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,22 @@ before_script: - flake8 . --count --exit-zero --exclude=deps/qualysapi --max-complexity=10 --max-line-length=127 --statistics script: - python setup.py install + # Test successful scan download and parsing - vuln_whisperer -c configs/test.ini --mock --mock_dir test + - rm -rf /tmp/VulnWhisperer + # Test one failed scan + - rm -f test/nessus/GET_scans_exports_164_download + - vuln_whisperer -c configs/test.ini --mock --mock_dir test; [[ $? -eq 1 ]] + - rm -rf /tmp/VulnWhisperer + # Test two failed scans + - rm -f test/qualys_vuln/scan_1553941061.87241 + - vuln_whisperer -c configs/test.ini --mock --mock_dir test; [[ $? -eq 2 ]] + - rm -rf /tmp/VulnWhisperer + # Test only nessus + - vuln_whisperer -c configs/test.ini -s nessus --mock --mock_dir test; [[ $? -eq 1 ]] + - rm -rf /tmp/VulnWhisperer + # Test only qualy_vuln + - vuln_whisperer -c configs/test.ini -s qualys_vuln --mock --mock_dir test; [[ $? -eq 1 ]] notifications: on_success: change on_failure: change # `always` will be the setting once code changes slow down diff --git a/bin/vuln_whisperer b/bin/vuln_whisperer index 23d2000..010e9db 100644 --- a/bin/vuln_whisperer +++ b/bin/vuln_whisperer @@ -11,12 +11,14 @@ import argparse import sys import logging + def isFileValid(parser, arg): if not os.path.exists(arg): parser.error("The file %s does not exist!" % arg) else: return arg + def main(): parser = argparse.ArgumentParser(description=""" VulnWhisperer is designed to create actionable data from\ @@ -31,8 +33,10 @@ def main(): help='JIRA required only! Scan name from scan to report') parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=True, help='Prints status out to screen (defaults to True)') - parser.add_argument('-u', '--username', dest='username', required=False, default=None, type=lambda x: x.strip(), help='The NESSUS username') - parser.add_argument('-p', '--password', dest='password', required=False, default=None, type=lambda x: x.strip(), help='The NESSUS password') + parser.add_argument('-u', '--username', dest='username', required=False, default=None, + help='The NESSUS username', type=lambda x: x.strip()) + parser.add_argument('-p', '--password', dest='password', required=False, default=None, + help='The NESSUS password', type=lambda x: x.strip()) parser.add_argument('-F', '--fancy', action='store_true', help='Enable colourful logging output') parser.add_argument('-d', '--debug', action='store_true', help='Enable debugging messages') parser.add_argument('--mock', action='store_true', help='Enable mocked API responses') @@ -62,30 +66,28 @@ def main(): mock_api = mockAPI(args.mock_dir, args.verbose) mock_api.mock_endpoints() + exit_code = 0 + try: if args.config and not args.section: # this remains a print since we are in the main binary print('WARNING: {warning}'.format(warning='No section was specified, vulnwhisperer will scrape enabled modules from config file. \ - \nPlease specify a section using -s. \ + \nPlease specify a section using -s. \ \nExample vuln_whisperer -c config.ini -s nessus')) logger.info('No section was specified, vulnwhisperer will scrape enabled modules from the config file.') - config = vwConfig(config_in=args.config) - enabled_sections = config.get_sections_with_attribute('enabled') - - for section in enabled_sections: - vw = vulnWhisperer(config=args.config, - profile=section, - verbose=args.verbose, - username=args.username, - password=args.password, - source=args.source, - scanname=args.scanname) - - exit_code = vw.whisper_vulnerabilities() - # TODO: fix this to NOT be exit 1 unless in error - close_logging_handlers(logger) - sys.exit(exit_code) + + config = vwConfig(config_in=args.config) + enabled_sections = config.get_sections_with_attribute('enabled') + for section in enabled_sections: + vw = vulnWhisperer(config=args.config, + profile=section, + verbose=args.verbose, + username=args.username, + password=args.password, + source=args.source, + scanname=args.scanname) + exit_code += vw.whisper_vulnerabilities() else: logger.info('Running vulnwhisperer for section {}'.format(args.section)) vw = vulnWhisperer(config=args.config, @@ -95,11 +97,10 @@ def main(): password=args.password, source=args.source, scanname=args.scanname) + exit_code += vw.whisper_vulnerabilities() - exit_code = vw.whisper_vulnerabilities() - # TODO: fix this to NOT be exit 1 unless in error - close_logging_handlers(logger) - sys.exit(exit_code) + close_logging_handlers(logger) + sys.exit(exit_code) except Exception as e: if args.verbose: diff --git a/vulnwhisp/base/config.py b/vulnwhisp/base/config.py index 7786883..e8490d6 100644 --- a/vulnwhisp/base/config.py +++ b/vulnwhisp/base/config.py @@ -1,9 +1,8 @@ -import os import sys import logging # Support for python3 -if (sys.version_info > (3, 0)): +if sys.version_info > (3, 0): import configparser as cp else: import ConfigParser as cp @@ -45,7 +44,6 @@ class vwConfig(object): return False return True - def update_jira_profiles(self, profiles): # create JIRA profiles in the ini config file self.logger.debug('Updating Jira profiles: {}'.format(str(profiles))) @@ -59,27 +57,27 @@ class vwConfig(object): except: self.logger.warn("Creating config section for '{}'".format(section_name)) self.config.add_section(section_name) - self.config.set(section_name,'source',profile.split('.')[0]) + self.config.set(section_name, 'source', profile.split('.')[0]) # in case any scan name contains '.' character - self.config.set(section_name,'scan_name','.'.join(profile.split('.')[1:])) - self.config.set(section_name,'jira_project', '') - self.config.set(section_name,'; if multiple components, separate by ","') - self.config.set(section_name,'components', '') - self.config.set(section_name,'; minimum criticality to report (low, medium, high or critical)') - self.config.set(section_name,'min_critical_to_report', 'high') - self.config.set(section_name,'; automatically report, boolean value ') - self.config.set(section_name,'autoreport', 'false') + self.config.set(section_name, 'scan_name', '.'.join(profile.split('.')[1:])) + self.config.set(section_name, 'jira_project', '') + self.config.set(section_name, '; if multiple components, separate by ","') + self.config.set(section_name, 'components', '') + self.config.set(section_name, '; minimum criticality to report (low, medium, high or critical)') + self.config.set(section_name, 'min_critical_to_report', 'high') + self.config.set(section_name, '; automatically report, boolean value ') + self.config.set(section_name, 'autoreport', 'false') # TODO: try/catch this # writing changes back to file with open(self.config_in, 'w') as configfile: self.config.write(configfile) self.logger.debug('Written configuration to {}'.format(self.config_in)) - + # FIXME: this is the same as return None, that is the default return for return-less functions return def normalize_section(self, profile): - profile = "jira.{}".format(profile.lower().replace(" ","_")) + profile = "jira.{}".format(profile.lower().replace(" ", "_")) self.logger.debug('Normalized profile as: {}'.format(profile)) return profile diff --git a/vulnwhisp/frameworks/nessus.py b/vulnwhisp/frameworks/nessus.py index 5423caf..7944963 100755 --- a/vulnwhisp/frameworks/nessus.py +++ b/vulnwhisp/frameworks/nessus.py @@ -1,18 +1,13 @@ +from datetime import datetime +import sys +import time +import json +import logging +import pytz import requests from requests.packages.urllib3.exceptions import InsecureRequestWarning requests.packages.urllib3.disable_warnings(InsecureRequestWarning) -import pytz -from datetime import datetime -import json -import sys -import time -import logging - -from requests.packages.urllib3.exceptions import InsecureRequestWarning -requests.packages.urllib3.disable_warnings(InsecureRequestWarning) - - class NessusAPI(object): SESSION = '/session' @@ -58,7 +53,7 @@ class NessusAPI(object): def login(self): resp = self.get_token() - if resp.status_code is 200: + if resp.status_code == 200: self.headers['X-Cookie'] = 'token={token}'.format(token=resp.json()['token']) else: raise Exception('[FAIL] Could not login to Nessus') @@ -101,14 +96,6 @@ class NessusAPI(object): token = self.request(self.SESSION, data=auth, json=False) return token - def logout(self): - self.logger.debug('Logging out') - self.request(self.SESSION, method='DELETE') - - def get_folders(self): - folders = self.request(self.FOLDERS, method='GET', json=True) - return folders - def get_scans(self): scans = self.request(self.SCANS, method='GET', json=True) return scans @@ -119,47 +106,10 @@ class NessusAPI(object): self.logger.debug('Found {} scan_ids'.format(len(scan_ids))) return scan_ids - def count_scan(self, scans, folder_id): - count = 0 - for scan in scans: - if scan['folder_id'] == folder_id: count = count + 1 - return count - - def print_scans(self, data): - for folder in data['folders']: - self.logger.info("\\{0} - ({1})\\".format(folder['name'], self.count_scan(data['scans'], folder['id']))) - for scan in data['scans']: - if scan['folder_id'] == folder['id']: - self.logger.info("\t\"{0}\" - sid:{1} - uuid: {2}".format(scan['name'].encode('utf-8'), scan['id'], scan['uuid'])) - - def get_scan_details(self, scan_id): - data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json=True) - return data - def get_scan_history(self, scan_id): data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json=True) return data['history'] - def get_scan_hosts(self, scan_id): - data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json=True) - return data['hosts'] - - def get_host_vulnerabilities(self, scan_id, host_id): - query = self.HOST_VULN.format(scan_id=scan_id, host_id=host_id) - data = self.request(query, method='GET', json=True) - return data - - def get_plugin_info(self, scan_id, host_id, plugin_id): - query = self.PLUGINS.format(scan_id=scan_id, host_id=host_id, plugin_id=plugin_id) - data = self.request(query, method='GET', json=True) - return data - - def export_scan(self, scan_id, history_id): - data = {'format': 'csv'} - query = self.EXPORT_REPORT.format(scan_id=scan_id, history_id=history_id) - req = self.request(query, data=data, method='POST') - return req - def download_scan(self, scan_id=None, history=None, export_format="", chapters="", dbpasswd="", profile=""): running = True counter = 0 @@ -195,17 +145,6 @@ class NessusAPI(object): content = self.request(self.EXPORT_TOKEN_DOWNLOAD.format(token_id=token_id), method='GET', download=True) return content - @staticmethod - def merge_dicts(self, *dict_args): - """ - Given any number of dicts, shallow copy and merge into a new dict, - precedence goes to key value pairs in latter dicts. - """ - result = {} - for dictionary in dict_args: - result.update(dictionary) - return result - def get_utc_from_local(self, date_time, local_tz=None, epoch=True): date_time = datetime.fromtimestamp(date_time) if local_tz is None: diff --git a/vulnwhisp/frameworks/qualys_vuln.py b/vulnwhisp/frameworks/qualys_vuln.py index 1c8d2b3..34367ff 100644 --- a/vulnwhisp/frameworks/qualys_vuln.py +++ b/vulnwhisp/frameworks/qualys_vuln.py @@ -3,12 +3,9 @@ __author__ = 'Nathan Young' import xml.etree.ElementTree as ET -import pandas as pd -import qualysapi -import requests -import sys import logging -import os +import qualysapi +import pandas as pd import dateutil.parser as dp @@ -60,12 +57,13 @@ class qualysWhisperAPI(object): 'scan_ref': scan_id } scan_json = self.qgc.request(self.SCANS, parameters) - + # First two columns are metadata we already have # Last column corresponds to "target_distribution_across_scanner_appliances" element # which doesn't follow the schema and breaks the pandas data manipulation return pd.read_json(scan_json).iloc[2:-1] + class qualysUtils: def __init__(self): self.logger = logging.getLogger('qualysUtils') @@ -77,15 +75,15 @@ class qualysUtils: class qualysVulnScan: - + def __init__( - self, - config=None, - file_in=None, - file_stream=False, - delimiter=',', - quotechar='"', - ): + self, + config=None, + file_in=None, + file_stream=False, + delimiter=',', + quotechar='"', + ): self.logger = logging.getLogger('qualysVulnScan') self.file_in = file_in self.file_stream = file_stream diff --git a/vulnwhisp/test/mock.py b/vulnwhisp/test/mock.py index 6d05e65..fcc7e1c 100644 --- a/vulnwhisp/test/mock.py +++ b/vulnwhisp/test/mock.py @@ -2,19 +2,20 @@ import os import logging import httpretty + class mockAPI(object): def __init__(self, mock_dir=None, debug=False): self.mock_dir = mock_dir if not self.mock_dir: # Try to guess the mock_dir if python setup.py develop was used self.mock_dir = '/'.join(__file__.split('/')[:-3]) + '/test' - + self.logger = logging.getLogger('mockAPI') if debug: self.logger.setLevel(logging.DEBUG) - self.logger.info('mockAPI initialised, API requests will be mocked'.format(self.mock_dir)) + self.logger.info('mockAPI initialised, API requests will be mocked') self.logger.debug('Test path resolved as {}'.format(self.mock_dir)) def get_directories(self, path): @@ -28,23 +29,23 @@ class mockAPI(object): def qualys_vuln_callback(self, request, uri, response_headers): self.logger.debug('Simulating response for {} ({})'.format(uri, request.body)) if 'list' in request.parsed_body['action']: - return [ 200, + return [200, response_headers, open('{}/{}'.format(self.qualys_vuln_path, 'scans')).read()] elif 'fetch' in request.parsed_body['action']: try: response_body = open('{}/{}'.format( - self.qualys_vuln_path, - request.parsed_body['scan_ref'][0].replace('/', '_')) + self.qualys_vuln_path, + request.parsed_body['scan_ref'][0].replace('/', '_')) ).read() except: # Can't find the file, just send an empty response response_body = '' - return [200, response_headers, response_body] + return [200, response_headers, response_body] def create_nessus_resource(self, framework): for filename in self.get_files('{}/{}'.format(self.mock_dir, framework)): - method, resource = filename.split('_',1) + method, resource = filename.split('_', 1) resource = resource.replace('_', '/') self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, method, resource)) httpretty.register_uri( diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index ded70b3..21351da 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -43,11 +43,11 @@ class vulnWhispererBase(object): if self.CONFIG_SECTION is None: raise Exception('Implementing class must define CONFIG_SECTION') + self.exit_code = 0 self.db_name = db_name self.purge = purge self.develop = develop - if config is not None: self.config = vwConfig(config_in=config) try: @@ -361,7 +361,7 @@ class vulnWhispererNessus(vulnWhispererBase): if not scan_list: self.logger.warn('No new scans to process. Exiting...') - return 0 + return self.exit_code # Create scan subfolders @@ -432,9 +432,15 @@ class vulnWhispererNessus(vulnWhispererBase): self.record_insert(record_meta) self.logger.info('File {filename} already exist! Updating database'.format(filename=relative_path_name)) else: - file_req = \ - self.nessus.download_scan(scan_id=scan_id, history=history_id, - export_format='csv', profile=self.CONFIG_SECTION) + try: + file_req = \ + self.nessus.download_scan(scan_id=scan_id, history=history_id, + export_format='csv', profile=self.CONFIG_SECTION) + except Exception as e: + self.logger.error('Could not download {} scan {}: {}'.format(self.CONFIG_SECTION, scan_id, str(e))) + self.exit_code += 1 + continue + clean_csv = \ pd.read_csv(io.StringIO(file_req.decode('utf-8'))) if len(clean_csv) > 2: @@ -479,8 +485,8 @@ class vulnWhispererNessus(vulnWhispererBase): self.logger.info('Scan aggregation complete! Connection to database closed.') else: self.logger.error('Failed to use scanner at {host}:{port}'.format(host=self.hostname, port=self.nessus_port)) - return 1 - return 0 + self.exit_code += 1 + return self.exit_code class vulnWhispererQualys(vulnWhispererBase): @@ -550,7 +556,6 @@ class vulnWhispererQualys(vulnWhispererBase): if debug: self.logger.setLevel(logging.DEBUG) - self.qualys_scan = qualysScanReport(config=config) self.latest_scans = self.qualys_scan.qw.get_all_scans() self.directory_check() @@ -672,7 +677,7 @@ class vulnWhispererQualys(vulnWhispererBase): else: self.logger.info('No new scans to process. Exiting...') self.conn.close() - return 0 + return self.exit_code class vulnWhispererOpenVAS(vulnWhispererBase): @@ -718,7 +723,6 @@ class vulnWhispererOpenVAS(vulnWhispererBase): if debug: self.logger.setLevel(logging.DEBUG) - self.directory_check() self.port = int(self.config.get(self.CONFIG_SECTION, 'port')) self.develop = True @@ -809,7 +813,7 @@ class vulnWhispererOpenVAS(vulnWhispererBase): else: self.logger.info('No new scans to process. Exiting...') self.conn.close() - return 0 + return self.exit_code class vulnWhispererQualysVuln(vulnWhispererBase): @@ -850,7 +854,6 @@ class vulnWhispererQualysVuln(vulnWhispererBase): scan_reference=None, output_format='json', cleanup=True): - try: launched_date if 'Z' in launched_date: launched_date = self.qualys_scan.utils.iso_to_epoch(launched_date) @@ -879,11 +882,16 @@ class vulnWhispererQualysVuln(vulnWhispererBase): self.logger.info('File {filename} already exist! Updating database'.format(filename=relative_path_name)) else: - self.logger.info('Processing report ID: {}'.format(report_id)) - vuln_ready = self.qualys_scan.process_data(scan_id=report_id) - vuln_ready['scan_name'] = scan_name - vuln_ready['scan_reference'] = report_id - vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True) + try: + self.logger.info('Processing report ID: {}'.format(report_id)) + vuln_ready = self.qualys_scan.process_data(scan_id=report_id) + vuln_ready['scan_name'] = scan_name + vuln_ready['scan_reference'] = report_id + vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True) + except Exception as e: + self.logger.error('Could not process {}: {}'.format(report_id, str(e))) + self.exit_code += 1 + return self.exit_code record_meta = ( scan_name, @@ -905,9 +913,7 @@ class vulnWhispererQualysVuln(vulnWhispererBase): f.write('\n') self.logger.info('Report written to {}'.format(report_name)) - - except Exception as e: - self.logger.error('Could not process {}: {}'.format(report_id, str(e))) + return self.exit_code def identify_scans_to_process(self): @@ -929,14 +935,14 @@ class vulnWhispererQualysVuln(vulnWhispererBase): counter += 1 r = app[1] self.logger.info('Processing {}/{}'.format(counter, len(self.scans_to_process))) - self.whisper_reports(report_id=r['id'], + self.exit_code += self.whisper_reports(report_id=r['id'], launched_date=r['date'], scan_name=r['name'], scan_reference=r['type']) else: self.logger.info('No new scans to process. Exiting...') self.conn.close() - return 0 + return self.exit_code class vulnWhispererJIRA(vulnWhispererBase):