Cyberdefense/VulnWhisperer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Feature error codes (#165)

Browse Source 
* Use error codes for failed scans

* Fix indentations

* Fix more indentation

* Continue after failed download

* Add tests for failed scans

* Add more tests

* move definition

* Update nessus.py

This function was used by function `print_scans` which at the same time was an unused one that had been deleted in the PR itself.
This commit is contained in:
pemontto
2019-04-05 20:36:13 +11:00
committed by Quim Montal
parent 71352aee57
commit 1ef67d48be
7 changed files with 106 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

15
.travis.yml
View File

@ -19,7 +19,22 @@ before_script:
- flake8 . --count --exit-zero --exclude=deps/qualysapi --max-complexity=10 --max-line-length=127 --statistics
script:
- python setup.py install
# Test successful scan download and parsing
- vuln_whisperer -c configs/test.ini --mock --mock_dir test
- rm -rf /tmp/VulnWhisperer
# Test one failed scan
- rm -f test/nessus/GET_scans_exports_164_download
- vuln_whisperer -c configs/test.ini --mock --mock_dir test; [[ $? -eq 1 ]]
- rm -rf /tmp/VulnWhisperer
# Test two failed scans
- rm -f test/qualys_vuln/scan_1553941061.87241
- vuln_whisperer -c configs/test.ini --mock --mock_dir test; [[ $? -eq 2 ]]
- rm -rf /tmp/VulnWhisperer
# Test only nessus
- vuln_whisperer -c configs/test.ini -s nessus --mock --mock_dir test; [[ $? -eq 1 ]]
- rm -rf /tmp/VulnWhisperer
# Test only qualy_vuln
- vuln_whisperer -c configs/test.ini -s qualys_vuln --mock --mock_dir test; [[ $? -eq 1 ]]
notifications:
on_success: change
on_failure: change # `always` will be the setting once code changes slow down

47
bin/vuln_whisperer
View File

@ -11,12 +11,14 @@ import argparse
import sys
import logging
def isFileValid(parser, arg):
if not os.path.exists(arg):
parser.error("The file %s does not exist!" % arg)
else:
return arg
def main():
parser = argparse.ArgumentParser(description=""" VulnWhisperer is designed to create actionable data from\
@ -31,8 +33,10 @@ def main():
help='JIRA required only! Scan name from scan to report')
parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=True,
help='Prints status out to screen (defaults to True)')
parser.add_argument('-u', '--username', dest='username', required=False, default=None, type=lambda x: x.strip(), help='The NESSUS username')
parser.add_argument('-p', '--password', dest='password', required=False, default=None, type=lambda x: x.strip(), help='The NESSUS password')
parser.add_argument('-u', '--username', dest='username', required=False, default=None,
help='The NESSUS username', type=lambda x: x.strip())
parser.add_argument('-p', '--password', dest='password', required=False, default=None,
help='The NESSUS password', type=lambda x: x.strip())
parser.add_argument('-F', '--fancy', action='store_true', help='Enable colourful logging output')
parser.add_argument('-d', '--debug', action='store_true', help='Enable debugging messages')
parser.add_argument('--mock', action='store_true', help='Enable mocked API responses')
@ -62,30 +66,28 @@ def main():
mock_api = mockAPI(args.mock_dir, args.verbose)
mock_api.mock_endpoints()
exit_code = 0
try:
if args.config and not args.section:
# this remains a print since we are in the main binary
print('WARNING: {warning}'.format(warning='No section was specified, vulnwhisperer will scrape enabled modules from config file. \
\nPlease specify a section using -s. \
\nPlease specify a section using -s. \
\nExample vuln_whisperer -c config.ini -s nessus'))
logger.info('No section was specified, vulnwhisperer will scrape enabled modules from the config file.')
config = vwConfig(config_in=args.config)
enabled_sections = config.get_sections_with_attribute('enabled')
for section in enabled_sections:
vw = vulnWhisperer(config=args.config,
profile=section,
verbose=args.verbose,
username=args.username,
password=args.password,
source=args.source,
scanname=args.scanname)
exit_code = vw.whisper_vulnerabilities()
# TODO: fix this to NOT be exit 1 unless in error
close_logging_handlers(logger)
sys.exit(exit_code)
config = vwConfig(config_in=args.config)
enabled_sections = config.get_sections_with_attribute('enabled')
for section in enabled_sections:
vw = vulnWhisperer(config=args.config,
profile=section,
verbose=args.verbose,
username=args.username,
password=args.password,
source=args.source,
scanname=args.scanname)
exit_code += vw.whisper_vulnerabilities()
else:
logger.info('Running vulnwhisperer for section {}'.format(args.section))
vw = vulnWhisperer(config=args.config,
@ -95,11 +97,10 @@ def main():
password=args.password,
source=args.source,
scanname=args.scanname)
exit_code += vw.whisper_vulnerabilities()
exit_code = vw.whisper_vulnerabilities()
# TODO: fix this to NOT be exit 1 unless in error
close_logging_handlers(logger)
sys.exit(exit_code)
close_logging_handlers(logger)
sys.exit(exit_code)
except Exception as e:
if args.verbose:

26
vulnwhisp/base/config.py
View File

@ -1,9 +1,8 @@
import os
import sys
import logging
# Support for python3
if (sys.version_info > (3, 0)):
if sys.version_info > (3, 0):
import configparser as cp
else:
import ConfigParser as cp
@ -45,7 +44,6 @@ class vwConfig(object):
return False
return True
def update_jira_profiles(self, profiles):
# create JIRA profiles in the ini config file
self.logger.debug('Updating Jira profiles: {}'.format(str(profiles)))
@ -59,27 +57,27 @@ class vwConfig(object):
except:
self.logger.warn("Creating config section for '{}'".format(section_name))
self.config.add_section(section_name)
self.config.set(section_name,'source',profile.split('.')[0])
self.config.set(section_name, 'source', profile.split('.')[0])
# in case any scan name contains '.' character
self.config.set(section_name,'scan_name','.'.join(profile.split('.')[1:]))
self.config.set(section_name,'jira_project', '')
self.config.set(section_name,'; if multiple components, separate by ","')
self.config.set(section_name,'components', '')
self.config.set(section_name,'; minimum criticality to report (low, medium, high or critical)')
self.config.set(section_name,'min_critical_to_report', 'high')
self.config.set(section_name,'; automatically report, boolean value ')
self.config.set(section_name,'autoreport', 'false')
self.config.set(section_name, 'scan_name', '.'.join(profile.split('.')[1:]))
self.config.set(section_name, 'jira_project', '')
self.config.set(section_name, '; if multiple components, separate by ","')
self.config.set(section_name, 'components', '')
self.config.set(section_name, '; minimum criticality to report (low, medium, high or critical)')
self.config.set(section_name, 'min_critical_to_report', 'high')
self.config.set(section_name, '; automatically report, boolean value ')
self.config.set(section_name, 'autoreport', 'false')
# TODO: try/catch this
# writing changes back to file
with open(self.config_in, 'w') as configfile:
self.config.write(configfile)
self.logger.debug('Written configuration to {}'.format(self.config_in))
# FIXME: this is the same as return None, that is the default return for return-less functions
return
def normalize_section(self, profile):
profile = "jira.{}".format(profile.lower().replace(" ","_"))
profile = "jira.{}".format(profile.lower().replace(" ", "_"))
self.logger.debug('Normalized profile as: {}'.format(profile))
return profile

75
vulnwhisp/frameworks/nessus.py
View File

@ -1,18 +1,13 @@
from datetime import datetime
import sys
import time
import json
import logging
import pytz
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
import pytz
from datetime import datetime
import json
import sys
import time
import logging
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
class NessusAPI(object):
SESSION = '/session'
@ -58,7 +53,7 @@ class NessusAPI(object):
def login(self):
resp = self.get_token()
if resp.status_code is 200:
if resp.status_code == 200:
self.headers['X-Cookie'] = 'token={token}'.format(token=resp.json()['token'])
else:
raise Exception('[FAIL] Could not login to Nessus')
@ -101,14 +96,6 @@ class NessusAPI(object):
token = self.request(self.SESSION, data=auth, json=False)
return token
def logout(self):
self.logger.debug('Logging out')
self.request(self.SESSION, method='DELETE')
def get_folders(self):
folders = self.request(self.FOLDERS, method='GET', json=True)
return folders
def get_scans(self):
scans = self.request(self.SCANS, method='GET', json=True)
return scans
@ -119,47 +106,10 @@ class NessusAPI(object):
self.logger.debug('Found {} scan_ids'.format(len(scan_ids)))
return scan_ids
def count_scan(self, scans, folder_id):
count = 0
for scan in scans:
if scan['folder_id'] == folder_id: count = count + 1
return count
def print_scans(self, data):
for folder in data['folders']:
self.logger.info("\\{0} - ({1})\\".format(folder['name'], self.count_scan(data['scans'], folder['id'])))
for scan in data['scans']:
if scan['folder_id'] == folder['id']:
self.logger.info("\t\"{0}\" - sid:{1} - uuid: {2}".format(scan['name'].encode('utf-8'), scan['id'], scan['uuid']))
def get_scan_details(self, scan_id):
data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json=True)
return data
def get_scan_history(self, scan_id):
data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json=True)
return data['history']
def get_scan_hosts(self, scan_id):
data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json=True)
return data['hosts']
def get_host_vulnerabilities(self, scan_id, host_id):
query = self.HOST_VULN.format(scan_id=scan_id, host_id=host_id)
data = self.request(query, method='GET', json=True)
return data
def get_plugin_info(self, scan_id, host_id, plugin_id):
query = self.PLUGINS.format(scan_id=scan_id, host_id=host_id, plugin_id=plugin_id)
data = self.request(query, method='GET', json=True)
return data
def export_scan(self, scan_id, history_id):
data = {'format': 'csv'}
query = self.EXPORT_REPORT.format(scan_id=scan_id, history_id=history_id)
req = self.request(query, data=data, method='POST')
return req
def download_scan(self, scan_id=None, history=None, export_format="", chapters="", dbpasswd="", profile=""):
running = True
counter = 0
@ -195,17 +145,6 @@ class NessusAPI(object):
content = self.request(self.EXPORT_TOKEN_DOWNLOAD.format(token_id=token_id), method='GET', download=True)
return content
@staticmethod
def merge_dicts(self, *dict_args):
"""
Given any number of dicts, shallow copy and merge into a new dict,
precedence goes to key value pairs in latter dicts.
"""
result = {}
for dictionary in dict_args:
result.update(dictionary)
return result
def get_utc_from_local(self, date_time, local_tz=None, epoch=True):
date_time = datetime.fromtimestamp(date_time)
if local_tz is None:

26
vulnwhisp/frameworks/qualys_vuln.py
View File

@ -3,12 +3,9 @@
__author__ = 'Nathan Young'
import xml.etree.ElementTree as ET
import pandas as pd
import qualysapi
import requests
import sys
import logging
import os
import qualysapi
import pandas as pd
import dateutil.parser as dp
@ -61,12 +58,13 @@ class qualysWhisperAPI(object):
'scan_ref': scan_id
}
scan_json = self.qgc.request(self.SCANS, parameters)
# First two columns are metadata we already have
# Last column corresponds to "target_distribution_across_scanner_appliances" element
# which doesn't follow the schema and breaks the pandas data manipulation
return pd.read_json(scan_json).iloc[2:-1]
class qualysUtils:
def __init__(self):
self.logger = logging.getLogger('qualysUtils')
@ -78,15 +76,15 @@ class qualysUtils:
class qualysVulnScan:
def __init__(
self,
config=None,
file_in=None,
file_stream=False,
delimiter=',',
quotechar='"',
):
self,
config=None,
file_in=None,
file_stream=False,
delimiter=',',
quotechar='"',
):
self.logger = logging.getLogger('qualysVulnScan')
self.file_in = file_in
self.file_stream = file_stream

15
vulnwhisp/test/mock.py
View File

@ -2,19 +2,20 @@ import os
import logging
import httpretty
class mockAPI(object):
def __init__(self, mock_dir=None, debug=False):
self.mock_dir = mock_dir
if not self.mock_dir:
# Try to guess the mock_dir if python setup.py develop was used
self.mock_dir = '/'.join(__file__.split('/')[:-3]) + '/test'
self.logger = logging.getLogger('mockAPI')
if debug:
self.logger.setLevel(logging.DEBUG)
self.logger.info('mockAPI initialised, API requests will be mocked'.format(self.mock_dir))
self.logger.info('mockAPI initialised, API requests will be mocked')
self.logger.debug('Test path resolved as {}'.format(self.mock_dir))
def get_directories(self, path):
@ -28,23 +29,23 @@ class mockAPI(object):
def qualys_vuln_callback(self, request, uri, response_headers):
self.logger.debug('Simulating response for {} ({})'.format(uri, request.body))
if 'list' in request.parsed_body['action']:
return [ 200,
return [200,
response_headers,
open('{}/{}'.format(self.qualys_vuln_path, 'scans')).read()]
elif 'fetch' in request.parsed_body['action']:
try:
response_body = open('{}/{}'.format(
self.qualys_vuln_path,
request.parsed_body['scan_ref'][0].replace('/', '_'))
self.qualys_vuln_path,
request.parsed_body['scan_ref'][0].replace('/', '_'))
).read()
except:
# Can't find the file, just send an empty response
response_body = ''
return [200, response_headers, response_body]
return [200, response_headers, response_body]
def create_nessus_resource(self, framework):
for filename in self.get_files('{}/{}'.format(self.mock_dir, framework)):
method, resource = filename.split('_',1)
method, resource = filename.split('_', 1)
resource = resource.replace('_', '/')
self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, method, resource))
httpretty.register_uri(

50
vulnwhisp/vulnwhisp.py
View File

@ -43,11 +43,11 @@ class vulnWhispererBase(object):
if self.CONFIG_SECTION is None:
raise Exception('Implementing class must define CONFIG_SECTION')
self.exit_code = 0
self.db_name = db_name
self.purge = purge
self.develop = develop
if config is not None:
self.config = vwConfig(config_in=config)
try:
@ -361,7 +361,7 @@ class vulnWhispererNessus(vulnWhispererBase):
if not scan_list:
self.logger.warn('No new scans to process. Exiting...')
return 0
return self.exit_code
# Create scan subfolders
@ -432,9 +432,15 @@ class vulnWhispererNessus(vulnWhispererBase):
self.record_insert(record_meta)
self.logger.info('File {filename} already exist! Updating database'.format(filename=relative_path_name))
else:
file_req = \
self.nessus.download_scan(scan_id=scan_id, history=history_id,
export_format='csv', profile=self.CONFIG_SECTION)
try:
file_req = \
self.nessus.download_scan(scan_id=scan_id, history=history_id,
export_format='csv', profile=self.CONFIG_SECTION)
except Exception as e:
self.logger.error('Could not download {} scan {}: {}'.format(self.CONFIG_SECTION, scan_id, str(e)))
self.exit_code += 1
continue
clean_csv = \
pd.read_csv(io.StringIO(file_req.decode('utf-8')))
if len(clean_csv) > 2:
@ -479,8 +485,8 @@ class vulnWhispererNessus(vulnWhispererBase):
self.logger.info('Scan aggregation complete! Connection to database closed.')
else:
self.logger.error('Failed to use scanner at {host}:{port}'.format(host=self.hostname, port=self.nessus_port))
return 1
return 0
self.exit_code += 1
return self.exit_code
class vulnWhispererQualys(vulnWhispererBase):
@ -550,7 +556,6 @@ class vulnWhispererQualys(vulnWhispererBase):
if debug:
self.logger.setLevel(logging.DEBUG)
self.qualys_scan = qualysScanReport(config=config)
self.latest_scans = self.qualys_scan.qw.get_all_scans()
self.directory_check()
@ -672,7 +677,7 @@ class vulnWhispererQualys(vulnWhispererBase):
else:
self.logger.info('No new scans to process. Exiting...')
self.conn.close()
return 0
return self.exit_code
class vulnWhispererOpenVAS(vulnWhispererBase):
@ -718,7 +723,6 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
if debug:
self.logger.setLevel(logging.DEBUG)
self.directory_check()
self.port = int(self.config.get(self.CONFIG_SECTION, 'port'))
self.develop = True
@ -809,7 +813,7 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
else:
self.logger.info('No new scans to process. Exiting...')
self.conn.close()
return 0
return self.exit_code
class vulnWhispererQualysVuln(vulnWhispererBase):
@ -850,7 +854,6 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
scan_reference=None,
output_format='json',
cleanup=True):
try:
launched_date
if 'Z' in launched_date:
launched_date = self.qualys_scan.utils.iso_to_epoch(launched_date)
@ -879,11 +882,16 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
self.logger.info('File {filename} already exist! Updating database'.format(filename=relative_path_name))
else:
self.logger.info('Processing report ID: {}'.format(report_id))
vuln_ready = self.qualys_scan.process_data(scan_id=report_id)
vuln_ready['scan_name'] = scan_name
vuln_ready['scan_reference'] = report_id
vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
try:
self.logger.info('Processing report ID: {}'.format(report_id))
vuln_ready = self.qualys_scan.process_data(scan_id=report_id)
vuln_ready['scan_name'] = scan_name
vuln_ready['scan_reference'] = report_id
vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
except Exception as e:
self.logger.error('Could not process {}: {}'.format(report_id, str(e)))
self.exit_code += 1
return self.exit_code
record_meta = (
scan_name,
@ -905,9 +913,7 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
f.write('\n')
self.logger.info('Report written to {}'.format(report_name))
except Exception as e:
self.logger.error('Could not process {}: {}'.format(report_id, str(e)))
return self.exit_code
def identify_scans_to_process(self):
@ -929,14 +935,14 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
counter += 1
r = app[1]
self.logger.info('Processing {}/{}'.format(counter, len(self.scans_to_process)))
self.whisper_reports(report_id=r['id'],
self.exit_code += self.whisper_reports(report_id=r['id'],
launched_date=r['date'],
scan_name=r['name'],
scan_reference=r['type'])
else:
self.logger.info('No new scans to process. Exiting...')
self.conn.close()
return 0
return self.exit_code
class vulnWhispererJIRA(vulnWhispererBase):