diff --git a/.gitignore b/.gitignore index b4a878c..9fc0cb6 100644 --- a/.gitignore +++ b/.gitignore @@ -4,6 +4,7 @@ logs/ elk6/vulnwhisperer.ini resources/elk6/vulnwhisperer.ini configs/frameworks_example.ini +tests/data # Byte-compiled / optimized / DLL files __pycache__/ diff --git a/.gitmodules b/.gitmodules index 4d6eb1b..546e654 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,3 @@ -[submodule "test"] - path = test +[submodule "tests/data"] + path = tests/data url = https://github.com/HASecuritySolutions/VulnWhisperer-tests diff --git a/.travis.yml b/.travis.yml index c806fb0..c412177 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,8 @@ language: python cache: pip python: - 2.7 - +env: + - TEST_PATH=tests/data # - 3.6 #matrix: # allow_failures: @@ -20,21 +21,22 @@ before_script: script: - python setup.py install # Test successful scan download and parsing - - vuln_whisperer -c configs/test.ini --mock --mock_dir test - rm -rf /tmp/VulnWhisperer + - vuln_whisperer -c configs/test.ini --mock --mock_dir ${TEST_PATH} # Test one failed scan - - rm -f test/nessus/GET_scans_exports_164_download - - vuln_whisperer -c configs/test.ini --mock --mock_dir test; [[ $? -eq 1 ]] - rm -rf /tmp/VulnWhisperer + - rm -f ${TEST_PATH}/nessus/GET_scans_exports_164_download + - vuln_whisperer -c configs/test.ini --mock --mock_dir ${TEST_PATH}; [[ $? -eq 1 ]] # Test two failed scans - - rm -f test/qualys_vuln/scan_1553941061.87241 - - vuln_whisperer -c configs/test.ini --mock --mock_dir test; [[ $? -eq 2 ]] - rm -rf /tmp/VulnWhisperer + - rm -f ${TEST_PATH}/qualys_vuln/scan_1553941061.87241 + - vuln_whisperer -c configs/test.ini --mock --mock_dir ${TEST_PATH}; [[ $? -eq 2 ]] # Test only nessus - - vuln_whisperer -c configs/test.ini -s nessus --mock --mock_dir test; [[ $? -eq 1 ]] - rm -rf /tmp/VulnWhisperer + - vuln_whisperer -c configs/test.ini -s nessus --mock --mock_dir ${TEST_PATH}; [[ $? -eq 1 ]] # Test only qualy_vuln - - vuln_whisperer -c configs/test.ini -s qualys_vuln --mock --mock_dir test; [[ $? -eq 1 ]] + - rm -rf /tmp/VulnWhisperer + - vuln_whisperer -c configs/test.ini -s qualys_vuln --mock --mock_dir ${TEST_PATH}; [[ $? -eq 1 ]] notifications: on_success: change on_failure: change # `always` will be the setting once code changes slow down diff --git a/bin/vuln_whisperer b/bin/vuln_whisperer index 010e9db..09ed142 100644 --- a/bin/vuln_whisperer +++ b/bin/vuln_whisperer @@ -37,10 +37,13 @@ def main(): help='The NESSUS username', type=lambda x: x.strip()) parser.add_argument('-p', '--password', dest='password', required=False, default=None, help='The NESSUS password', type=lambda x: x.strip()) - parser.add_argument('-F', '--fancy', action='store_true', help='Enable colourful logging output') - parser.add_argument('-d', '--debug', action='store_true', help='Enable debugging messages') - parser.add_argument('--mock', action='store_true', help='Enable mocked API responses') - parser.add_argument('--mock_dir', dest='mock_dir', required=False, default='test', + parser.add_argument('-F', '--fancy', action='store_true', + help='Enable colourful logging output') + parser.add_argument('-d', '--debug', action='store_true', + help='Enable debugging messages') + parser.add_argument('--mock', action='store_true', + help='Enable mocked API responses') + parser.add_argument('--mock_dir', dest='mock_dir', required=False, default=None, help='Path of test directory') args = parser.parse_args() diff --git a/test b/tests/data similarity index 100% rename from test rename to tests/data diff --git a/vulnwhisp/frameworks/nessus.py b/vulnwhisp/frameworks/nessus.py index 7944963..23c67d6 100755 --- a/vulnwhisp/frameworks/nessus.py +++ b/vulnwhisp/frameworks/nessus.py @@ -1,11 +1,13 @@ -from datetime import datetime -import sys -import time import json import logging +import sys +import time +from datetime import datetime + import pytz import requests from requests.packages.urllib3.exceptions import InsecureRequestWarning + requests.packages.urllib3.disable_warnings(InsecureRequestWarning) @@ -34,7 +36,10 @@ class NessusAPI(object): self.base = 'https://{hostname}:{port}'.format(hostname=hostname, port=port) self.verbose = verbose - self.headers = { + self.session = requests.Session() + self.session.verify = False + self.session.stream = True + self.session.headers = { 'Origin': self.base, 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'en-US,en;q=0.8', @@ -52,27 +57,24 @@ class NessusAPI(object): self.scan_ids = self.get_scan_ids() def login(self): - resp = self.get_token() + auth = '{"username":"%s", "password":"%s"}' % (self.user, self.password) + resp = self.request(self.SESSION, data=auth, json_output=False) if resp.status_code == 200: - self.headers['X-Cookie'] = 'token={token}'.format(token=resp.json()['token']) + self.session.headers['X-Cookie'] = 'token={token}'.format(token=resp.json()['token']) else: raise Exception('[FAIL] Could not login to Nessus') - def request(self, url, data=None, headers=None, method='POST', download=False, json=False): - if headers is None: - headers = self.headers + def request(self, url, data=None, headers=None, method='POST', download=False, json_output=False): timeout = 0 success = False - + + method = method.lower() url = self.base + url self.logger.debug('Requesting to url {}'.format(url)) - methods = {'GET': requests.get, - 'POST': requests.post, - 'DELETE': requests.delete} while (timeout <= 10) and (not success): - data = methods[method](url, data=data, headers=self.headers, verify=False) - if data.status_code == 401: + response = getattr(self.session, method)(url, data=data) + if response.status_code == 401: if url == self.base + self.SESSION: break try: @@ -84,20 +86,22 @@ class NessusAPI(object): else: success = True - if json: - data = data.json() + if json_output: + return response.json() if download: self.logger.debug('Returning data.content') - return data.content - return data - - def get_token(self): - auth = '{"username":"%s", "password":"%s"}' % (self.user, self.password) - token = self.request(self.SESSION, data=auth, json=False) - return token + response_data = '' + count = 0 + for chunk in response.iter_content(chunk_size=8192): + count += 1 + if chunk: + response_data += chunk + self.logger.debug('Processed {} chunks'.format(count)) + return response_data + return response def get_scans(self): - scans = self.request(self.SCANS, method='GET', json=True) + scans = self.request(self.SCANS, method='GET', json_output=True) return scans def get_scan_ids(self): @@ -107,10 +111,10 @@ class NessusAPI(object): return scan_ids def get_scan_history(self, scan_id): - data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json=True) + data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json_output=True) return data['history'] - def download_scan(self, scan_id=None, history=None, export_format="", chapters="", dbpasswd="", profile=""): + def download_scan(self, scan_id=None, history=None, export_format="", profile=""): running = True counter = 0 @@ -120,7 +124,7 @@ class NessusAPI(object): else: query = self.EXPORT_HISTORY.format(scan_id=scan_id, history_id=history) scan_id = str(scan_id) - req = self.request(query, data=json.dumps(data), method='POST', json=True) + req = self.request(query, data=json.dumps(data), method='POST', json_output=True) try: file_id = req['file'] token_id = req['token'] if 'token' in req else req['temp_token'] @@ -131,7 +135,7 @@ class NessusAPI(object): time.sleep(2) counter += 2 report_status = self.request(self.EXPORT_STATUS.format(scan_id=scan_id, file_id=file_id), method='GET', - json=True) + json_output=True) running = report_status['status'] != 'ready' sys.stdout.write(".") sys.stdout.flush() @@ -139,7 +143,7 @@ class NessusAPI(object): if counter % 60 == 0: self.logger.info("Completed: {}".format(counter)) self.logger.info("Done: {}".format(counter)) - if profile=='tenable': + if profile == 'tenable': content = self.request(self.EXPORT_FILE_DOWNLOAD.format(scan_id=scan_id, file_id=file_id), method='GET', download=True) else: content = self.request(self.EXPORT_TOKEN_DOWNLOAD.format(token_id=token_id), method='GET', download=True) diff --git a/vulnwhisp/frameworks/qualys_vuln.py b/vulnwhisp/frameworks/qualys_vuln.py index 0ba409d..69cddfa 100644 --- a/vulnwhisp/frameworks/qualys_vuln.py +++ b/vulnwhisp/frameworks/qualys_vuln.py @@ -2,12 +2,13 @@ # -*- coding: utf-8 -*- __author__ = 'Nathan Young' -import xml.etree.ElementTree as ET -import sys import logging -import qualysapi -import pandas as pd +import sys +import xml.etree.ElementTree as ET + import dateutil.parser as dp +import pandas as pd +import qualysapi class qualysWhisperAPI(object): @@ -78,13 +79,13 @@ class qualysUtils: class qualysVulnScan: def __init__( - self, - config=None, - file_in=None, - file_stream=False, - delimiter=',', - quotechar='"', - ): + self, + config=None, + file_in=None, + file_stream=False, + delimiter=',', + quotechar='"', + ): self.logger = logging.getLogger('qualysVulnScan') self.file_in = file_in self.file_stream = file_stream @@ -109,7 +110,10 @@ class qualysVulnScan: self.logger.info('Downloading scan ID: {}'.format(scan_id)) scan_report = self.qw.get_scan_details(scan_id=scan_id) if not scan_report.empty: - keep_columns = ['category', 'cve_id', 'cvss3_base', 'cvss3_temporal', 'cvss_base', 'cvss_temporal', 'dns', 'exploitability', 'fqdn', 'impact', 'ip', 'ip_status', 'netbios', 'os', 'pci_vuln', 'port', 'protocol', 'qid', 'results', 'severity', 'solution', 'ssl', 'threat', 'title', 'type', 'vendor_reference'] + keep_columns = ['category', 'cve_id', 'cvss3_base', 'cvss3_temporal', 'cvss_base', + 'cvss_temporal', 'dns', 'exploitability', 'fqdn', 'impact', 'ip', 'ip_status', + 'netbios', 'os', 'pci_vuln', 'port', 'protocol', 'qid', 'results', 'severity', + 'solution', 'ssl', 'threat', 'title', 'type', 'vendor_reference'] scan_report = scan_report.filter(keep_columns) scan_report['severity'] = scan_report['severity'].astype(int).astype(str) scan_report['qid'] = scan_report['qid'].astype(int).astype(str) diff --git a/vulnwhisp/test/mock.py b/vulnwhisp/test/mock.py index fcc7e1c..5d48729 100644 --- a/vulnwhisp/test/mock.py +++ b/vulnwhisp/test/mock.py @@ -6,12 +6,12 @@ import httpretty class mockAPI(object): def __init__(self, mock_dir=None, debug=False): self.mock_dir = mock_dir + if not self.mock_dir: # Try to guess the mock_dir if python setup.py develop was used - self.mock_dir = '/'.join(__file__.split('/')[:-3]) + '/test' + self.mock_dir = '/'.join(__file__.split('/')[:-3]) + '/tests/data' self.logger = logging.getLogger('mockAPI') - if debug: self.logger.setLevel(logging.DEBUG) diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index 21351da..782af0e 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -410,7 +410,7 @@ class vulnWhispererNessus(vulnWhispererBase): if status in ['completed', 'imported']: file_name = '%s_%s_%s_%s.%s' % (scan_name, scan_id, history_id, norm_time, 'csv') - repls = (('\\', '_'), ('/', '_'), ('/', '_'), (' ', '_')) + repls = (('\\', '_'), ('/', '_'), (' ', '_')) file_name = reduce(lambda a, kv: a.replace(*kv), repls, file_name) relative_path_name = self.path_check(folder_name + '/' + file_name)