Merge pull request #167 from pemontto/feature-nessus-stream

Feature nessus stream
2019-04-08 11:45:14 +02:00
parent 12e6c6d0d5 302037893d
commit 913bbfb2de
9 changed files with 73 additions and 59 deletions
--- a/.gitignore
+++ b/.gitignore
@ -4,6 +4,7 @@ logs/
 elk6/vulnwhisperer.ini
 resources/elk6/vulnwhisperer.ini
 configs/frameworks_example.ini
+tests/data

 # Byte-compiled / optimized / DLL files
 __pycache__/
--- a/.gitmodules
+++ b/.gitmodules
@ -1,3 +1,3 @@
-[submodule "test"]
-	path = test
+[submodule "tests/data"]
+	path = tests/data
 	url = https://github.com/HASecuritySolutions/VulnWhisperer-tests
--- a/.travis.yml
+++ b/.travis.yml
@ -3,7 +3,8 @@ language: python
 cache: pip
 python:
    - 2.7
-
+env:
+    - TEST_PATH=tests/data
 #    - 3.6
 #matrix:
 #    allow_failures:
@ -20,21 +21,22 @@ before_script:
 script:
    - python setup.py install
    # Test successful scan download and parsing
-    - vuln_whisperer -c configs/test.ini --mock --mock_dir test
    - rm -rf /tmp/VulnWhisperer
+    - vuln_whisperer -c configs/test.ini --mock --mock_dir ${TEST_PATH}
    # Test one failed scan
-    - rm -f test/nessus/GET_scans_exports_164_download
-    - vuln_whisperer -c configs/test.ini --mock --mock_dir test; [[ $? -eq 1 ]]
    - rm -rf /tmp/VulnWhisperer
+    - rm -f ${TEST_PATH}/nessus/GET_scans_exports_164_download
+    - vuln_whisperer -c configs/test.ini --mock --mock_dir ${TEST_PATH}; [[ $? -eq 1 ]]
    # Test two failed scans
-    - rm -f test/qualys_vuln/scan_1553941061.87241
-    - vuln_whisperer -c configs/test.ini --mock --mock_dir test; [[ $? -eq 2 ]]
    - rm -rf /tmp/VulnWhisperer
+    - rm -f ${TEST_PATH}/qualys_vuln/scan_1553941061.87241
+    - vuln_whisperer -c configs/test.ini --mock --mock_dir ${TEST_PATH}; [[ $? -eq 2 ]]
    # Test only nessus
-    - vuln_whisperer -c configs/test.ini -s nessus --mock --mock_dir test; [[ $? -eq 1 ]]
    - rm -rf /tmp/VulnWhisperer
+    - vuln_whisperer -c configs/test.ini -s nessus --mock --mock_dir ${TEST_PATH}; [[ $? -eq 1 ]]
    # Test only qualy_vuln
-    - vuln_whisperer -c configs/test.ini -s qualys_vuln --mock --mock_dir test; [[ $? -eq 1 ]]
+    - rm -rf /tmp/VulnWhisperer
+    - vuln_whisperer -c configs/test.ini -s qualys_vuln --mock --mock_dir ${TEST_PATH}; [[ $? -eq 1 ]]
 notifications:
    on_success: change
    on_failure: change  # `always` will be the setting once code changes slow down
--- a/bin/vuln_whisperer
+++ b/bin/vuln_whisperer
@ -37,10 +37,13 @@ def main():
                        help='The NESSUS username', type=lambda x: x.strip())
    parser.add_argument('-p', '--password', dest='password', required=False, default=None,
                        help='The NESSUS password', type=lambda x: x.strip())
-    parser.add_argument('-F', '--fancy', action='store_true', help='Enable colourful logging output')
-    parser.add_argument('-d', '--debug', action='store_true', help='Enable debugging messages')
-    parser.add_argument('--mock', action='store_true', help='Enable mocked API responses')
-    parser.add_argument('--mock_dir', dest='mock_dir', required=False, default='test',
+    parser.add_argument('-F', '--fancy', action='store_true',
+                        help='Enable colourful logging output')
+    parser.add_argument('-d', '--debug', action='store_true',
+                        help='Enable debugging messages')
+    parser.add_argument('--mock', action='store_true',
+                        help='Enable mocked API responses')
+    parser.add_argument('--mock_dir', dest='mock_dir', required=False, default=None,
                        help='Path of test directory')
    args = parser.parse_args()

--- a/tests/data
+++ b/tests/data
--- a/vulnwhisp/frameworks/nessus.py
+++ b/vulnwhisp/frameworks/nessus.py
@ -1,11 +1,13 @@
-from datetime import datetime
-import sys
-import time
 import json
 import logging
+import sys
+import time
+from datetime import datetime
+
 import pytz
 import requests
 from requests.packages.urllib3.exceptions import InsecureRequestWarning
+
 requests.packages.urllib3.disable_warnings(InsecureRequestWarning)


@ -34,7 +36,10 @@ class NessusAPI(object):
        self.base = 'https://{hostname}:{port}'.format(hostname=hostname, port=port)
        self.verbose = verbose

-        self.headers = {
+        self.session = requests.Session()
+        self.session.verify = False
+        self.session.stream = True
+        self.session.headers = {
            'Origin': self.base,
            'Accept-Encoding': 'gzip, deflate, br',
            'Accept-Language': 'en-US,en;q=0.8',
@ -52,27 +57,24 @@ class NessusAPI(object):
        self.scan_ids = self.get_scan_ids()

    def login(self):
-        resp = self.get_token()
+        auth = '{"username":"%s", "password":"%s"}' % (self.user, self.password)
+        resp = self.request(self.SESSION, data=auth, json_output=False)
        if resp.status_code == 200:
-            self.headers['X-Cookie'] = 'token={token}'.format(token=resp.json()['token'])
+            self.session.headers['X-Cookie'] = 'token={token}'.format(token=resp.json()['token'])
        else:
            raise Exception('[FAIL] Could not login to Nessus')

-    def request(self, url, data=None, headers=None, method='POST', download=False, json=False):
-        if headers is None:
-            headers = self.headers
+    def request(self, url, data=None, headers=None, method='POST', download=False, json_output=False):
        timeout = 0
        success = False
-
+        
+        method = method.lower()
        url = self.base + url
        self.logger.debug('Requesting to url {}'.format(url))
-        methods = {'GET': requests.get,
-                   'POST': requests.post,
-                   'DELETE': requests.delete}

        while (timeout <= 10) and (not success):
-            data = methods[method](url, data=data, headers=self.headers, verify=False)
-            if data.status_code == 401:
+            response = getattr(self.session, method)(url, data=data)
+            if response.status_code == 401:
                if url == self.base + self.SESSION:
                    break
                try:
@ -84,20 +86,22 @@ class NessusAPI(object):
            else:
                success = True

-        if json:
-            data = data.json()
+        if json_output:
+            return response.json()
        if download:
            self.logger.debug('Returning data.content')
-            return data.content
-        return data
-
-    def get_token(self):
-        auth = '{"username":"%s", "password":"%s"}' % (self.user, self.password)
-        token = self.request(self.SESSION, data=auth, json=False)
-        return token
+            response_data = ''
+            count = 0
+            for chunk in response.iter_content(chunk_size=8192):
+                count += 1
+                if chunk:
+                    response_data += chunk
+            self.logger.debug('Processed {} chunks'.format(count))
+            return response_data
+        return response

    def get_scans(self):
-        scans = self.request(self.SCANS, method='GET', json=True)
+        scans = self.request(self.SCANS, method='GET', json_output=True)
        return scans

    def get_scan_ids(self):
@ -107,10 +111,10 @@ class NessusAPI(object):
        return scan_ids

    def get_scan_history(self, scan_id):
-        data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json=True)
+        data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json_output=True)
        return data['history']

-    def download_scan(self, scan_id=None, history=None, export_format="", chapters="", dbpasswd="", profile=""):
+    def download_scan(self, scan_id=None, history=None, export_format="", profile=""):
        running = True
        counter = 0

@ -120,7 +124,7 @@ class NessusAPI(object):
        else:
            query = self.EXPORT_HISTORY.format(scan_id=scan_id, history_id=history)
            scan_id = str(scan_id)
-        req = self.request(query, data=json.dumps(data), method='POST', json=True)
+        req = self.request(query, data=json.dumps(data), method='POST', json_output=True)
        try:
            file_id = req['file']
            token_id = req['token'] if 'token' in req else req['temp_token']
@ -131,7 +135,7 @@ class NessusAPI(object):
            time.sleep(2)
            counter += 2
            report_status = self.request(self.EXPORT_STATUS.format(scan_id=scan_id, file_id=file_id), method='GET',
-                                         json=True)
+                                         json_output=True)
            running = report_status['status'] != 'ready'
            sys.stdout.write(".")
            sys.stdout.flush()
@ -139,7 +143,7 @@ class NessusAPI(object):
            if counter % 60 == 0:
                self.logger.info("Completed: {}".format(counter))
        self.logger.info("Done: {}".format(counter))
-        if profile=='tenable':
+        if profile == 'tenable':
            content = self.request(self.EXPORT_FILE_DOWNLOAD.format(scan_id=scan_id, file_id=file_id), method='GET', download=True)
        else:
            content = self.request(self.EXPORT_TOKEN_DOWNLOAD.format(token_id=token_id), method='GET', download=True)
--- a/vulnwhisp/frameworks/qualys_vuln.py
+++ b/vulnwhisp/frameworks/qualys_vuln.py
@ -2,12 +2,13 @@
 # -*- coding: utf-8 -*-
 __author__ = 'Nathan Young'

-import xml.etree.ElementTree as ET
-import sys
 import logging
-import qualysapi
-import pandas as pd
+import sys
+import xml.etree.ElementTree as ET
+
 import dateutil.parser as dp
+import pandas as pd
+import qualysapi


 class qualysWhisperAPI(object):
@ -78,13 +79,13 @@ class qualysUtils:
 class qualysVulnScan:

    def __init__(
-                self,
-                config=None,
-                file_in=None,
-                file_stream=False,
-                delimiter=',',
-                quotechar='"',
-            ):
+        self,
+        config=None,
+        file_in=None,
+        file_stream=False,
+        delimiter=',',
+        quotechar='"',
+    ):
        self.logger = logging.getLogger('qualysVulnScan')
        self.file_in = file_in
        self.file_stream = file_stream
@ -109,7 +110,10 @@ class qualysVulnScan:
        self.logger.info('Downloading scan ID: {}'.format(scan_id))
        scan_report = self.qw.get_scan_details(scan_id=scan_id)
        if not scan_report.empty:
-            keep_columns = ['category', 'cve_id', 'cvss3_base', 'cvss3_temporal', 'cvss_base', 'cvss_temporal', 'dns', 'exploitability', 'fqdn', 'impact', 'ip', 'ip_status', 'netbios', 'os', 'pci_vuln', 'port', 'protocol', 'qid', 'results', 'severity', 'solution', 'ssl', 'threat', 'title', 'type', 'vendor_reference']
+            keep_columns = ['category', 'cve_id', 'cvss3_base', 'cvss3_temporal', 'cvss_base',
+                'cvss_temporal', 'dns', 'exploitability', 'fqdn', 'impact', 'ip', 'ip_status',
+                'netbios', 'os', 'pci_vuln', 'port', 'protocol', 'qid', 'results', 'severity',
+                'solution', 'ssl', 'threat', 'title', 'type', 'vendor_reference']
            scan_report = scan_report.filter(keep_columns)
            scan_report['severity'] = scan_report['severity'].astype(int).astype(str)
            scan_report['qid'] = scan_report['qid'].astype(int).astype(str)
--- a/vulnwhisp/test/mock.py
+++ b/vulnwhisp/test/mock.py
@ -6,12 +6,12 @@ import httpretty
 class mockAPI(object):
    def __init__(self, mock_dir=None, debug=False):
        self.mock_dir = mock_dir
+
        if not self.mock_dir:
            # Try to guess the mock_dir if python setup.py develop was used
-            self.mock_dir = '/'.join(__file__.split('/')[:-3]) + '/test'
+            self.mock_dir = '/'.join(__file__.split('/')[:-3]) + '/tests/data'

        self.logger = logging.getLogger('mockAPI')
-
        if debug:
            self.logger.setLevel(logging.DEBUG)

--- a/vulnwhisp/vulnwhisp.py
+++ b/vulnwhisp/vulnwhisp.py
@ -410,7 +410,7 @@ class vulnWhispererNessus(vulnWhispererBase):
                if status in ['completed', 'imported']:
                    file_name = '%s_%s_%s_%s.%s' % (scan_name, scan_id,
                                                    history_id, norm_time, 'csv')
-                    repls = (('\\', '_'), ('/', '_'), ('/', '_'), (' ', '_'))
+                    repls = (('\\', '_'), ('/', '_'), (' ', '_'))
                    file_name = reduce(lambda a, kv: a.replace(*kv), repls, file_name)
                    relative_path_name = self.path_check(folder_name + '/' + file_name)