Merge branch 'beta-1.8' of https://github.com/HASecuritySolutions/VulnWhisperer into beta-1.8

.travis.yml

@@ -19,7 +19,22 @@ before_script:
     - flake8 . --count --exit-zero --exclude=deps/qualysapi --max-complexity=10 --max-line-length=127 --statistics
 script:
     - python setup.py install
+    # Test successful scan download and parsing
     - vuln_whisperer -c configs/test.ini --mock --mock_dir test
+    - rm -rf /tmp/VulnWhisperer
+    # Test one failed scan
+    - rm -f test/nessus/GET_scans_exports_164_download
+    - vuln_whisperer -c configs/test.ini --mock --mock_dir test; [[ $? -eq 1 ]]
+    - rm -rf /tmp/VulnWhisperer
+    # Test two failed scans
+    - rm -f test/qualys_vuln/scan_1553941061.87241
+    - vuln_whisperer -c configs/test.ini --mock --mock_dir test; [[ $? -eq 2 ]]
+    - rm -rf /tmp/VulnWhisperer
+    # Test only nessus
+    - vuln_whisperer -c configs/test.ini -s nessus --mock --mock_dir test; [[ $? -eq 1 ]]
+    - rm -rf /tmp/VulnWhisperer
+    # Test only qualy_vuln
+    - vuln_whisperer -c configs/test.ini -s qualys_vuln --mock --mock_dir test; [[ $? -eq 1 ]]
 notifications:
     on_success: change
     on_failure: change  # `always` will be the setting once code changes slow down

bin/vuln_whisperer

@@ -11,12 +11,14 @@ import argparse
 import sys
 import logging
 
+
 def isFileValid(parser, arg):
     if not os.path.exists(arg):
         parser.error("The file %s does not exist!" % arg)
     else:
         return arg
 
+
 def main():
 
     parser = argparse.ArgumentParser(description=""" VulnWhisperer is designed to create actionable data from\
@@ -31,8 +33,10 @@ def main():
                         help='JIRA required only! Scan name from scan to report')
     parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=True,
                         help='Prints status out to screen (defaults to True)')
-    parser.add_argument('-u', '--username', dest='username', required=False, default=None, type=lambda x: x.strip(), help='The NESSUS username')
+    parser.add_argument('-u', '--username', dest='username', required=False, default=None,
-    parser.add_argument('-p', '--password', dest='password', required=False, default=None, type=lambda x: x.strip(), help='The NESSUS password')
+                        help='The NESSUS username', type=lambda x: x.strip())
+    parser.add_argument('-p', '--password', dest='password', required=False, default=None,
+                        help='The NESSUS password', type=lambda x: x.strip())
     parser.add_argument('-F', '--fancy', action='store_true', help='Enable colourful logging output')
     parser.add_argument('-d', '--debug', action='store_true', help='Enable debugging messages')
     parser.add_argument('--mock', action='store_true', help='Enable mocked API responses')
@@ -62,6 +66,8 @@ def main():
         mock_api = mockAPI(args.mock_dir, args.verbose)
         mock_api.mock_endpoints()
 
+    exit_code = 0
+
     try:
         if args.config and not args.section:
             # this remains a print since we are in the main binary
@@ -69,6 +75,7 @@ def main():
                             \nPlease specify a section using -s. \
                                                     \nExample vuln_whisperer -c config.ini -s nessus'))
             logger.info('No section was specified, vulnwhisperer will scrape enabled modules from the config file.')
+            
             config = vwConfig(config_in=args.config)
             enabled_sections = config.get_sections_with_attribute('enabled')
 
@@ -80,12 +87,7 @@ def main():
                                    password=args.password,
                                    source=args.source,
                                    scanname=args.scanname)
-
+                exit_code += vw.whisper_vulnerabilities()
-                exit_code = vw.whisper_vulnerabilities()
-            # TODO: fix this to NOT be exit 1 unless in error
-            close_logging_handlers(logger)
-	    sys.exit(exit_code)
-
         else:
             logger.info('Running vulnwhisperer for section {}'.format(args.section))
             vw = vulnWhisperer(config=args.config,
@@ -95,9 +97,8 @@ def main():
                                password=args.password,
                                source=args.source,
                                scanname=args.scanname)
+            exit_code += vw.whisper_vulnerabilities()
 
-            exit_code = vw.whisper_vulnerabilities()
-            # TODO: fix this to NOT be exit 1 unless in error
         close_logging_handlers(logger)
         sys.exit(exit_code)
 

vulnwhisp/base/config.py

@@ -1,9 +1,8 @@
-import os
 import sys
 import logging
 
 # Support for python3
-if (sys.version_info > (3, 0)):
+if sys.version_info > (3, 0):
     import configparser as cp
 else:
     import ConfigParser as cp
@@ -45,7 +44,6 @@ class vwConfig(object):
                 return False
         return True
 
-
     def update_jira_profiles(self, profiles):
         # create JIRA profiles in the ini config file 
         self.logger.debug('Updating Jira profiles: {}'.format(str(profiles)))
@@ -59,16 +57,16 @@ class vwConfig(object):
             except:
                 self.logger.warn("Creating config section for '{}'".format(section_name))
                 self.config.add_section(section_name)
-                self.config.set(section_name,'source',profile.split('.')[0])
+                self.config.set(section_name, 'source', profile.split('.')[0])
                 # in case any scan name contains '.' character
-                self.config.set(section_name,'scan_name','.'.join(profile.split('.')[1:]))
+                self.config.set(section_name, 'scan_name', '.'.join(profile.split('.')[1:]))
-                self.config.set(section_name,'jira_project', '')
+                self.config.set(section_name, 'jira_project', '')
-                self.config.set(section_name,'; if multiple components, separate by ","')
+                self.config.set(section_name, '; if multiple components, separate by ","')
-                self.config.set(section_name,'components', '')
+                self.config.set(section_name, 'components', '')
-                self.config.set(section_name,'; minimum criticality to report (low, medium, high or critical)')
+                self.config.set(section_name, '; minimum criticality to report (low, medium, high or critical)')
-                self.config.set(section_name,'min_critical_to_report', 'high')
+                self.config.set(section_name, 'min_critical_to_report', 'high')
-                self.config.set(section_name,'; automatically report, boolean value ')
+                self.config.set(section_name, '; automatically report, boolean value ')
-                self.config.set(section_name,'autoreport', 'false')
+                self.config.set(section_name, 'autoreport', 'false')
         
         # TODO: try/catch this
         # writing changes back to file
@@ -80,6 +78,6 @@ class vwConfig(object):
         return
 
     def normalize_section(self, profile):
-        profile = "jira.{}".format(profile.lower().replace(" ","_"))
+        profile = "jira.{}".format(profile.lower().replace(" ", "_"))
         self.logger.debug('Normalized profile as: {}'.format(profile))
         return profile

vulnwhisp/frameworks/nessus.py

@@ -1,18 +1,13 @@
+from datetime import datetime
+import sys
+import time
+import json
+import logging
+import pytz
 import requests
 from requests.packages.urllib3.exceptions import InsecureRequestWarning
 requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
 
-import pytz
-from datetime import datetime
-import json
-import sys
-import time
-import logging
-
-from requests.packages.urllib3.exceptions import InsecureRequestWarning
-requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
-
-
 
 class NessusAPI(object):
     SESSION = '/session'
@@ -58,7 +53,7 @@ class NessusAPI(object):
 
     def login(self):
         resp = self.get_token()
-        if resp.status_code is 200:
+        if resp.status_code == 200:
             self.headers['X-Cookie'] = 'token={token}'.format(token=resp.json()['token'])
         else:
             raise Exception('[FAIL] Could not login to Nessus')
@@ -101,14 +96,6 @@ class NessusAPI(object):
         token = self.request(self.SESSION, data=auth, json=False)
         return token
 
-    def logout(self):
-        self.logger.debug('Logging out')
-        self.request(self.SESSION, method='DELETE')
-
-    def get_folders(self):
-        folders = self.request(self.FOLDERS, method='GET', json=True)
-        return folders
-
     def get_scans(self):
         scans = self.request(self.SCANS, method='GET', json=True)
         return scans
@@ -119,47 +106,10 @@ class NessusAPI(object):
         self.logger.debug('Found {} scan_ids'.format(len(scan_ids)))
         return scan_ids
 
-    def count_scan(self, scans, folder_id):
-        count = 0
-        for scan in scans:
-            if scan['folder_id'] == folder_id: count = count + 1
-        return count
-
-    def print_scans(self, data):
-        for folder in data['folders']:
-            self.logger.info("\\{0} - ({1})\\".format(folder['name'], self.count_scan(data['scans'], folder['id'])))
-            for scan in data['scans']:
-                if scan['folder_id'] == folder['id']:
-                    self.logger.info("\t\"{0}\" - sid:{1} - uuid: {2}".format(scan['name'].encode('utf-8'), scan['id'], scan['uuid']))
-
-    def get_scan_details(self, scan_id):
-        data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json=True)
-        return data
-
     def get_scan_history(self, scan_id):
         data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json=True)
         return data['history']
 
-    def get_scan_hosts(self, scan_id):
-        data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json=True)
-        return data['hosts']
-
-    def get_host_vulnerabilities(self, scan_id, host_id):
-        query = self.HOST_VULN.format(scan_id=scan_id, host_id=host_id)
-        data = self.request(query, method='GET', json=True)
-        return data
-
-    def get_plugin_info(self, scan_id, host_id, plugin_id):
-        query = self.PLUGINS.format(scan_id=scan_id, host_id=host_id, plugin_id=plugin_id)
-        data = self.request(query, method='GET', json=True)
-        return data
-
-    def export_scan(self, scan_id, history_id):
-        data = {'format': 'csv'}
-        query = self.EXPORT_REPORT.format(scan_id=scan_id, history_id=history_id)
-        req = self.request(query, data=data, method='POST')
-        return req
-
     def download_scan(self, scan_id=None, history=None, export_format="", chapters="", dbpasswd="", profile=""):
         running = True
         counter = 0
@@ -195,17 +145,6 @@ class NessusAPI(object):
             content = self.request(self.EXPORT_TOKEN_DOWNLOAD.format(token_id=token_id), method='GET', download=True)
         return content
 
-    @staticmethod
-    def merge_dicts(self, *dict_args):
-        """
-        Given any number of dicts, shallow copy and merge into a new dict,
-        precedence goes to key value pairs in latter dicts.
-        """
-        result = {}
-        for dictionary in dict_args:
-            result.update(dictionary)
-        return result
-
     def get_utc_from_local(self, date_time, local_tz=None, epoch=True):
         date_time = datetime.fromtimestamp(date_time)
         if local_tz is None:

vulnwhisp/frameworks/qualys_vuln.py

@@ -3,12 +3,9 @@
 __author__ = 'Nathan Young'
 
 import xml.etree.ElementTree as ET
-import pandas as pd
-import qualysapi
-import requests
-import sys
 import logging
-import os
+import qualysapi
+import pandas as pd
 import dateutil.parser as dp
 
 
@@ -66,6 +63,7 @@ class qualysWhisperAPI(object):
         # which doesn't follow the schema and breaks the pandas data manipulation
         return pd.read_json(scan_json).iloc[2:-1]
 
+
 class qualysUtils:
     def __init__(self):
         self.logger = logging.getLogger('qualysUtils')

vulnwhisp/test/mock.py

@@ -2,6 +2,7 @@ import os
 import logging
 import httpretty
 
+
 class mockAPI(object):
     def __init__(self, mock_dir=None, debug=False):
         self.mock_dir = mock_dir
@@ -14,7 +15,7 @@ class mockAPI(object):
         if debug:
             self.logger.setLevel(logging.DEBUG)
 
-        self.logger.info('mockAPI initialised, API requests will be mocked'.format(self.mock_dir))
+        self.logger.info('mockAPI initialised, API requests will be mocked')
         self.logger.debug('Test path resolved as {}'.format(self.mock_dir))
 
     def get_directories(self, path):
@@ -28,7 +29,7 @@ class mockAPI(object):
     def qualys_vuln_callback(self, request, uri, response_headers):
         self.logger.debug('Simulating response for {} ({})'.format(uri, request.body))
         if 'list' in request.parsed_body['action']:
-            return [ 200,
+            return [200,
                     response_headers,
                     open('{}/{}'.format(self.qualys_vuln_path, 'scans')).read()]
         elif 'fetch' in request.parsed_body['action']:
@@ -44,7 +45,7 @@ class mockAPI(object):
 
     def create_nessus_resource(self, framework):
         for filename in self.get_files('{}/{}'.format(self.mock_dir, framework)):
-            method, resource = filename.split('_',1)
+            method, resource = filename.split('_', 1)
             resource = resource.replace('_', '/')
             self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, method, resource))
             httpretty.register_uri(

vulnwhisp/vulnwhisp.py

@@ -43,11 +43,11 @@ class vulnWhispererBase(object):
         if self.CONFIG_SECTION is None:
                 raise Exception('Implementing class must define CONFIG_SECTION')
 
+        self.exit_code = 0
         self.db_name = db_name
         self.purge = purge
         self.develop = develop
 
-
         if config is not None:
             self.config = vwConfig(config_in=config)
             try:
@@ -361,7 +361,7 @@ class vulnWhispererNessus(vulnWhispererBase):
 
             if not scan_list:
                 self.logger.warn('No new scans to process. Exiting...')
-                return 0
+                return self.exit_code
 
             # Create scan subfolders
 
@@ -432,9 +432,15 @@ class vulnWhispererNessus(vulnWhispererBase):
                             self.record_insert(record_meta)
                             self.logger.info('File {filename} already exist! Updating database'.format(filename=relative_path_name))
                     else:
+                        try:
                             file_req = \
                                 self.nessus.download_scan(scan_id=scan_id, history=history_id,
                                                         export_format='csv', profile=self.CONFIG_SECTION)
+                        except Exception as e:
+                            self.logger.error('Could not download {} scan {}: {}'.format(self.CONFIG_SECTION, scan_id, str(e)))
+                            self.exit_code += 1
+                            continue
+                            
                         clean_csv = \
                             pd.read_csv(io.StringIO(file_req.decode('utf-8')))
                         if len(clean_csv) > 2:
@@ -479,8 +485,8 @@ class vulnWhispererNessus(vulnWhispererBase):
             self.logger.info('Scan aggregation complete! Connection to database closed.')
         else:
             self.logger.error('Failed to use scanner at {host}:{port}'.format(host=self.hostname, port=self.nessus_port))
-            return 1
+            self.exit_code += 1
-        return 0
+        return self.exit_code
 
 
 class vulnWhispererQualys(vulnWhispererBase):
@@ -550,7 +556,6 @@ class vulnWhispererQualys(vulnWhispererBase):
         if debug:
             self.logger.setLevel(logging.DEBUG)
         
-
         self.qualys_scan = qualysScanReport(config=config)
         self.latest_scans = self.qualys_scan.qw.get_all_scans()
         self.directory_check()
@@ -672,7 +677,7 @@ class vulnWhispererQualys(vulnWhispererBase):
         else:
             self.logger.info('No new scans to process. Exiting...')
         self.conn.close()
-        return 0
+        return self.exit_code
 
 
 class vulnWhispererOpenVAS(vulnWhispererBase):
@@ -718,7 +723,6 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
         if debug:
             self.logger.setLevel(logging.DEBUG)
 
-
         self.directory_check()
         self.port = int(self.config.get(self.CONFIG_SECTION, 'port'))
         self.develop = True
@@ -809,7 +813,7 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
         else:
             self.logger.info('No new scans to process. Exiting...')
         self.conn.close()
-        return 0
+        return self.exit_code
 
 
 class vulnWhispererQualysVuln(vulnWhispererBase):
@@ -850,7 +854,6 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
                         scan_reference=None,
                         output_format='json',
                         cleanup=True):
-        try:
             launched_date
             if 'Z' in launched_date:
                 launched_date = self.qualys_scan.utils.iso_to_epoch(launched_date)
@@ -879,11 +882,16 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
                 self.logger.info('File {filename} already exist! Updating database'.format(filename=relative_path_name))
 
             else:
+                try:
                     self.logger.info('Processing report ID: {}'.format(report_id))
                     vuln_ready = self.qualys_scan.process_data(scan_id=report_id)
                     vuln_ready['scan_name'] = scan_name
                     vuln_ready['scan_reference'] = report_id
                     vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
+                except Exception as e:
+                    self.logger.error('Could not process {}: {}'.format(report_id, str(e)))
+                    self.exit_code += 1
+                    return self.exit_code
 
                 record_meta = (
                     scan_name,
@@ -905,9 +913,7 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
                         f.write('\n')
 
                 self.logger.info('Report written to {}'.format(report_name))
-
+            return self.exit_code
-        except Exception as e:
-            self.logger.error('Could not process {}: {}'.format(report_id, str(e)))
 
 
     def identify_scans_to_process(self):
@@ -929,14 +935,14 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
                 counter += 1
                 r = app[1]
                 self.logger.info('Processing {}/{}'.format(counter, len(self.scans_to_process)))
-                self.whisper_reports(report_id=r['id'],
+                self.exit_code += self.whisper_reports(report_id=r['id'],
                                      launched_date=r['date'],
                                      scan_name=r['name'],
                                      scan_reference=r['type'])
         else:
             self.logger.info('No new scans to process. Exiting...')
         self.conn.close()
-        return 0
+        return self.exit_code
 
 
 class vulnWhispererJIRA(vulnWhispererBase):