fix logging and remove username/password

bin/vuln_whisperer

@@ -32,12 +32,8 @@ def main():
                         help='JIRA required only! Source scanner to report')
     parser.add_argument('-n', '--scanname', dest='scanname', required=False,
                         help='JIRA required only! Scan name from scan to report')
-    parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=True,
+    parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
                         help='Prints status out to screen (defaults to True)')
-    parser.add_argument('-u', '--username', dest='username', required=False, default=None,
-                        help='The NESSUS username', type=lambda x: x.strip())
-    parser.add_argument('-p', '--password', dest='password', required=False, default=None,
-                        help='The NESSUS password', type=lambda x: x.strip())
     parser.add_argument('-F', '--fancy', action='store_true',
                         help='Enable colourful logging output')
     parser.add_argument('-d', '--debug', action='store_true',
@@ -51,14 +47,14 @@ def main():
     # First setup logging
     logging.basicConfig(
         stream=sys.stdout,
-        #format only applies when not using -F flag for colouring
+        # format only applies when not using -F flag for colouring
         format='%(levelname)s:%(name)s:%(funcName)s:%(message)s',
-        level=logging.DEBUG if args.debug else logging.INFO
+        level=logging.DEBUG if args.debug else logging.INFO if args.verbose else logging.WARNING
     )
     logger = logging.getLogger()
     # we set up the logger to log as well to file
     fh = logging.FileHandler('vulnwhisperer.log')
-    fh.setLevel(logging.DEBUG if args.debug else logging.INFO)
+    fh.setLevel(logging.DEBUG if args.debug else logging.INFO if args.verbose else logging.WARNING)
     fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s - %(funcName)s:%(message)s", "%Y-%m-%d %H:%M:%S"))
     logger.addHandler(fh)
 
@@ -87,8 +83,7 @@ def main():
                 vw = vulnWhisperer(config=args.config,
                                    profile=section,
                                    verbose=args.verbose,
-                                   username=args.username,
+                                   debug=args.debug,
-                                   password=args.password,
                                    source=args.source,
                                    scanname=args.scanname)
                 exit_code += vw.whisper_vulnerabilities()
@@ -97,8 +92,7 @@ def main():
             vw = vulnWhisperer(config=args.config,
                                profile=args.section,
                                verbose=args.verbose,
-                               username=args.username,
+                               debug=args.debug,
-                               password=args.password,
                                source=args.source,
                                scanname=args.scanname)
             exit_code += vw.whisper_vulnerabilities()
@@ -107,10 +101,8 @@ def main():
         sys.exit(exit_code)
 
     except Exception as e:
-        if args.verbose:
+        logger.error('{}'.format(str(e)))
-            # this will remain a print since we are in the main binary
+        print('ERROR: {error}'.format(error=e))
-            logger.error('{}'.format(str(e)))
-            print('ERROR: {error}'.format(error=e))
         # TODO: fix this to NOT be exit 2 unless in error
         close_logging_handlers(logger)
         sys.exit(2)

configs/frameworks_example.ini

@@ -9,7 +9,7 @@ password=nessus_password
 write_path=/opt/VulnWhisperer/data/nessus/
 db_path=/opt/VulnWhisperer/data/database
 trash=false
-verbose=true
+verbose=false
 
 [tenable]
 enabled=true
@@ -22,7 +22,7 @@ password=tenable.io_password
 write_path=/opt/VulnWhisperer/data/tenable/
 db_path=/opt/VulnWhisperer/data/database
 trash=false
-verbose=true
+verbose=false
 
 [qualys_web]
 #Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
@@ -48,7 +48,7 @@ username = exampleuser
 password = examplepass
 write_path=/opt/VulnWhisperer/data/qualys_vuln/
 db_path=/opt/VulnWhisperer/data/database
-verbose=true
+verbose=false
 
 [detectify]
 #Reference https://developer.detectify.com/
@@ -70,7 +70,7 @@ username = exampleuser
 password = examplepass
 write_path=/opt/VulnWhisperer/data/openvas/
 db_path=/opt/VulnWhisperer/data/database
-verbose=true
+verbose=false
 
 [jira]
 enabled = false
@@ -88,7 +88,7 @@ dns_resolv = False
 #scan_name = Test Scan
 #jira_project = PROJECT
 ; if multiple components, separate by "," = None
-#components = 
+#components =
 ; minimum criticality to report (low, medium, high or critical) = None
 #min_critical_to_report = high
 

configs/test.ini

@@ -9,7 +9,7 @@ password=nessus_password
 write_path=/opt/VulnWhisperer/data/nessus/
 db_path=/opt/VulnWhisperer/data/database
 trash=false
-verbose=true
+verbose=false
 
 [tenable]
 enabled=true
@@ -22,73 +22,73 @@ password=tenable.io_password
 write_path=/opt/VulnWhisperer/data/tenable/
 db_path=/opt/VulnWhisperer/data/database
 trash=false
-verbose=true
+verbose=false
 
 [qualys_web]
 #Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
-enabled = true
+enabled=true
-hostname = qualys_web
+hostname=qualys_web
-username = exampleuser
+username=exampleuser
-password = examplepass
+password=examplepass
 write_path=/opt/VulnWhisperer/data/qualys_web/
 db_path=/opt/VulnWhisperer/data/database
-verbose=true
+verbose=false
 
 # Set the maximum number of retries each connection should attempt.
 #Note, this applies only to failed connections and timeouts, never to requests where the server returns a response.
-max_retries = 10
+max_retries=10
 # Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID.
-template_id = 289109
+template_id=289109
 
 [qualys_vuln]
 #Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
-enabled = true
+enabled=true
-hostname = qualys_vuln
+hostname=qualys_vuln
-username = exampleuser
+username=exampleuser
-password = examplepass
+password=examplepass
 write_path=/opt/VulnWhisperer/data/qualys_vuln/
 db_path=/opt/VulnWhisperer/data/database
-verbose=true
+verbose=false
 
 [detectify]
 #Reference https://developer.detectify.com/
-enabled = false
+enabled=false
-hostname = detectify
+hostname=detectify
 #username variable used as apiKey
-username = exampleuser
+username=exampleuser
 #password variable used as secretKey
-password = examplepass
+password=examplepass
 write_path =/opt/VulnWhisperer/data/detectify/
-db_path = /opt/VulnWhisperer/data/database
+db_path=/opt/VulnWhisperer/data/database
-verbose = true
+verbose=false
 
 [openvas]
-enabled = true
+enabled=true
-hostname = openvas
+hostname=openvas
-port = 4000
+port=4000
-username = exampleuser
+username=exampleuser
-password = examplepass
+password=examplepass
 write_path=/opt/VulnWhisperer/data/openvas/
 db_path=/opt/VulnWhisperer/data/database
-verbose=true
+verbose=false
 
 [jira]
-enabled = false
+enabled=false
-hostname = jira-host
+hostname=jira-host
-username = username
+username=username
-password = password
+password=password
-write_path = /opt/VulnWhisperer/data/jira/
+write_path=/opt/VulnWhisperer/data/jira/
-db_path = /opt/VulnWhisperer/data/database
+db_path=/opt/VulnWhisperer/data/database
-verbose = true
+verbose=false
-dns_resolv = False
+dns_resolv=False
 
 #Sample jira report scan, will automatically be created for existent scans
 #[jira.qualys_vuln.test_scan]
-#source = qualys_vuln
+#source=qualys_vuln
-#scan_name = Test Scan
+#scan_name=Test Scan
-#jira_project = PROJECT
+#jira_project=PROJECT
-; if multiple components, separate by "," = None
+; if multiple components, separate by ","=None
-#components = 
+#components =
-; minimum criticality to report (low, medium, high or critical) = None
+; minimum criticality to report (low, medium, high or critical)=None
-#min_critical_to_report = high
+#min_critical_to_report=high
 

vulnwhisp/frameworks/nessus.py

@@ -44,8 +44,7 @@ class NessusAPI(object):
 
     def __init__(self, hostname=None, port=None, username=None, password=None, verbose=True, profile=None, access_key=None, secret_key=None):
         self.logger = logging.getLogger('NessusAPI')
-        if verbose:
+        self.logger.setLevel(logging.DEBUG if verbose else logging.INFO)
-            self.logger.setLevel(logging.DEBUG)
         if not all((username, password)) and not all((access_key, secret_key)):
             raise Exception('ERROR: Missing username, password or API keys.')
 
@@ -165,8 +164,6 @@ class NessusAPI(object):
             report_status = self.request(self.EXPORT_STATUS.format(scan_id=scan_id, file_id=file_id), method='GET',
                                          json_output=True)
             running = report_status['status'] != 'ready'
-            sys.stdout.write('.')
-            sys.stdout.flush()
         if self.profile == 'tenable' or self.api_keys:
             content = self.request(self.EXPORT_FILE_DOWNLOAD.format(scan_id=scan_id, file_id=file_id), method='GET', download=True)
         else:

vulnwhisp/test/mock.py

@@ -31,14 +31,14 @@ class mockAPI(object):
         for filename in self.get_files('{}/{}'.format(self.mock_dir, framework)):
             method, resource = filename.split('_', 1)
             resource = resource.replace('_', '/')
-            self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, method, resource))
+            self.logger.info('Adding mocked {} endpoint {} {}'.format(framework, method, resource))
             httpretty.register_uri(
                 getattr(httpretty, method), 'https://{}:443/{}'.format(framework, resource),
                 body=open('{}/{}/{}'.format(self.mock_dir, framework, filename)).read()
             )
 
     def qualys_vuln_callback(self, request, uri, response_headers):
-        self.logger.debug('Simulating response for {} ({})'.format(uri, request.body))
+        self.logger.info('Simulating response for {} ({})'.format(uri, request.body))
         if 'list' in request.parsed_body['action']:
             return [200,
                     response_headers,
@@ -56,19 +56,19 @@ class mockAPI(object):
 
     def create_qualys_vuln_resource(self, framework):
         # Create health check endpoint
-        self.logger.debug('Adding mocked {} endpoint GET msp/about.php'.format(framework))
+        self.logger.info('Adding mocked {} endpoint GET msp/about.php'.format(framework))
         httpretty.register_uri(
                 httpretty.GET,
                 'https://{}:443/msp/about.php'.format(framework),
                 body='')
 
-        self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, 'POST', 'api/2.0/fo/scan'))
+        self.logger.info('Adding mocked {} endpoint {} {}'.format(framework, 'POST', 'api/2.0/fo/scan'))
         httpretty.register_uri(
             httpretty.POST, 'https://{}:443/api/2.0/fo/scan/'.format(framework),
             body=self.qualys_vuln_callback)
 
     def qualys_web_callback(self, request, uri, response_headers):
-        self.logger.debug('Simulating response for {} ({})'.format(uri, request.body))
+        self.logger.info('Simulating response for {} ({})'.format(uri, request.body))
         report_id = request.parsed_body.split('<WasScan><id>')[1].split('<')[0]
         response_body = open('{}/create_{}'.format(self.qualys_web_path, report_id)).read()
         return [200, response_headers, response_body]
@@ -78,19 +78,19 @@ class mockAPI(object):
             if filename.startswith('POST') or filename.startswith('GET'):
                 method, resource = filename.split('_', 1)
                 resource = resource.replace('_', '/')
-                self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, method, resource))
+                self.logger.info('Adding mocked {} endpoint {} {}'.format(framework, method, resource))
                 httpretty.register_uri(
                     getattr(httpretty, method), 'https://{}:443/{}'.format(framework, resource),
                     body=open('{}/{}/{}'.format(self.mock_dir, framework, filename)).read()
                 )
 
-        self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, 'POST', 'qps/rest/3.0/create/was/report'))
+        self.logger.info('Adding mocked {} endpoint {} {}'.format(framework, 'POST', 'qps/rest/3.0/create/was/report'))
         httpretty.register_uri(
             httpretty.POST, 'https://{}:443/qps/rest/3.0/create/was/report'.format(framework),
             body=self.qualys_web_callback)
 
     def openvas_callback(self, request, uri, response_headers):
-        self.logger.debug('Simulating response for {} ({})'.format(uri, request.body))
+        self.logger.info('Simulating response for {} ({})'.format(uri, request.body))
         if request.querystring['cmd'][0] in ['get_reports', 'get_report_formats']:
             response_body = open('{}/{}'.format(self.openvas_path, request.querystring['cmd'][0])).read()
 

vulnwhisp/vulnwhisp.py

@@ -32,16 +32,11 @@ class vulnWhispererBase(object):
             config=None,
             db_name='report_tracker.db',
             purge=False,
-            verbose=None,
+            verbose=False,
             debug=False,
-            username=None,
-            password=None,
             section=None,
             develop=False,
         ):
-        self.logger = logging.getLogger('vulnWhispererBase')
-        if debug:
-            self.logger.setLevel(logging.DEBUG)
 
         if self.CONFIG_SECTION is None:
                 raise Exception('Implementing class must define CONFIG_SECTION')
@@ -66,9 +61,11 @@ class vulnWhispererBase(object):
                 self.password = None
             self.write_path = self.config.get(self.CONFIG_SECTION, 'write_path')
             self.db_path = self.config.get(self.CONFIG_SECTION, 'db_path')
-            self.verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
-
 
+        self.logger = logging.getLogger('vulnWhispererBase')
+        self.logger.setLevel(logging.INFO)
+        self.logger.info('Running {} framwork'.format(self.CONFIG_SECTION))
+        self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)
 
         if self.db_name is not None:
             if self.db_path:
@@ -254,17 +251,17 @@ class vulnWhispererBase(object):
         """Map and transform common data values"""
         self.logger.info('Start common normalisation')
 
-        self.logger.info('Normalising CVSS')
+        self.logger.debug('Normalising CVSS')
         for cvss_version in ['cvss', 'cvss3']:
             if cvss_version + '_base' in df:
-                self.logger.info('Normalising {} base'.format(cvss_version))
+                self.logger.debug('Normalising {} base'.format(cvss_version))
                 # CVSS = cvss_temporal or cvss_base
                 df[cvss_version] = df[cvss_version + '_base']
                 df.loc[df[cvss_version + '_temporal'] != '', cvss_version] = df[cvss_version + '_temporal']
 
             # Combine CVSS and CVSS3 vectors
             if cvss_version + '_vector' in df and cvss_version + '_temporal_vector' in df:
-                self.logger.info('Normalising {} vector'.format(cvss_version))
+                self.logger.debug('Normalising {} vector'.format(cvss_version))
                 df[cvss_version + '_vector'] = (
                     df[[cvss_version + '_vector', cvss_version + '_temporal_vector']]
                     .apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1)
@@ -273,18 +270,17 @@ class vulnWhispererBase(object):
                 df.drop(cvss_version + '_temporal_vector', axis=1, inplace=True)
 
             if cvss_version in df:
-                self.logger.info('Normalising {} severity'.format(cvss_version))
+                self.logger.debug('Normalising {} severity'.format(cvss_version))
                 # Map CVSS to severity name
                 df.loc[df[cvss_version].astype(str) == '', cvss_version] = None
                 df[cvss_version] = df[cvss_version].astype('float')
-                # df.loc[df[cvss_version].isnull(), cvss_version + '_severity'] = 'info'
+                df.loc[df[cvss_version] == 0, cvss_version + '_severity'] = 'informational'
-                df.loc[df[cvss_version] == 0, cvss_version + '_severity'] = 'info'
                 df.loc[(df[cvss_version] > 0) & (df[cvss_version] < 3), cvss_version + '_severity'] = 'low'
                 df.loc[(df[cvss_version] >= 3) & (df[cvss_version] < 6), cvss_version + '_severity'] = 'medium'
                 df.loc[(df[cvss_version] >= 6) & (df[cvss_version] < 9), cvss_version + '_severity'] = 'high'
                 df.loc[(df[cvss_version] > 9) & (df[cvss_version].notnull()), cvss_version + '_severity'] = 'critical'
 
-        self.logger.info('Creating Unique Document ID')
+        self.logger.debug('Creating Unique Document ID')
         df['_unique'] = df.index.values
         if 'history_id' in df:
             df['_unique'] = df[['scan_id', 'history_id', '_unique']].apply(lambda x: '_'.join(x.astype(str)), axis=1)
@@ -309,20 +305,18 @@ class vulnWhispererNessus(vulnWhispererBase):
             config=None,
             db_name='report_tracker.db',
             purge=False,
-            verbose=None,
+            verbose=False,
             debug=False,
-            username=None,
-            password=None,
             profile='nessus'
     ):
         self.CONFIG_SECTION=profile
 
-        super(vulnWhispererNessus, self).__init__(config=config)
+        super(vulnWhispererNessus, self).__init__(config=config, verbose=verbose, debug=debug)
 
         self.logger = logging.getLogger('vulnWhispererNessus')
-        if debug:
+        if not verbose:
-            self.logger.setLevel(logging.DEBUG)
+            verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
-        self.port = int(self.config.get(self.CONFIG_SECTION, 'port'))
+        self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)
 
         self.develop = True
         self.purge = purge
@@ -347,7 +341,8 @@ class vulnWhispererNessus(vulnWhispererBase):
                                 password=self.password,
                                 profile=self.CONFIG_SECTION,
                                 access_key=self.access_key,
-                                secret_key=self.secret_key
+                                secret_key=self.secret_key,
+                                verbose=verbose,
                                 )
                 self.nessus_connect = True
                 self.logger.info('Connected to {} on {host}:{port}'.format(self.CONFIG_SECTION, host=self.hostname,
@@ -557,16 +552,15 @@ class vulnWhispererQualys(vulnWhispererBase):
             config=None,
             db_name='report_tracker.db',
             purge=False,
-            verbose=None,
+            verbose=False,
             debug=False,
-            username=None,
-            password=None,
         ):
 
-        super(vulnWhispererQualys, self).__init__(config=config)
+        super(vulnWhispererQualys, self).__init__(config=config, debug=debug)
         self.logger = logging.getLogger('vulnWhispererQualys')
-        if debug:
+        if not verbose:
-            self.logger.setLevel(logging.DEBUG)
+            verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
+        self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)
 
         self.qualys_scan = qualysScanReport(config=config)
         self.latest_scans = self.qualys_scan.qw.get_all_scans()
@@ -732,15 +726,14 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
             config=None,
             db_name='report_tracker.db',
             purge=False,
-            verbose=None,
+            verbose=False,
             debug=False,
-            username=None,
-            password=None,
     ):
-        super(vulnWhispererOpenVAS, self).__init__(config=config)
+        super(vulnWhispererOpenVAS, self).__init__(config=config, debug=debug)
         self.logger = logging.getLogger('vulnWhispererOpenVAS')
-        if debug:
+        if not verbose:
-            self.logger.setLevel(logging.DEBUG)
+            verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
+        self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)
 
         self.directory_check()
         self.port = int(self.config.get(self.CONFIG_SECTION, 'port'))
@@ -857,16 +850,15 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
             config=None,
             db_name='report_tracker.db',
             purge=False,
-            verbose=None,
+            verbose=False,
             debug=False,
-            username=None,
-            password=None,
         ):
 
-        super(vulnWhispererQualysVuln, self).__init__(config=config)
+        super(vulnWhispererQualysVuln, self).__init__(config=config, debug=debug)
         self.logger = logging.getLogger('vulnWhispererQualysVuln')
-        if debug:
+        if not verbose:
-            self.logger.setLevel(logging.DEBUG)
+            verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
+        self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)
 
         self.qualys_scan = qualysVulnScan(config=config)
         self.directory_check()
@@ -989,17 +981,16 @@ class vulnWhispererJIRA(vulnWhispererBase):
             config=None,
             db_name='report_tracker.db',
             purge=False,
-            verbose=None,
+            verbose=False,
             debug=False,
-            username=None,
-            password=None,
         ):
-        super(vulnWhispererJIRA, self).__init__(config=config)
+        super(vulnWhispererJIRA, self).__init__(config=config, debug=debug)
+
         self.logger = logging.getLogger('vulnWhispererJira')
-        if debug:
+        if not verbose:
-            self.logger.setLevel(logging.DEBUG)
+            verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
-        self.config_path = config
+        self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)
-        self.config = vwConfig(config)
+
         self.host_resolv_cache = {}
         self.directory_check()
 
@@ -1269,26 +1260,20 @@ class vulnWhisperer(object):
 
     def __init__(self,
                  profile=None,
-                 verbose=None,
+                 verbose=False,
-                 username=None,
+                 debug=False,
-                 password=None,
                  config=None,
                  source=None,
                  scanname=None):
 
         self.logger = logging.getLogger('vulnWhisperer')
-        if verbose:
+        self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)
-            self.logger.setLevel(logging.DEBUG)
         self.profile = profile
         self.config = config
-        self.username = username
-        self.password = password
-        self.verbose = verbose
         self.source = source
         self.scanname = scanname
         self.exit_code = 0
 
-
     def whisper_vulnerabilities(self):
 
         if self.profile == 'nessus':