From f441f4f992d30c7c2600916a6549ba21792f9f87 Mon Sep 17 00:00:00 2001 From: pemontto Date: Thu, 2 May 2019 18:04:06 +0100 Subject: [PATCH] fix logging and remove username/password --- bin/vuln_whisperer | 24 +++----- configs/frameworks_example.ini | 10 ++-- configs/test.ini | 82 +++++++++++++------------- vulnwhisp/frameworks/nessus.py | 5 +- vulnwhisp/test/mock.py | 16 +++--- vulnwhisp/vulnwhisp.py | 101 ++++++++++++++------------------- 6 files changed, 106 insertions(+), 132 deletions(-) diff --git a/bin/vuln_whisperer b/bin/vuln_whisperer index 37e0abf..44c401e 100644 --- a/bin/vuln_whisperer +++ b/bin/vuln_whisperer @@ -32,12 +32,8 @@ def main(): help='JIRA required only! Source scanner to report') parser.add_argument('-n', '--scanname', dest='scanname', required=False, help='JIRA required only! Scan name from scan to report') - parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=True, + parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', help='Prints status out to screen (defaults to True)') - parser.add_argument('-u', '--username', dest='username', required=False, default=None, - help='The NESSUS username', type=lambda x: x.strip()) - parser.add_argument('-p', '--password', dest='password', required=False, default=None, - help='The NESSUS password', type=lambda x: x.strip()) parser.add_argument('-F', '--fancy', action='store_true', help='Enable colourful logging output') parser.add_argument('-d', '--debug', action='store_true', @@ -51,14 +47,14 @@ def main(): # First setup logging logging.basicConfig( stream=sys.stdout, - #format only applies when not using -F flag for colouring + # format only applies when not using -F flag for colouring format='%(levelname)s:%(name)s:%(funcName)s:%(message)s', - level=logging.DEBUG if args.debug else logging.INFO + level=logging.DEBUG if args.debug else logging.INFO if args.verbose else logging.WARNING ) logger = logging.getLogger() # we set up the logger to log as well to file fh = logging.FileHandler('vulnwhisperer.log') - fh.setLevel(logging.DEBUG if args.debug else logging.INFO) + fh.setLevel(logging.DEBUG if args.debug else logging.INFO if args.verbose else logging.WARNING) fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s - %(funcName)s:%(message)s", "%Y-%m-%d %H:%M:%S")) logger.addHandler(fh) @@ -87,8 +83,7 @@ def main(): vw = vulnWhisperer(config=args.config, profile=section, verbose=args.verbose, - username=args.username, - password=args.password, + debug=args.debug, source=args.source, scanname=args.scanname) exit_code += vw.whisper_vulnerabilities() @@ -97,8 +92,7 @@ def main(): vw = vulnWhisperer(config=args.config, profile=args.section, verbose=args.verbose, - username=args.username, - password=args.password, + debug=args.debug, source=args.source, scanname=args.scanname) exit_code += vw.whisper_vulnerabilities() @@ -107,10 +101,8 @@ def main(): sys.exit(exit_code) except Exception as e: - if args.verbose: - # this will remain a print since we are in the main binary - logger.error('{}'.format(str(e))) - print('ERROR: {error}'.format(error=e)) + logger.error('{}'.format(str(e))) + print('ERROR: {error}'.format(error=e)) # TODO: fix this to NOT be exit 2 unless in error close_logging_handlers(logger) sys.exit(2) diff --git a/configs/frameworks_example.ini b/configs/frameworks_example.ini index 61a8af5..e1a2eb1 100755 --- a/configs/frameworks_example.ini +++ b/configs/frameworks_example.ini @@ -9,7 +9,7 @@ password=nessus_password write_path=/opt/VulnWhisperer/data/nessus/ db_path=/opt/VulnWhisperer/data/database trash=false -verbose=true +verbose=false [tenable] enabled=true @@ -22,7 +22,7 @@ password=tenable.io_password write_path=/opt/VulnWhisperer/data/tenable/ db_path=/opt/VulnWhisperer/data/database trash=false -verbose=true +verbose=false [qualys_web] #Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API @@ -48,7 +48,7 @@ username = exampleuser password = examplepass write_path=/opt/VulnWhisperer/data/qualys_vuln/ db_path=/opt/VulnWhisperer/data/database -verbose=true +verbose=false [detectify] #Reference https://developer.detectify.com/ @@ -70,7 +70,7 @@ username = exampleuser password = examplepass write_path=/opt/VulnWhisperer/data/openvas/ db_path=/opt/VulnWhisperer/data/database -verbose=true +verbose=false [jira] enabled = false @@ -88,7 +88,7 @@ dns_resolv = False #scan_name = Test Scan #jira_project = PROJECT ; if multiple components, separate by "," = None -#components = +#components = ; minimum criticality to report (low, medium, high or critical) = None #min_critical_to_report = high diff --git a/configs/test.ini b/configs/test.ini index ed73b36..ff5abb2 100755 --- a/configs/test.ini +++ b/configs/test.ini @@ -9,7 +9,7 @@ password=nessus_password write_path=/opt/VulnWhisperer/data/nessus/ db_path=/opt/VulnWhisperer/data/database trash=false -verbose=true +verbose=false [tenable] enabled=true @@ -22,73 +22,73 @@ password=tenable.io_password write_path=/opt/VulnWhisperer/data/tenable/ db_path=/opt/VulnWhisperer/data/database trash=false -verbose=true +verbose=false [qualys_web] #Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API -enabled = true -hostname = qualys_web -username = exampleuser -password = examplepass +enabled=true +hostname=qualys_web +username=exampleuser +password=examplepass write_path=/opt/VulnWhisperer/data/qualys_web/ db_path=/opt/VulnWhisperer/data/database -verbose=true +verbose=false # Set the maximum number of retries each connection should attempt. #Note, this applies only to failed connections and timeouts, never to requests where the server returns a response. -max_retries = 10 +max_retries=10 # Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID. -template_id = 289109 +template_id=289109 [qualys_vuln] #Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API -enabled = true -hostname = qualys_vuln -username = exampleuser -password = examplepass +enabled=true +hostname=qualys_vuln +username=exampleuser +password=examplepass write_path=/opt/VulnWhisperer/data/qualys_vuln/ db_path=/opt/VulnWhisperer/data/database -verbose=true +verbose=false [detectify] #Reference https://developer.detectify.com/ -enabled = false -hostname = detectify +enabled=false +hostname=detectify #username variable used as apiKey -username = exampleuser +username=exampleuser #password variable used as secretKey -password = examplepass +password=examplepass write_path =/opt/VulnWhisperer/data/detectify/ -db_path = /opt/VulnWhisperer/data/database -verbose = true +db_path=/opt/VulnWhisperer/data/database +verbose=false [openvas] -enabled = true -hostname = openvas -port = 4000 -username = exampleuser -password = examplepass +enabled=true +hostname=openvas +port=4000 +username=exampleuser +password=examplepass write_path=/opt/VulnWhisperer/data/openvas/ db_path=/opt/VulnWhisperer/data/database -verbose=true +verbose=false [jira] -enabled = false -hostname = jira-host -username = username -password = password -write_path = /opt/VulnWhisperer/data/jira/ -db_path = /opt/VulnWhisperer/data/database -verbose = true -dns_resolv = False +enabled=false +hostname=jira-host +username=username +password=password +write_path=/opt/VulnWhisperer/data/jira/ +db_path=/opt/VulnWhisperer/data/database +verbose=false +dns_resolv=False #Sample jira report scan, will automatically be created for existent scans #[jira.qualys_vuln.test_scan] -#source = qualys_vuln -#scan_name = Test Scan -#jira_project = PROJECT -; if multiple components, separate by "," = None -#components = -; minimum criticality to report (low, medium, high or critical) = None -#min_critical_to_report = high +#source=qualys_vuln +#scan_name=Test Scan +#jira_project=PROJECT +; if multiple components, separate by ","=None +#components = +; minimum criticality to report (low, medium, high or critical)=None +#min_critical_to_report=high diff --git a/vulnwhisp/frameworks/nessus.py b/vulnwhisp/frameworks/nessus.py index 66b5626..1b1373e 100755 --- a/vulnwhisp/frameworks/nessus.py +++ b/vulnwhisp/frameworks/nessus.py @@ -44,8 +44,7 @@ class NessusAPI(object): def __init__(self, hostname=None, port=None, username=None, password=None, verbose=True, profile=None, access_key=None, secret_key=None): self.logger = logging.getLogger('NessusAPI') - if verbose: - self.logger.setLevel(logging.DEBUG) + self.logger.setLevel(logging.DEBUG if verbose else logging.INFO) if not all((username, password)) and not all((access_key, secret_key)): raise Exception('ERROR: Missing username, password or API keys.') @@ -165,8 +164,6 @@ class NessusAPI(object): report_status = self.request(self.EXPORT_STATUS.format(scan_id=scan_id, file_id=file_id), method='GET', json_output=True) running = report_status['status'] != 'ready' - sys.stdout.write('.') - sys.stdout.flush() if self.profile == 'tenable' or self.api_keys: content = self.request(self.EXPORT_FILE_DOWNLOAD.format(scan_id=scan_id, file_id=file_id), method='GET', download=True) else: diff --git a/vulnwhisp/test/mock.py b/vulnwhisp/test/mock.py index 3bef89e..15af41c 100644 --- a/vulnwhisp/test/mock.py +++ b/vulnwhisp/test/mock.py @@ -31,14 +31,14 @@ class mockAPI(object): for filename in self.get_files('{}/{}'.format(self.mock_dir, framework)): method, resource = filename.split('_', 1) resource = resource.replace('_', '/') - self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, method, resource)) + self.logger.info('Adding mocked {} endpoint {} {}'.format(framework, method, resource)) httpretty.register_uri( getattr(httpretty, method), 'https://{}:443/{}'.format(framework, resource), body=open('{}/{}/{}'.format(self.mock_dir, framework, filename)).read() ) def qualys_vuln_callback(self, request, uri, response_headers): - self.logger.debug('Simulating response for {} ({})'.format(uri, request.body)) + self.logger.info('Simulating response for {} ({})'.format(uri, request.body)) if 'list' in request.parsed_body['action']: return [200, response_headers, @@ -56,19 +56,19 @@ class mockAPI(object): def create_qualys_vuln_resource(self, framework): # Create health check endpoint - self.logger.debug('Adding mocked {} endpoint GET msp/about.php'.format(framework)) + self.logger.info('Adding mocked {} endpoint GET msp/about.php'.format(framework)) httpretty.register_uri( httpretty.GET, 'https://{}:443/msp/about.php'.format(framework), body='') - self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, 'POST', 'api/2.0/fo/scan')) + self.logger.info('Adding mocked {} endpoint {} {}'.format(framework, 'POST', 'api/2.0/fo/scan')) httpretty.register_uri( httpretty.POST, 'https://{}:443/api/2.0/fo/scan/'.format(framework), body=self.qualys_vuln_callback) def qualys_web_callback(self, request, uri, response_headers): - self.logger.debug('Simulating response for {} ({})'.format(uri, request.body)) + self.logger.info('Simulating response for {} ({})'.format(uri, request.body)) report_id = request.parsed_body.split('')[1].split('<')[0] response_body = open('{}/create_{}'.format(self.qualys_web_path, report_id)).read() return [200, response_headers, response_body] @@ -78,19 +78,19 @@ class mockAPI(object): if filename.startswith('POST') or filename.startswith('GET'): method, resource = filename.split('_', 1) resource = resource.replace('_', '/') - self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, method, resource)) + self.logger.info('Adding mocked {} endpoint {} {}'.format(framework, method, resource)) httpretty.register_uri( getattr(httpretty, method), 'https://{}:443/{}'.format(framework, resource), body=open('{}/{}/{}'.format(self.mock_dir, framework, filename)).read() ) - self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, 'POST', 'qps/rest/3.0/create/was/report')) + self.logger.info('Adding mocked {} endpoint {} {}'.format(framework, 'POST', 'qps/rest/3.0/create/was/report')) httpretty.register_uri( httpretty.POST, 'https://{}:443/qps/rest/3.0/create/was/report'.format(framework), body=self.qualys_web_callback) def openvas_callback(self, request, uri, response_headers): - self.logger.debug('Simulating response for {} ({})'.format(uri, request.body)) + self.logger.info('Simulating response for {} ({})'.format(uri, request.body)) if request.querystring['cmd'][0] in ['get_reports', 'get_report_formats']: response_body = open('{}/{}'.format(self.openvas_path, request.querystring['cmd'][0])).read() diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index 8f99c3c..8b04560 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -32,16 +32,11 @@ class vulnWhispererBase(object): config=None, db_name='report_tracker.db', purge=False, - verbose=None, + verbose=False, debug=False, - username=None, - password=None, section=None, develop=False, ): - self.logger = logging.getLogger('vulnWhispererBase') - if debug: - self.logger.setLevel(logging.DEBUG) if self.CONFIG_SECTION is None: raise Exception('Implementing class must define CONFIG_SECTION') @@ -66,9 +61,11 @@ class vulnWhispererBase(object): self.password = None self.write_path = self.config.get(self.CONFIG_SECTION, 'write_path') self.db_path = self.config.get(self.CONFIG_SECTION, 'db_path') - self.verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose') - + self.logger = logging.getLogger('vulnWhispererBase') + self.logger.setLevel(logging.INFO) + self.logger.info('Running {} framwork'.format(self.CONFIG_SECTION)) + self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING) if self.db_name is not None: if self.db_path: @@ -254,17 +251,17 @@ class vulnWhispererBase(object): """Map and transform common data values""" self.logger.info('Start common normalisation') - self.logger.info('Normalising CVSS') + self.logger.debug('Normalising CVSS') for cvss_version in ['cvss', 'cvss3']: if cvss_version + '_base' in df: - self.logger.info('Normalising {} base'.format(cvss_version)) + self.logger.debug('Normalising {} base'.format(cvss_version)) # CVSS = cvss_temporal or cvss_base df[cvss_version] = df[cvss_version + '_base'] df.loc[df[cvss_version + '_temporal'] != '', cvss_version] = df[cvss_version + '_temporal'] # Combine CVSS and CVSS3 vectors if cvss_version + '_vector' in df and cvss_version + '_temporal_vector' in df: - self.logger.info('Normalising {} vector'.format(cvss_version)) + self.logger.debug('Normalising {} vector'.format(cvss_version)) df[cvss_version + '_vector'] = ( df[[cvss_version + '_vector', cvss_version + '_temporal_vector']] .apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1) @@ -273,18 +270,17 @@ class vulnWhispererBase(object): df.drop(cvss_version + '_temporal_vector', axis=1, inplace=True) if cvss_version in df: - self.logger.info('Normalising {} severity'.format(cvss_version)) + self.logger.debug('Normalising {} severity'.format(cvss_version)) # Map CVSS to severity name df.loc[df[cvss_version].astype(str) == '', cvss_version] = None df[cvss_version] = df[cvss_version].astype('float') - # df.loc[df[cvss_version].isnull(), cvss_version + '_severity'] = 'info' - df.loc[df[cvss_version] == 0, cvss_version + '_severity'] = 'info' + df.loc[df[cvss_version] == 0, cvss_version + '_severity'] = 'informational' df.loc[(df[cvss_version] > 0) & (df[cvss_version] < 3), cvss_version + '_severity'] = 'low' df.loc[(df[cvss_version] >= 3) & (df[cvss_version] < 6), cvss_version + '_severity'] = 'medium' df.loc[(df[cvss_version] >= 6) & (df[cvss_version] < 9), cvss_version + '_severity'] = 'high' df.loc[(df[cvss_version] > 9) & (df[cvss_version].notnull()), cvss_version + '_severity'] = 'critical' - self.logger.info('Creating Unique Document ID') + self.logger.debug('Creating Unique Document ID') df['_unique'] = df.index.values if 'history_id' in df: df['_unique'] = df[['scan_id', 'history_id', '_unique']].apply(lambda x: '_'.join(x.astype(str)), axis=1) @@ -309,20 +305,18 @@ class vulnWhispererNessus(vulnWhispererBase): config=None, db_name='report_tracker.db', purge=False, - verbose=None, + verbose=False, debug=False, - username=None, - password=None, profile='nessus' ): self.CONFIG_SECTION=profile - super(vulnWhispererNessus, self).__init__(config=config) + super(vulnWhispererNessus, self).__init__(config=config, verbose=verbose, debug=debug) self.logger = logging.getLogger('vulnWhispererNessus') - if debug: - self.logger.setLevel(logging.DEBUG) - self.port = int(self.config.get(self.CONFIG_SECTION, 'port')) + if not verbose: + verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose') + self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING) self.develop = True self.purge = purge @@ -347,7 +341,8 @@ class vulnWhispererNessus(vulnWhispererBase): password=self.password, profile=self.CONFIG_SECTION, access_key=self.access_key, - secret_key=self.secret_key + secret_key=self.secret_key, + verbose=verbose, ) self.nessus_connect = True self.logger.info('Connected to {} on {host}:{port}'.format(self.CONFIG_SECTION, host=self.hostname, @@ -557,16 +552,15 @@ class vulnWhispererQualys(vulnWhispererBase): config=None, db_name='report_tracker.db', purge=False, - verbose=None, + verbose=False, debug=False, - username=None, - password=None, ): - super(vulnWhispererQualys, self).__init__(config=config) + super(vulnWhispererQualys, self).__init__(config=config, debug=debug) self.logger = logging.getLogger('vulnWhispererQualys') - if debug: - self.logger.setLevel(logging.DEBUG) + if not verbose: + verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose') + self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING) self.qualys_scan = qualysScanReport(config=config) self.latest_scans = self.qualys_scan.qw.get_all_scans() @@ -732,15 +726,14 @@ class vulnWhispererOpenVAS(vulnWhispererBase): config=None, db_name='report_tracker.db', purge=False, - verbose=None, + verbose=False, debug=False, - username=None, - password=None, ): - super(vulnWhispererOpenVAS, self).__init__(config=config) + super(vulnWhispererOpenVAS, self).__init__(config=config, debug=debug) self.logger = logging.getLogger('vulnWhispererOpenVAS') - if debug: - self.logger.setLevel(logging.DEBUG) + if not verbose: + verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose') + self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING) self.directory_check() self.port = int(self.config.get(self.CONFIG_SECTION, 'port')) @@ -857,16 +850,15 @@ class vulnWhispererQualysVuln(vulnWhispererBase): config=None, db_name='report_tracker.db', purge=False, - verbose=None, + verbose=False, debug=False, - username=None, - password=None, ): - super(vulnWhispererQualysVuln, self).__init__(config=config) + super(vulnWhispererQualysVuln, self).__init__(config=config, debug=debug) self.logger = logging.getLogger('vulnWhispererQualysVuln') - if debug: - self.logger.setLevel(logging.DEBUG) + if not verbose: + verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose') + self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING) self.qualys_scan = qualysVulnScan(config=config) self.directory_check() @@ -989,17 +981,16 @@ class vulnWhispererJIRA(vulnWhispererBase): config=None, db_name='report_tracker.db', purge=False, - verbose=None, + verbose=False, debug=False, - username=None, - password=None, ): - super(vulnWhispererJIRA, self).__init__(config=config) + super(vulnWhispererJIRA, self).__init__(config=config, debug=debug) + self.logger = logging.getLogger('vulnWhispererJira') - if debug: - self.logger.setLevel(logging.DEBUG) - self.config_path = config - self.config = vwConfig(config) + if not verbose: + verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose') + self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING) + self.host_resolv_cache = {} self.directory_check() @@ -1269,26 +1260,20 @@ class vulnWhisperer(object): def __init__(self, profile=None, - verbose=None, - username=None, - password=None, + verbose=False, + debug=False, config=None, source=None, scanname=None): self.logger = logging.getLogger('vulnWhisperer') - if verbose: - self.logger.setLevel(logging.DEBUG) + self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING) self.profile = profile self.config = config - self.username = username - self.password = password - self.verbose = verbose self.source = source self.scanname = scanname self.exit_code = 0 - def whisper_vulnerabilities(self): if self.profile == 'nessus':