Support tenable API keys (#176)

* support tenable API keys

* more flexible config support

* add nessus API key support

* fix whitespace

configs/frameworks_example.ini

@@ -2,6 +2,8 @@
 enabled=true
 hostname=localhost
 port=8834
+access_key=
+secret_key=
 username=nessus_username
 password=nessus_password
 write_path=/opt/VulnWhisperer/data/nessus/
@@ -13,6 +15,8 @@ verbose=true
 enabled=true
 hostname=cloud.tenable.com
 port=443
+access_key=
+secret_key=
 username=tenable.io_username
 password=tenable.io_password
 write_path=/opt/VulnWhisperer/data/tenable/

configs/test.ini

@@ -2,6 +2,8 @@
 enabled=true
 hostname=nessus
 port=443
+access_key=
+secret_key=
 username=nessus_username
 password=nessus_password
 write_path=/opt/VulnWhisperer/data/nessus/
@@ -13,6 +15,8 @@ verbose=true
 enabled=true
 hostname=tenable
 port=443
+access_key=
+secret_key=
 username=tenable.io_username
 password=tenable.io_password
 write_path=/opt/VulnWhisperer/data/tenable/

vulnwhisp/base/config.py

@@ -31,7 +31,7 @@ class vwConfig(object):
         for section in self.config.sections():
             try:
                 if self.get(section, attribute) in check:
-                    sections.append(section)	
+                    sections.append(section)
             except:
                 self.logger.warn("Section {} has no option '{}'".format(section, attribute))
         return sections
@@ -45,7 +45,7 @@ class vwConfig(object):
         return True
 
     def update_jira_profiles(self, profiles):
-        # create JIRA profiles in the ini config file 
+        # create JIRA profiles in the ini config file
         self.logger.debug('Updating Jira profiles: {}'.format(str(profiles)))
 
         for profile in profiles:
@@ -67,7 +67,7 @@ class vwConfig(object):
                 self.config.set(section_name, 'min_critical_to_report', 'high')
                 self.config.set(section_name, '; automatically report, boolean value ')
                 self.config.set(section_name, 'autoreport', 'false')
-        
+
         # TODO: try/catch this
         # writing changes back to file
         with open(self.config_in, 'w') as configfile:

vulnwhisp/frameworks/nessus.py

@@ -24,15 +24,19 @@ class NessusAPI(object):
     EXPORT_STATUS = EXPORT + '/{file_id}/status'
     EXPORT_HISTORY = EXPORT + '?history_id={history_id}'
 
-    def __init__(self, hostname=None, port=None, username=None, password=None, verbose=True):
+    def __init__(self, hostname=None, port=None, username=None, password=None, verbose=True, profile=None, access_key=None, secret_key=None):
         self.logger = logging.getLogger('NessusAPI')
         if verbose:
             self.logger.setLevel(logging.DEBUG)
-        if username is None or password is None:
+        if not all((username, password)) and not all((access_key, secret_key)):
-            raise Exception('ERROR: Missing username or password.')
+            raise Exception('ERROR: Missing username, password or API keys.')
 
+        self.profile = profile
         self.user = username
         self.password = password
+        self.api_keys = False
+        self.access_key = access_key
+        self.secret_key = secret_key
         self.base = 'https://{hostname}:{port}'.format(hostname=hostname, port=port)
         self.verbose = verbose
 
@@ -52,7 +56,13 @@ class NessusAPI(object):
             'X-Cookie': None
         }
 
-        self.login()
+        if all((self.access_key, self.secret_key)):
+            self.logger.debug('Using {} API keys'.format(self.profile))
+            self.api_keys = True
+            self.session.headers['X-ApiKeys'] = 'accessKey={}; secretKey={}'.format(self.access_key, self.secret_key)
+        else:
+            self.login()
+
         self.scans = self.get_scans()
         self.scan_ids = self.get_scan_ids()
 
@@ -67,7 +77,7 @@ class NessusAPI(object):
     def request(self, url, data=None, headers=None, method='POST', download=False, json_output=False):
         timeout = 0
         success = False
-        
+
         method = method.lower()
         url = self.base + url
         self.logger.debug('Requesting to url {}'.format(url))
@@ -78,8 +88,10 @@ class NessusAPI(object):
                 if url == self.base + self.SESSION:
                     break
                 try:
-                    self.login()
                     timeout += 1
+                    if self.api_keys:
+                        continue
+                    self.login()
                     self.logger.info('Token refreshed')
                 except Exception as e:
                     self.logger.error('Could not refresh token\nReason: {}'.format(str(e)))
@@ -114,7 +126,7 @@ class NessusAPI(object):
         data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json_output=True)
         return data['history']
 
-    def download_scan(self, scan_id=None, history=None, export_format="", profile=""):
+    def download_scan(self, scan_id=None, history=None, export_format=""):
         running = True
         counter = 0
 
@@ -127,7 +139,8 @@ class NessusAPI(object):
         req = self.request(query, data=json.dumps(data), method='POST', json_output=True)
         try:
             file_id = req['file']
-            token_id = req['token'] if 'token' in req else req['temp_token']
+            if self.profile == 'nessus':
+                token_id = req['token'] if 'token' in req else req['temp_token']
         except Exception as e:
             self.logger.error('{}'.format(str(e)))
         self.logger.info('Download for file id {}'.format(str(file_id)))
@@ -143,7 +156,7 @@ class NessusAPI(object):
             if counter % 60 == 0:
                 self.logger.info("Completed: {}".format(counter))
         self.logger.info("Done: {}".format(counter))
-        if profile == 'tenable':
+        if self.profile == 'tenable' or self.api_keys:
             content = self.request(self.EXPORT_FILE_DOWNLOAD.format(scan_id=scan_id, file_id=file_id), method='GET', download=True)
         else:
             content = self.request(self.EXPORT_TOKEN_DOWNLOAD.format(token_id=token_id), method='GET', download=True)

vulnwhisp/frameworks/qualys_web.py

@@ -428,7 +428,7 @@ class qualysScanReport:
 
         merged_df = merged_df.drop(['QID_y', 'QID_x'], axis=1)
         merged_df = merged_df.rename(columns={'Id': 'QID'})
-        
+
         merged_df = merged_df.assign(**df_dict['SCAN_META'].to_dict(orient='records')[0])
 
         merged_df = pd.merge(merged_df, df_dict['CATEGORY_HEADER'], how='left', left_on=['Category', 'Severity Level'],

vulnwhisp/test/mock.py

@@ -35,13 +35,13 @@ class mockAPI(object):
         elif 'fetch' in request.parsed_body['action']:
             try:
                 response_body = open('{}/{}'.format(
-                                      self.qualys_vuln_path, 
+                                      self.qualys_vuln_path,
                                       request.parsed_body['scan_ref'][0].replace('/', '_'))
                                     ).read()
             except:
                 # Can't find the file, just send an empty response
                 response_body = ''
-        return [200, response_headers, response_body] 
+        return [200, response_headers, response_body]
 
     def create_nessus_resource(self, framework):
         for filename in self.get_files('{}/{}'.format(self.mock_dir, framework)):
@@ -60,7 +60,7 @@ class mockAPI(object):
                 httpretty.GET,
                 'https://{}:443/{}'.format(framework, 'msp/about.php'),
                 body='')
-        
+
         self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, 'POST', 'api/2.0/fo/scan'))
         httpretty.register_uri(
             httpretty.POST, 'https://{}:443/{}'.format(framework, 'api/2.0/fo/scan/'),

vulnwhisp/vulnwhisp.py

@@ -55,8 +55,12 @@ class vulnWhispererBase(object):
             except:
                 self.enabled = False
             self.hostname = self.config.get(self.CONFIG_SECTION, 'hostname')
-            self.username = self.config.get(self.CONFIG_SECTION, 'username')
+            try:
-            self.password = self.config.get(self.CONFIG_SECTION, 'password')
+                self.username = self.config.get(self.CONFIG_SECTION, 'username')
+                self.password = self.config.get(self.CONFIG_SECTION, 'password')
+            except:
+                self.username = None
+                self.password = None
             self.write_path = self.config.get(self.CONFIG_SECTION, 'write_path')
             self.db_path = self.config.get(self.CONFIG_SECTION, 'db_path')
             self.verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
@@ -144,7 +148,7 @@ class vulnWhispererBase(object):
 
     def record_insert(self, record):
         #for backwards compatibility with older versions without "reported" field
-        
+
         try:
             #-1 to get the latest column, 1 to get the column name (old version would be "processed", new "reported")
             #TODO delete backward compatibility check after some versions
@@ -171,7 +175,7 @@ class vulnWhispererBase(object):
             return True
         except Exception as e:
             self.logger.error('Failed while setting scan with file {} as processed'.format(filename))
-            
+
         return False
 
     def retrieve_uuids(self):
@@ -200,7 +204,7 @@ class vulnWhispererBase(object):
     def get_latest_results(self, source, scan_name):
         processed = 0
         results = []
-        
+
         try:
             self.conn.text_factory = str
             self.cur.execute('SELECT filename FROM scan_history WHERE source="{}" AND scan_name="{}" ORDER BY last_modified DESC LIMIT 1;'.format(source, scan_name))
@@ -219,10 +223,10 @@ class vulnWhispererBase(object):
         except Exception as e:
             self.logger.error("Error when getting latest results from {}.{} : {}".format(source, scan_name, e))
         return results, reported
-        
+
     def get_scan_profiles(self):
         # Returns a list of source.scan_name elements from the database
-        
+
         # we get the list of sources
         try:
             self.conn.text_factory = str
@@ -231,7 +235,7 @@ class vulnWhispererBase(object):
         except:
             sources = []
             self.logger.error("Process failed at executing 'SELECT DISTINCT source FROM scan_history;'")
-        
+
         results = []
 
         # we get the list of scans within each source
@@ -274,6 +278,8 @@ class vulnWhispererNessus(vulnWhispererBase):
 
         self.develop = True
         self.purge = purge
+        self.access_key = None
+        self.secret_key = None
 
         if config is not None:
             try:
@@ -283,19 +289,30 @@ class vulnWhispererNessus(vulnWhispererBase):
                                                         'trash')
 
                 try:
-                    self.logger.info('Attempting to connect to nessus...')
+                    self.access_key = self.config.get(self.CONFIG_SECTION,'access_key')
+                    self.secret_key = self.config.get(self.CONFIG_SECTION,'secret_key')
+                except:
+                    pass
+
+                try:
+                    self.logger.info('Attempting to connect to {}...'.format(self.CONFIG_SECTION))
                     self.nessus = \
                         NessusAPI(hostname=self.hostname,
                                   port=self.nessus_port,
                                   username=self.username,
-                                  password=self.password)
+                                  password=self.password,
+                                  profile=self.CONFIG_SECTION,
+                                  access_key=self.access_key,
+                                  secret_key=self.secret_key
+                                  )
                     self.nessus_connect = True
-                    self.logger.info('Connected to nessus on {host}:{port}'.format(host=self.hostname,
+                    self.logger.info('Connected to {} on {host}:{port}'.format(self.CONFIG_SECTION, host=self.hostname,
                                                                                    port=str(self.nessus_port)))
                 except Exception as e:
                     self.logger.error('Exception: {}'.format(str(e)))
                     raise Exception(
-                        'Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format(
+                        'Could not connect to {} -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format(
+                            self.CONFIG_SECTION,
                             config=self.config.config_in,
                             e=e))
             except Exception as e:
@@ -435,7 +452,7 @@ class vulnWhispererNessus(vulnWhispererBase):
                         try:
                             file_req = \
                                 self.nessus.download_scan(scan_id=scan_id, history=history_id,
-                                                        export_format='csv', profile=self.CONFIG_SECTION)
+                                                        export_format='csv')
                         except Exception as e:
                             self.logger.error('Could not download {} scan {}: {}'.format(self.CONFIG_SECTION, scan_id, str(e)))
                             self.exit_code += 1
@@ -556,7 +573,7 @@ class vulnWhispererQualys(vulnWhispererBase):
         self.logger = logging.getLogger('vulnWhispererQualys')
         if debug:
             self.logger.setLevel(logging.DEBUG)
-        
+
         self.qualys_scan = qualysScanReport(config=config)
         self.latest_scans = self.qualys_scan.qw.get_all_scans()
         self.directory_check()
@@ -643,8 +660,7 @@ class vulnWhispererQualys(vulnWhispererBase):
 
                     if cleanup:
                         self.logger.info('Removing report {} from Qualys Database'.format(generated_report_id))
-                        cleaning_up = \
+                        cleaning_up = self.qualys_scan.qw.delete_report(generated_report_id)
-                            self.qualys_scan.qw.delete_report(generated_report_id)
                         os.remove(self.path_check(str(generated_report_id) + '.csv'))
                         self.logger.info('Deleted report from local disk: {}'.format(self.path_check(str(generated_report_id))))
                 else:
@@ -838,7 +854,7 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
             username=None,
             password=None,
         ):
-        
+
         super(vulnWhispererQualysVuln, self).__init__(config=config)
         self.logger = logging.getLogger('vulnWhispererQualysVuln')
         if debug:
@@ -967,8 +983,8 @@ class vulnWhispererJIRA(vulnWhispererBase):
         self.config_path = config
         self.config = vwConfig(config)
         self.host_resolv_cache = {}
-        self.directory_check()   
+        self.directory_check()
-                 
+
         if config is not None:
             try:
                 self.logger.info('Attempting to connect to jira...')
@@ -985,16 +1001,16 @@ class vulnWhispererJIRA(vulnWhispererBase):
                     'Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format(
                         config=self.config.config_in, e=e))
                 sys.exit(1)
-   
+
         profiles = []
         profiles = self.get_scan_profiles()
-        
+
         if not self.config.exists_jira_profiles(profiles):
             self.config.update_jira_profiles(profiles)
             self.logger.info("Jira profiles have been created in {config}, please fill the variables before rerunning the module.".format(config=self.config_path))
             sys.exit(0)
-    
+
-   
+
     def get_env_variables(self, source, scan_name):
         # function returns an array with [jira_project, jira_components, datafile_path]
 
@@ -1005,32 +1021,32 @@ class vulnWhispererJIRA(vulnWhispererBase):
         if project == "":
             self.logger.error('JIRA project is missing on the configuration file!')
             sys.exit(0)
-        
+
         # check that project actually exists
         if not self.jira.project_exists(project):
             self.logger.error("JIRA project '{project}' doesn't exist!".format(project=project))
             sys.exit(0)
-         
+
         components = self.config.get(jira_section,'components').split(',')
-        
+
         #cleaning empty array from ''
         if not components[0]:
             components = []
-        
+
         min_critical = self.config.get(jira_section,'min_critical_to_report')
         if not min_critical:
             self.logger.error('"min_critical_to_report" variable on config file is empty.')
             sys.exit(0)
-            
+
         #datafile path
         filename, reported = self.get_latest_results(source, scan_name)
         fullpath = ""
-        
+
         # search data files under user specified directory
         for root, dirnames, filenames in os.walk(vwConfig(self.config_path).get(source,'write_path')):
             if filename in filenames:
                 fullpath = "{}/{}".format(root,filename)
-        
+
         if reported:
             self.logger.warn('Last Scan of "{scan_name}" for source "{source}" has already been reported; will be skipped.'.format(scan_name=scan_name, source=source))
             return [False] * 5
@@ -1038,7 +1054,7 @@ class vulnWhispererJIRA(vulnWhispererBase):
         if not fullpath:
             self.logger.error('Scan of "{scan_name}" for source "{source}" has not been found. Please check that the scanner data files are in place.'.format(scan_name=scan_name, source=source))
             sys.exit(1)
-    
+
         dns_resolv = self.config.get('jira','dns_resolv')
         if dns_resolv in ('False', 'false', ''):
             dns_resolv = False
@@ -1052,22 +1068,22 @@ class vulnWhispererJIRA(vulnWhispererBase):
 
 
     def parse_nessus_vulnerabilities(self, fullpath, source, scan_name, min_critical):
-        
+
         vulnerabilities = []
 
         # we need to parse the CSV
-        risks = ['none', 'low', 'medium', 'high', 'critical'] 
+        risks = ['none', 'low', 'medium', 'high', 'critical']
         min_risk = int([i for i,x in enumerate(risks) if x == min_critical][0])
 
         df = pd.read_csv(fullpath, delimiter=',')
-        
+
         #nessus fields we want - ['Host','Protocol','Port', 'Name', 'Synopsis', 'Description', 'Solution', 'See Also']
         for index in range(len(df)):
             # filtering vulnerabilities by criticality, discarding low risk
             to_report = int([i for i,x in enumerate(risks) if x == df.loc[index]['Risk'].lower()][0])
             if to_report < min_risk:
                 continue
-            
+
             if not vulnerabilities or df.loc[index]['Name'] not in [entry['title'] for entry in vulnerabilities]:
                 vuln = {}
                 #vulnerabilities should have all the info for creating all JIRA labels
@@ -1081,7 +1097,7 @@ class vulnWhispererJIRA(vulnWhispererBase):
                 vuln['ips'] = []
                 vuln['ips'].append("{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], df.loc[index]['Port']))
                 vuln['risk'] = df.loc[index]['Risk'].lower()
-                
+
                 # Nessus "nan" value gets automatically casted to float by python
                 if not (type(df.loc[index]['See Also']) is float):
                     vuln['references'] = df.loc[index]['See Also'].split("\\n")
@@ -1094,24 +1110,24 @@ class vulnWhispererJIRA(vulnWhispererBase):
                 for vuln in vulnerabilities:
                     if vuln['title'] == df.loc[index]['Name']:
                         vuln['ips'].append("{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], df.loc[index]['Port']))
-        
+
         return vulnerabilities
-    
+
     def parse_qualys_vuln_vulnerabilities(self, fullpath, source, scan_name, min_critical, dns_resolv = False):
         #parsing of the qualys vulnerabilities schema
         #parse json
         vulnerabilities = []
 
-        risks = ['info', 'low', 'medium', 'high', 'critical'] 
+        risks = ['info', 'low', 'medium', 'high', 'critical']
         # +1 as array is 0-4, but score is 1-5
         min_risk = int([i for i,x in enumerate(risks) if x == min_critical][0])+1
-        
+
         try:
-            data=[json.loads(line) for line in open(fullpath).readlines()] 
+            data=[json.loads(line) for line in open(fullpath).readlines()]
         except Exception as e:
             self.logger.warn("Scan has no vulnerabilities, skipping.")
             return vulnerabilities
-       
+
         #qualys fields we want - []
         for index in range(len(data)):
             if int(data[index]['risk']) < min_risk:
@@ -1120,7 +1136,7 @@ class vulnWhispererJIRA(vulnWhispererBase):
             elif data[index]['type'] == 'Practice' or data[index]['type'] == 'Ig':
                 self.logger.debug("Vulnerability '{vuln}' ignored, as it is 'Practice/Potential', not verified.".format(vuln=data[index]['plugin_name']))
                 continue
-            
+
             if not vulnerabilities or data[index]['plugin_name'] not in [entry['title'] for entry in vulnerabilities]:
                 vuln = {}
                 #vulnerabilities should have all the info for creating all JIRA labels
@@ -1133,12 +1149,12 @@ class vulnWhispererJIRA(vulnWhispererBase):
                 vuln['solution'] = data[index]['solution'].replace('\\n',' ')
                 vuln['ips'] = []
                 #TODO ADDED DNS RESOLUTION FROM QUALYS! \n SEPARATORS INSTEAD OF \\n!
-                
+
                 vuln['ips'].append("{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv)))
 
                 #different risk system than Nessus!
                 vuln['risk'] = risks[int(data[index]['risk'])-1]
-                
+
                 # Nessus "nan" value gets automatically casted to float by python
                 if not (type(data[index]['vendor_reference']) is float or data[index]['vendor_reference'] == None):
                     vuln['references'] = data[index]['vendor_reference'].split("\\n")
@@ -1156,8 +1172,8 @@ class vulnWhispererJIRA(vulnWhispererBase):
     def get_asset_fields(self, vuln, dns_resolv):
         values = {}
         values['ip'] = vuln['ip']
-        values['protocol'] = vuln['protocol'] 
+        values['protocol'] = vuln['protocol']
-        values['port'] = vuln['port'] 
+        values['port'] = vuln['port']
         values['dns'] = ''
         if dns_resolv:
             if vuln['dns']:
@@ -1207,12 +1223,12 @@ class vulnWhispererJIRA(vulnWhispererBase):
         #***Qualys VM parsing***
         if source == "qualys_vuln":
             vulnerabilities = self.parse_qualys_vuln_vulnerabilities(fullpath, source, scan_name, min_critical, dns_resolv)
-        
+
         #***JIRA sync***
         if vulnerabilities:
             self.logger.info('{source} data has been successfuly parsed'.format(source=source.upper()))
             self.logger.info('Starting JIRA sync')
-            
+
             self.jira.sync(vulnerabilities, project, components)
         else:
             self.logger.info("[{source}.{scan_name}] No vulnerabilities or vulnerabilities not parsed.".format(source=source, scan_name=scan_name))
@@ -1259,9 +1275,6 @@ class vulnWhisperer(object):
 
         if self.profile == 'nessus':
             vw = vulnWhispererNessus(config=self.config,
-                                     username=self.username,
-                                     password=self.password,
-                                     verbose=self.verbose,
                                      profile=self.profile)
             self.exit_code += vw.whisper_nessus()
 
@@ -1275,16 +1288,13 @@ class vulnWhisperer(object):
 
         elif self.profile == 'tenable':
             vw = vulnWhispererNessus(config=self.config,
-                                     username=self.username,
-                                     password=self.password,
-                                     verbose=self.verbose,
                                      profile=self.profile)
             self.exit_code += vw.whisper_nessus()
 
         elif self.profile == 'qualys_vuln':
             vw = vulnWhispererQualysVuln(config=self.config)
             self.exit_code += vw.process_vuln_scans()
-        
+
         elif self.profile == 'jira':
             #first we check config fields are created, otherwise we create them
             vw = vulnWhispererJIRA(config=self.config)