Support tenable API keys (#176)
* support tenable API keys * more flexible config support * add nessus API key support * fix whitespace
This commit is contained in:
pemontto
@ -55,8 +55,12 @@ class vulnWhispererBase(object):
|
||||
except:
|
||||
self.enabled = False
|
||||
self.hostname = self.config.get(self.CONFIG_SECTION, 'hostname')
|
||||
self.username = self.config.get(self.CONFIG_SECTION, 'username')
|
||||
self.password = self.config.get(self.CONFIG_SECTION, 'password')
|
||||
try:
|
||||
self.username = self.config.get(self.CONFIG_SECTION, 'username')
|
||||
self.password = self.config.get(self.CONFIG_SECTION, 'password')
|
||||
except:
|
||||
self.username = None
|
||||
self.password = None
|
||||
self.write_path = self.config.get(self.CONFIG_SECTION, 'write_path')
|
||||
self.db_path = self.config.get(self.CONFIG_SECTION, 'db_path')
|
||||
self.verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
|
||||
@ -144,7 +148,7 @@ class vulnWhispererBase(object):
|
||||
|
||||
def record_insert(self, record):
|
||||
#for backwards compatibility with older versions without "reported" field
|
||||
|
||||
|
||||
try:
|
||||
#-1 to get the latest column, 1 to get the column name (old version would be "processed", new "reported")
|
||||
#TODO delete backward compatibility check after some versions
|
||||
@ -171,7 +175,7 @@ class vulnWhispererBase(object):
|
||||
return True
|
||||
except Exception as e:
|
||||
self.logger.error('Failed while setting scan with file {} as processed'.format(filename))
|
||||
|
||||
|
||||
return False
|
||||
|
||||
def retrieve_uuids(self):
|
||||
@ -200,7 +204,7 @@ class vulnWhispererBase(object):
|
||||
def get_latest_results(self, source, scan_name):
|
||||
processed = 0
|
||||
results = []
|
||||
|
||||
|
||||
try:
|
||||
self.conn.text_factory = str
|
||||
self.cur.execute('SELECT filename FROM scan_history WHERE source="{}" AND scan_name="{}" ORDER BY last_modified DESC LIMIT 1;'.format(source, scan_name))
|
||||
@ -219,10 +223,10 @@ class vulnWhispererBase(object):
|
||||
except Exception as e:
|
||||
self.logger.error("Error when getting latest results from {}.{} : {}".format(source, scan_name, e))
|
||||
return results, reported
|
||||
|
||||
|
||||
def get_scan_profiles(self):
|
||||
# Returns a list of source.scan_name elements from the database
|
||||
|
||||
|
||||
# we get the list of sources
|
||||
try:
|
||||
self.conn.text_factory = str
|
||||
@ -231,7 +235,7 @@ class vulnWhispererBase(object):
|
||||
except:
|
||||
sources = []
|
||||
self.logger.error("Process failed at executing 'SELECT DISTINCT source FROM scan_history;'")
|
||||
|
||||
|
||||
results = []
|
||||
|
||||
# we get the list of scans within each source
|
||||
@ -274,6 +278,8 @@ class vulnWhispererNessus(vulnWhispererBase):
|
||||
|
||||
self.develop = True
|
||||
self.purge = purge
|
||||
self.access_key = None
|
||||
self.secret_key = None
|
||||
|
||||
if config is not None:
|
||||
try:
|
||||
@ -283,19 +289,30 @@ class vulnWhispererNessus(vulnWhispererBase):
|
||||
'trash')
|
||||
|
||||
try:
|
||||
self.logger.info('Attempting to connect to nessus...')
|
||||
self.access_key = self.config.get(self.CONFIG_SECTION,'access_key')
|
||||
self.secret_key = self.config.get(self.CONFIG_SECTION,'secret_key')
|
||||
except:
|
||||
pass
|
||||
|
||||
try:
|
||||
self.logger.info('Attempting to connect to {}...'.format(self.CONFIG_SECTION))
|
||||
self.nessus = \
|
||||
NessusAPI(hostname=self.hostname,
|
||||
port=self.nessus_port,
|
||||
username=self.username,
|
||||
password=self.password)
|
||||
password=self.password,
|
||||
profile=self.CONFIG_SECTION,
|
||||
access_key=self.access_key,
|
||||
secret_key=self.secret_key
|
||||
)
|
||||
self.nessus_connect = True
|
||||
self.logger.info('Connected to nessus on {host}:{port}'.format(host=self.hostname,
|
||||
self.logger.info('Connected to {} on {host}:{port}'.format(self.CONFIG_SECTION, host=self.hostname,
|
||||
port=str(self.nessus_port)))
|
||||
except Exception as e:
|
||||
self.logger.error('Exception: {}'.format(str(e)))
|
||||
raise Exception(
|
||||
'Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format(
|
||||
'Could not connect to {} -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format(
|
||||
self.CONFIG_SECTION,
|
||||
config=self.config.config_in,
|
||||
e=e))
|
||||
except Exception as e:
|
||||
@ -435,7 +452,7 @@ class vulnWhispererNessus(vulnWhispererBase):
|
||||
try:
|
||||
file_req = \
|
||||
self.nessus.download_scan(scan_id=scan_id, history=history_id,
|
||||
export_format='csv', profile=self.CONFIG_SECTION)
|
||||
export_format='csv')
|
||||
except Exception as e:
|
||||
self.logger.error('Could not download {} scan {}: {}'.format(self.CONFIG_SECTION, scan_id, str(e)))
|
||||
self.exit_code += 1
|
||||
@ -556,7 +573,7 @@ class vulnWhispererQualys(vulnWhispererBase):
|
||||
self.logger = logging.getLogger('vulnWhispererQualys')
|
||||
if debug:
|
||||
self.logger.setLevel(logging.DEBUG)
|
||||
|
||||
|
||||
self.qualys_scan = qualysScanReport(config=config)
|
||||
self.latest_scans = self.qualys_scan.qw.get_all_scans()
|
||||
self.directory_check()
|
||||
@ -643,8 +660,7 @@ class vulnWhispererQualys(vulnWhispererBase):
|
||||
|
||||
if cleanup:
|
||||
self.logger.info('Removing report {} from Qualys Database'.format(generated_report_id))
|
||||
cleaning_up = \
|
||||
self.qualys_scan.qw.delete_report(generated_report_id)
|
||||
cleaning_up = self.qualys_scan.qw.delete_report(generated_report_id)
|
||||
os.remove(self.path_check(str(generated_report_id) + '.csv'))
|
||||
self.logger.info('Deleted report from local disk: {}'.format(self.path_check(str(generated_report_id))))
|
||||
else:
|
||||
@ -838,7 +854,7 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
|
||||
username=None,
|
||||
password=None,
|
||||
):
|
||||
|
||||
|
||||
super(vulnWhispererQualysVuln, self).__init__(config=config)
|
||||
self.logger = logging.getLogger('vulnWhispererQualysVuln')
|
||||
if debug:
|
||||
@ -967,8 +983,8 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
self.config_path = config
|
||||
self.config = vwConfig(config)
|
||||
self.host_resolv_cache = {}
|
||||
self.directory_check()
|
||||
|
||||
self.directory_check()
|
||||
|
||||
if config is not None:
|
||||
try:
|
||||
self.logger.info('Attempting to connect to jira...')
|
||||
@ -985,16 +1001,16 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
'Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format(
|
||||
config=self.config.config_in, e=e))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
profiles = []
|
||||
profiles = self.get_scan_profiles()
|
||||
|
||||
|
||||
if not self.config.exists_jira_profiles(profiles):
|
||||
self.config.update_jira_profiles(profiles)
|
||||
self.logger.info("Jira profiles have been created in {config}, please fill the variables before rerunning the module.".format(config=self.config_path))
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
|
||||
|
||||
def get_env_variables(self, source, scan_name):
|
||||
# function returns an array with [jira_project, jira_components, datafile_path]
|
||||
|
||||
@ -1005,32 +1021,32 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
if project == "":
|
||||
self.logger.error('JIRA project is missing on the configuration file!')
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
# check that project actually exists
|
||||
if not self.jira.project_exists(project):
|
||||
self.logger.error("JIRA project '{project}' doesn't exist!".format(project=project))
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
components = self.config.get(jira_section,'components').split(',')
|
||||
|
||||
|
||||
#cleaning empty array from ''
|
||||
if not components[0]:
|
||||
components = []
|
||||
|
||||
|
||||
min_critical = self.config.get(jira_section,'min_critical_to_report')
|
||||
if not min_critical:
|
||||
self.logger.error('"min_critical_to_report" variable on config file is empty.')
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
#datafile path
|
||||
filename, reported = self.get_latest_results(source, scan_name)
|
||||
fullpath = ""
|
||||
|
||||
|
||||
# search data files under user specified directory
|
||||
for root, dirnames, filenames in os.walk(vwConfig(self.config_path).get(source,'write_path')):
|
||||
if filename in filenames:
|
||||
fullpath = "{}/{}".format(root,filename)
|
||||
|
||||
|
||||
if reported:
|
||||
self.logger.warn('Last Scan of "{scan_name}" for source "{source}" has already been reported; will be skipped.'.format(scan_name=scan_name, source=source))
|
||||
return [False] * 5
|
||||
@ -1038,7 +1054,7 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
if not fullpath:
|
||||
self.logger.error('Scan of "{scan_name}" for source "{source}" has not been found. Please check that the scanner data files are in place.'.format(scan_name=scan_name, source=source))
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
dns_resolv = self.config.get('jira','dns_resolv')
|
||||
if dns_resolv in ('False', 'false', ''):
|
||||
dns_resolv = False
|
||||
@ -1052,22 +1068,22 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
|
||||
|
||||
def parse_nessus_vulnerabilities(self, fullpath, source, scan_name, min_critical):
|
||||
|
||||
|
||||
vulnerabilities = []
|
||||
|
||||
# we need to parse the CSV
|
||||
risks = ['none', 'low', 'medium', 'high', 'critical']
|
||||
risks = ['none', 'low', 'medium', 'high', 'critical']
|
||||
min_risk = int([i for i,x in enumerate(risks) if x == min_critical][0])
|
||||
|
||||
df = pd.read_csv(fullpath, delimiter=',')
|
||||
|
||||
|
||||
#nessus fields we want - ['Host','Protocol','Port', 'Name', 'Synopsis', 'Description', 'Solution', 'See Also']
|
||||
for index in range(len(df)):
|
||||
# filtering vulnerabilities by criticality, discarding low risk
|
||||
to_report = int([i for i,x in enumerate(risks) if x == df.loc[index]['Risk'].lower()][0])
|
||||
if to_report < min_risk:
|
||||
continue
|
||||
|
||||
|
||||
if not vulnerabilities or df.loc[index]['Name'] not in [entry['title'] for entry in vulnerabilities]:
|
||||
vuln = {}
|
||||
#vulnerabilities should have all the info for creating all JIRA labels
|
||||
@ -1081,7 +1097,7 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
vuln['ips'] = []
|
||||
vuln['ips'].append("{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], df.loc[index]['Port']))
|
||||
vuln['risk'] = df.loc[index]['Risk'].lower()
|
||||
|
||||
|
||||
# Nessus "nan" value gets automatically casted to float by python
|
||||
if not (type(df.loc[index]['See Also']) is float):
|
||||
vuln['references'] = df.loc[index]['See Also'].split("\\n")
|
||||
@ -1094,24 +1110,24 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
for vuln in vulnerabilities:
|
||||
if vuln['title'] == df.loc[index]['Name']:
|
||||
vuln['ips'].append("{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], df.loc[index]['Port']))
|
||||
|
||||
|
||||
return vulnerabilities
|
||||
|
||||
|
||||
def parse_qualys_vuln_vulnerabilities(self, fullpath, source, scan_name, min_critical, dns_resolv = False):
|
||||
#parsing of the qualys vulnerabilities schema
|
||||
#parse json
|
||||
vulnerabilities = []
|
||||
|
||||
risks = ['info', 'low', 'medium', 'high', 'critical']
|
||||
risks = ['info', 'low', 'medium', 'high', 'critical']
|
||||
# +1 as array is 0-4, but score is 1-5
|
||||
min_risk = int([i for i,x in enumerate(risks) if x == min_critical][0])+1
|
||||
|
||||
|
||||
try:
|
||||
data=[json.loads(line) for line in open(fullpath).readlines()]
|
||||
data=[json.loads(line) for line in open(fullpath).readlines()]
|
||||
except Exception as e:
|
||||
self.logger.warn("Scan has no vulnerabilities, skipping.")
|
||||
return vulnerabilities
|
||||
|
||||
|
||||
#qualys fields we want - []
|
||||
for index in range(len(data)):
|
||||
if int(data[index]['risk']) < min_risk:
|
||||
@ -1120,7 +1136,7 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
elif data[index]['type'] == 'Practice' or data[index]['type'] == 'Ig':
|
||||
self.logger.debug("Vulnerability '{vuln}' ignored, as it is 'Practice/Potential', not verified.".format(vuln=data[index]['plugin_name']))
|
||||
continue
|
||||
|
||||
|
||||
if not vulnerabilities or data[index]['plugin_name'] not in [entry['title'] for entry in vulnerabilities]:
|
||||
vuln = {}
|
||||
#vulnerabilities should have all the info for creating all JIRA labels
|
||||
@ -1133,12 +1149,12 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
vuln['solution'] = data[index]['solution'].replace('\\n',' ')
|
||||
vuln['ips'] = []
|
||||
#TODO ADDED DNS RESOLUTION FROM QUALYS! \n SEPARATORS INSTEAD OF \\n!
|
||||
|
||||
|
||||
vuln['ips'].append("{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv)))
|
||||
|
||||
#different risk system than Nessus!
|
||||
vuln['risk'] = risks[int(data[index]['risk'])-1]
|
||||
|
||||
|
||||
# Nessus "nan" value gets automatically casted to float by python
|
||||
if not (type(data[index]['vendor_reference']) is float or data[index]['vendor_reference'] == None):
|
||||
vuln['references'] = data[index]['vendor_reference'].split("\\n")
|
||||
@ -1156,8 +1172,8 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
def get_asset_fields(self, vuln, dns_resolv):
|
||||
values = {}
|
||||
values['ip'] = vuln['ip']
|
||||
values['protocol'] = vuln['protocol']
|
||||
values['port'] = vuln['port']
|
||||
values['protocol'] = vuln['protocol']
|
||||
values['port'] = vuln['port']
|
||||
values['dns'] = ''
|
||||
if dns_resolv:
|
||||
if vuln['dns']:
|
||||
@ -1207,12 +1223,12 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
#***Qualys VM parsing***
|
||||
if source == "qualys_vuln":
|
||||
vulnerabilities = self.parse_qualys_vuln_vulnerabilities(fullpath, source, scan_name, min_critical, dns_resolv)
|
||||
|
||||
|
||||
#***JIRA sync***
|
||||
if vulnerabilities:
|
||||
self.logger.info('{source} data has been successfuly parsed'.format(source=source.upper()))
|
||||
self.logger.info('Starting JIRA sync')
|
||||
|
||||
|
||||
self.jira.sync(vulnerabilities, project, components)
|
||||
else:
|
||||
self.logger.info("[{source}.{scan_name}] No vulnerabilities or vulnerabilities not parsed.".format(source=source, scan_name=scan_name))
|
||||
@ -1259,9 +1275,6 @@ class vulnWhisperer(object):
|
||||
|
||||
if self.profile == 'nessus':
|
||||
vw = vulnWhispererNessus(config=self.config,
|
||||
username=self.username,
|
||||
password=self.password,
|
||||
verbose=self.verbose,
|
||||
profile=self.profile)
|
||||
self.exit_code += vw.whisper_nessus()
|
||||
|
||||
@ -1275,16 +1288,13 @@ class vulnWhisperer(object):
|
||||
|
||||
elif self.profile == 'tenable':
|
||||
vw = vulnWhispererNessus(config=self.config,
|
||||
username=self.username,
|
||||
password=self.password,
|
||||
verbose=self.verbose,
|
||||
profile=self.profile)
|
||||
self.exit_code += vw.whisper_nessus()
|
||||
|
||||
elif self.profile == 'qualys_vuln':
|
||||
vw = vulnWhispererQualysVuln(config=self.config)
|
||||
self.exit_code += vw.process_vuln_scans()
|
||||
|
||||
|
||||
elif self.profile == 'jira':
|
||||
#first we check config fields are created, otherwise we create them
|
||||
vw = vulnWhispererJIRA(config=self.config)
|
||||
|
||||
Reference in New Issue
Block a user