Pycharm indenting PEP8
This commit is contained in:
@ -3,6 +3,7 @@
|
||||
from __future__ import absolute_import
|
||||
from six.moves import range
|
||||
from functools import reduce
|
||||
|
||||
__author__ = 'Austin Taylor'
|
||||
|
||||
from .base.config import vwConfig
|
||||
@ -24,7 +25,6 @@ import socket
|
||||
|
||||
|
||||
class vulnWhispererBase(object):
|
||||
|
||||
CONFIG_SECTION = None
|
||||
|
||||
def __init__(
|
||||
@ -68,8 +68,6 @@ class vulnWhispererBase(object):
|
||||
self.db_path = self.config.get(self.CONFIG_SECTION, 'db_path')
|
||||
self.verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
|
||||
|
||||
|
||||
|
||||
if self.db_name is not None:
|
||||
if self.db_path:
|
||||
self.database = os.path.join(self.db_path,
|
||||
@ -91,7 +89,8 @@ class vulnWhispererBase(object):
|
||||
self.cur = self.conn.cursor()
|
||||
self.logger.info('Connected to database at {loc}'.format(loc=self.database))
|
||||
except Exception as e:
|
||||
self.logger.error('Could not connect to database at {loc}\nReason: {e} - Please ensure the path exist'.format(
|
||||
self.logger.error(
|
||||
'Could not connect to database at {loc}\nReason: {e} - Please ensure the path exist'.format(
|
||||
e=e,
|
||||
loc=self.database))
|
||||
else:
|
||||
@ -188,7 +187,8 @@ class vulnWhispererBase(object):
|
||||
"""
|
||||
try:
|
||||
self.conn.text_factory = str
|
||||
self.cur.execute('SELECT uuid FROM scan_history where source = "{config_section}"'.format(config_section=self.CONFIG_SECTION))
|
||||
self.cur.execute('SELECT uuid FROM scan_history where source = "{config_section}"'.format(
|
||||
config_section=self.CONFIG_SECTION))
|
||||
results = frozenset([r[0] for r in self.cur.fetchall()])
|
||||
except:
|
||||
results = []
|
||||
@ -211,7 +211,9 @@ class vulnWhispererBase(object):
|
||||
|
||||
try:
|
||||
self.conn.text_factory = str
|
||||
self.cur.execute('SELECT filename FROM scan_history WHERE source="{}" AND scan_name="{}" ORDER BY last_modified DESC LIMIT 1;'.format(source, scan_name))
|
||||
self.cur.execute(
|
||||
'SELECT filename FROM scan_history WHERE source="{}" AND scan_name="{}" ORDER BY last_modified DESC LIMIT 1;'.format(
|
||||
source, scan_name))
|
||||
# should always return just one filename
|
||||
results = [r[0] for r in self.cur.fetchall()][0]
|
||||
|
||||
@ -219,10 +221,13 @@ class vulnWhispererBase(object):
|
||||
# TODO delete backward compatibility check after some versions
|
||||
last_column_table = self.cur.execute('PRAGMA table_info(scan_history)').fetchall()[-1][1]
|
||||
if results and last_column_table == self.table_columns[-1]:
|
||||
reported = self.cur.execute('SELECT reported FROM scan_history WHERE filename="{}"'.format(results)).fetchall()
|
||||
reported = self.cur.execute(
|
||||
'SELECT reported FROM scan_history WHERE filename="{}"'.format(results)).fetchall()
|
||||
reported = reported[0][0]
|
||||
if reported:
|
||||
self.logger.debug("Last downloaded scan from source {source} scan_name {scan_name} has already been reported".format(source=source, scan_name=scan_name))
|
||||
self.logger.debug(
|
||||
"Last downloaded scan from source {source} scan_name {scan_name} has already been reported".format(
|
||||
source=source, scan_name=scan_name))
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error("Error when getting latest results from {}.{} : {}".format(source, scan_name, e))
|
||||
@ -257,8 +262,8 @@ class vulnWhispererBase(object):
|
||||
|
||||
return results
|
||||
|
||||
class vulnWhispererNessus(vulnWhispererBase):
|
||||
|
||||
class vulnWhispererNessus(vulnWhispererBase):
|
||||
CONFIG_SECTION = None
|
||||
|
||||
def __init__(
|
||||
@ -325,8 +330,6 @@ class vulnWhispererNessus(vulnWhispererBase):
|
||||
return False
|
||||
# sys.exit(1)
|
||||
|
||||
|
||||
|
||||
def scan_count(self, scans, completed=False):
|
||||
"""
|
||||
|
||||
@ -368,7 +371,6 @@ class vulnWhispererNessus(vulnWhispererBase):
|
||||
scan_records = [s for s in scan_records if s['status'] == 'completed']
|
||||
return scan_records
|
||||
|
||||
|
||||
def whisper_nessus(self):
|
||||
if self.nessus_connect:
|
||||
scan_data = self.nessus.scans
|
||||
@ -423,7 +425,8 @@ class vulnWhispererNessus(vulnWhispererBase):
|
||||
s['uuid'],
|
||||
)
|
||||
|
||||
# TODO Create directory sync function which scans the directory for files that exist already and populates the database
|
||||
# TODO Create directory sync function which scans the directory for files that exist already and
|
||||
# populates the database
|
||||
|
||||
folder_id = s['folder_id']
|
||||
if self.CONFIG_SECTION == 'tenable':
|
||||
@ -453,22 +456,26 @@ class vulnWhispererNessus(vulnWhispererBase):
|
||||
0,
|
||||
)
|
||||
self.record_insert(record_meta)
|
||||
self.logger.info('File {filename} already exist! Updating database'.format(filename=relative_path_name))
|
||||
self.logger.info(
|
||||
'File {filename} already exist! Updating database'.format(filename=relative_path_name))
|
||||
else:
|
||||
try:
|
||||
file_req = \
|
||||
self.nessus.download_scan(scan_id=scan_id, history=history_id,
|
||||
export_format='csv')
|
||||
except Exception as e:
|
||||
self.logger.error('Could not download {} scan {}: {}'.format(self.CONFIG_SECTION, scan_id, str(e)))
|
||||
self.logger.error(
|
||||
'Could not download {} scan {}: {}'.format(self.CONFIG_SECTION, scan_id, str(e)))
|
||||
self.exit_code += 1
|
||||
continue
|
||||
|
||||
clean_csv = \
|
||||
pd.read_csv(io.StringIO(file_req.decode('utf-8')))
|
||||
if len(clean_csv) > 2:
|
||||
self.logger.info('Processing {}/{} for scan: {}'.format(scan_count, len(scan_list), scan_name.encode('utf8')))
|
||||
columns_to_cleanse = ['CVSS','CVE','Description','Synopsis','Solution','See Also','Plugin Output', 'MAC Address']
|
||||
self.logger.info('Processing {}/{} for scan: {}'.format(scan_count, len(scan_list),
|
||||
scan_name.encode('utf8')))
|
||||
columns_to_cleanse = ['CVSS', 'CVE', 'Description', 'Synopsis', 'Solution', 'See Also',
|
||||
'Plugin Output', 'MAC Address']
|
||||
|
||||
for col in columns_to_cleanse:
|
||||
if col in clean_csv:
|
||||
@ -489,7 +496,8 @@ class vulnWhispererNessus(vulnWhispererBase):
|
||||
)
|
||||
self.record_insert(record_meta)
|
||||
self.logger.info('{filename} records written to {path} '.format(filename=clean_csv.shape[0],
|
||||
path=file_name.encode('utf8')))
|
||||
path=file_name.encode(
|
||||
'utf8')))
|
||||
else:
|
||||
record_meta = (
|
||||
scan_name,
|
||||
@ -504,17 +512,18 @@ class vulnWhispererNessus(vulnWhispererBase):
|
||||
0,
|
||||
)
|
||||
self.record_insert(record_meta)
|
||||
self.logger.warn('{} has no host available... Updating database and skipping!'.format(file_name))
|
||||
self.logger.warn(
|
||||
'{} has no host available... Updating database and skipping!'.format(file_name))
|
||||
self.conn.close()
|
||||
self.logger.info('Scan aggregation complete! Connection to database closed.')
|
||||
else:
|
||||
self.logger.error('Failed to use scanner at {host}:{port}'.format(host=self.hostname, port=self.nessus_port))
|
||||
self.logger.error(
|
||||
'Failed to use scanner at {host}:{port}'.format(host=self.hostname, port=self.nessus_port))
|
||||
self.exit_code += 1
|
||||
return self.exit_code
|
||||
|
||||
|
||||
class vulnWhispererQualys(vulnWhispererBase):
|
||||
|
||||
CONFIG_SECTION = 'qualys_web'
|
||||
COLUMN_MAPPING = {'Access Path': 'access_path',
|
||||
'Ajax Request': 'ajax_request',
|
||||
@ -564,6 +573,7 @@ class vulnWhispererQualys(vulnWhispererBase):
|
||||
'Vulnerability Category': 'vulnerability_category',
|
||||
'WASC': 'wasc',
|
||||
'Web Application Name': 'web_application_name'}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config=None,
|
||||
@ -671,7 +681,8 @@ class vulnWhispererQualys(vulnWhispererBase):
|
||||
self.logger.info('Removing report {} from Qualys Database'.format(generated_report_id))
|
||||
cleaning_up = self.qualys_scan.qw.delete_report(generated_report_id)
|
||||
os.remove(self.path_check(str(generated_report_id) + '.csv'))
|
||||
self.logger.info('Deleted report from local disk: {}'.format(self.path_check(str(generated_report_id))))
|
||||
self.logger.info(
|
||||
'Deleted report from local disk: {}'.format(self.path_check(str(generated_report_id))))
|
||||
else:
|
||||
self.logger.error('Could not process report ID: {}'.format(status))
|
||||
|
||||
@ -679,7 +690,6 @@ class vulnWhispererQualys(vulnWhispererBase):
|
||||
self.logger.error('Could not process {}: {}'.format(report_id, str(e)))
|
||||
return vuln_ready
|
||||
|
||||
|
||||
def identify_scans_to_process(self):
|
||||
if self.uuids:
|
||||
self.scans_to_process = self.latest_scans[~self.latest_scans['id'].isin(self.uuids)]
|
||||
@ -687,7 +697,6 @@ class vulnWhispererQualys(vulnWhispererBase):
|
||||
self.scans_to_process = self.latest_scans
|
||||
self.logger.info('Identified {new} scans to be processed'.format(new=len(self.scans_to_process)))
|
||||
|
||||
|
||||
def process_web_assets(self):
|
||||
counter = 0
|
||||
self.identify_scans_to_process()
|
||||
@ -768,7 +777,6 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
|
||||
if report_id:
|
||||
self.logger.info('Processing report ID: {}'.format(report_id))
|
||||
|
||||
|
||||
scan_name = report_id.replace('-', '')
|
||||
report_name = 'openvas_scan_{scan_name}_{last_updated}.{extension}'.format(scan_name=scan_name,
|
||||
last_updated=launched_date,
|
||||
@ -836,7 +844,8 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
|
||||
for scan in self.scans_to_process.iterrows():
|
||||
counter += 1
|
||||
info = scan[1]
|
||||
self.logger.info('Processing {}/{} - Report ID: {}'.format(counter, len(self.scans_to_process), info['report_ids']))
|
||||
self.logger.info(
|
||||
'Processing {}/{} - Report ID: {}'.format(counter, len(self.scans_to_process), info['report_ids']))
|
||||
self.whisper_reports(report_id=info['report_ids'],
|
||||
launched_date=info['epoch'])
|
||||
self.logger.info('Processing complete')
|
||||
@ -847,7 +856,6 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
|
||||
|
||||
|
||||
class vulnWhispererQualysVuln(vulnWhispererBase):
|
||||
|
||||
CONFIG_SECTION = 'qualys_vuln'
|
||||
COLUMN_MAPPING = {'cvss_base': 'cvss',
|
||||
'cvss3_base': 'cvss3',
|
||||
@ -948,7 +956,6 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
|
||||
self.logger.info('Report written to {}'.format(report_name))
|
||||
return self.exit_code
|
||||
|
||||
|
||||
def identify_scans_to_process(self):
|
||||
self.latest_scans = self.qualys_scan.qw.get_all_scans()
|
||||
if self.uuids:
|
||||
@ -959,7 +966,6 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
|
||||
self.scans_to_process = self.latest_scans
|
||||
self.logger.info('Identified {new} scans to be processed'.format(new=len(self.scans_to_process)))
|
||||
|
||||
|
||||
def process_vuln_scans(self):
|
||||
counter = 0
|
||||
self.identify_scans_to_process()
|
||||
@ -979,7 +985,6 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
|
||||
|
||||
|
||||
class vulnWhispererJIRA(vulnWhispererBase):
|
||||
|
||||
CONFIG_SECTION = 'jira'
|
||||
|
||||
def __init__(
|
||||
@ -1031,10 +1036,11 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
|
||||
if not self.config.exists_jira_profiles(profiles):
|
||||
self.config.update_jira_profiles(profiles)
|
||||
self.logger.info("Jira profiles have been created in {config}, please fill the variables before rerunning the module.".format(config=self.config_path))
|
||||
self.logger.info(
|
||||
"Jira profiles have been created in {config}, please fill the variables before rerunning the module.".format(
|
||||
config=self.config_path))
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
def get_env_variables(self, source, scan_name):
|
||||
# function returns an array with [jira_project, jira_components, datafile_path]
|
||||
|
||||
@ -1072,11 +1078,15 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
fullpath = "{}/{}".format(root, filename)
|
||||
|
||||
if reported:
|
||||
self.logger.warn('Last Scan of "{scan_name}" for source "{source}" has already been reported; will be skipped.'.format(scan_name=scan_name, source=source))
|
||||
self.logger.warn(
|
||||
'Last Scan of "{scan_name}" for source "{source}" has already been reported; will be skipped.'.format(
|
||||
scan_name=scan_name, source=source))
|
||||
return [False] * 5
|
||||
|
||||
if not fullpath:
|
||||
self.logger.error('Scan of "{scan_name}" for source "{source}" has not been found. Please check that the scanner data files are in place.'.format(scan_name=scan_name, source=source))
|
||||
self.logger.error(
|
||||
'Scan of "{scan_name}" for source "{source}" has not been found. Please check that the scanner data files are in place.'.format(
|
||||
scan_name=scan_name, source=source))
|
||||
sys.exit(1)
|
||||
|
||||
dns_resolv = self.config.get('jira', 'dns_resolv')
|
||||
@ -1090,7 +1100,6 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
|
||||
return project, components, fullpath, min_critical, dns_resolv
|
||||
|
||||
|
||||
def parse_nessus_vulnerabilities(self, fullpath, source, scan_name, min_critical):
|
||||
|
||||
vulnerabilities = []
|
||||
@ -1119,7 +1128,8 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
vuln['consequence'] = df.loc[index]['Description'].replace('\\n', ' ')
|
||||
vuln['solution'] = df.loc[index]['Solution'].replace('\\n', ' ')
|
||||
vuln['ips'] = []
|
||||
vuln['ips'].append("{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], df.loc[index]['Port']))
|
||||
vuln['ips'].append(
|
||||
"{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], df.loc[index]['Port']))
|
||||
vuln['risk'] = df.loc[index]['Risk'].lower()
|
||||
|
||||
# Nessus "nan" value gets automatically casted to float by python
|
||||
@ -1133,7 +1143,8 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
# grouping assets by vulnerability to open on single ticket, as each asset has its own nessus entry
|
||||
for vuln in vulnerabilities:
|
||||
if vuln['title'] == df.loc[index]['Name']:
|
||||
vuln['ips'].append("{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], df.loc[index]['Port']))
|
||||
vuln['ips'].append("{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'],
|
||||
df.loc[index]['Port']))
|
||||
|
||||
return vulnerabilities
|
||||
|
||||
@ -1158,7 +1169,8 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
continue
|
||||
|
||||
elif data[index]['type'] == 'Practice' or data[index]['type'] == 'Ig':
|
||||
self.logger.debug("Vulnerability '{vuln}' ignored, as it is 'Practice/Potential', not verified.".format(vuln=data[index]['plugin_name']))
|
||||
self.logger.debug("Vulnerability '{vuln}' ignored, as it is 'Practice/Potential', not verified.".format(
|
||||
vuln=data[index]['plugin_name']))
|
||||
continue
|
||||
|
||||
if not vulnerabilities or data[index]['plugin_name'] not in [entry['title'] for entry in vulnerabilities]:
|
||||
@ -1174,7 +1186,8 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
vuln['ips'] = []
|
||||
# TODO ADDED DNS RESOLUTION FROM QUALYS! \n SEPARATORS INSTEAD OF \\n!
|
||||
|
||||
vuln['ips'].append("{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv)))
|
||||
vuln['ips'].append(
|
||||
"{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv)))
|
||||
|
||||
# different risk system than Nessus!
|
||||
vuln['risk'] = risks[int(data[index]['risk']) - 1]
|
||||
@ -1189,7 +1202,8 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
# grouping assets by vulnerability to open on single ticket, as each asset has its own nessus entry
|
||||
for vuln in vulnerabilities:
|
||||
if vuln['title'] == data[index]['plugin_name']:
|
||||
vuln['ips'].append("{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv)))
|
||||
vuln['ips'].append(
|
||||
"{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv)))
|
||||
|
||||
return vulnerabilities
|
||||
|
||||
@ -1229,14 +1243,16 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
|
||||
return 0
|
||||
|
||||
|
||||
def jira_sync(self, source, scan_name):
|
||||
self.logger.info("Jira Sync triggered for source '{source}' and scan '{scan_name}'".format(source=source, scan_name=scan_name))
|
||||
self.logger.info("Jira Sync triggered for source '{source}' and scan '{scan_name}'".format(source=source,
|
||||
scan_name=scan_name))
|
||||
|
||||
project, components, fullpath, min_critical, dns_resolv = self.get_env_variables(source, scan_name)
|
||||
|
||||
if not project:
|
||||
self.logger.debug("Skipping scan for source '{source}' and scan '{scan_name}': vulnerabilities have already been reported.".format(source=source, scan_name=scan_name))
|
||||
self.logger.debug(
|
||||
"Skipping scan for source '{source}' and scan '{scan_name}': vulnerabilities have already been reported.".format(
|
||||
source=source, scan_name=scan_name))
|
||||
return False
|
||||
|
||||
vulnerabilities = []
|
||||
@ -1247,7 +1263,8 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
|
||||
# ***Qualys VM parsing***
|
||||
if source == "qualys_vuln":
|
||||
vulnerabilities = self.parse_qualys_vuln_vulnerabilities(fullpath, source, scan_name, min_critical, dns_resolv)
|
||||
vulnerabilities = self.parse_qualys_vuln_vulnerabilities(fullpath, source, scan_name, min_critical,
|
||||
dns_resolv)
|
||||
|
||||
# ***JIRA sync***
|
||||
if vulnerabilities:
|
||||
@ -1256,7 +1273,9 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
|
||||
self.jira.sync(vulnerabilities, project, components)
|
||||
else:
|
||||
self.logger.info("[{source}.{scan_name}] No vulnerabilities or vulnerabilities not parsed.".format(source=source, scan_name=scan_name))
|
||||
self.logger.info(
|
||||
"[{source}.{scan_name}] No vulnerabilities or vulnerabilities not parsed.".format(source=source,
|
||||
scan_name=scan_name))
|
||||
self.set_latest_scan_reported(fullpath.split("/")[-1])
|
||||
return False
|
||||
|
||||
@ -1279,10 +1298,13 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
try:
|
||||
self.jira_sync(self.config.get(scan, 'source'), self.config.get(scan, 'scan_name'))
|
||||
except Exception as e:
|
||||
self.logger.error("VulnWhisperer wasn't able to report the vulnerabilities from the '{}'s source".format(self.config.get(scan, 'source')))
|
||||
self.logger.error(
|
||||
"VulnWhisperer wasn't able to report the vulnerabilities from the '{}'s source".format(
|
||||
self.config.get(scan, 'source')))
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
class vulnWhisperer(object):
|
||||
|
||||
def __init__(self,
|
||||
@ -1306,7 +1328,6 @@ class vulnWhisperer(object):
|
||||
self.scanname = scanname
|
||||
self.exit_code = 0
|
||||
|
||||
|
||||
def whisper_vulnerabilities(self):
|
||||
|
||||
if self.profile == 'nessus':
|
||||
|
Reference in New Issue
Block a user