fix logging and remove username/password

This commit is contained in:
pemontto
2019-05-02 18:04:06 +01:00
parent 5df4d127ca
commit f441f4f992
6 changed files with 106 additions and 132 deletions

View File

@ -32,12 +32,8 @@ def main():
help='JIRA required only! Source scanner to report')
parser.add_argument('-n', '--scanname', dest='scanname', required=False,
help='JIRA required only! Scan name from scan to report')
parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=True,
parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
help='Prints status out to screen (defaults to True)')
parser.add_argument('-u', '--username', dest='username', required=False, default=None,
help='The NESSUS username', type=lambda x: x.strip())
parser.add_argument('-p', '--password', dest='password', required=False, default=None,
help='The NESSUS password', type=lambda x: x.strip())
parser.add_argument('-F', '--fancy', action='store_true',
help='Enable colourful logging output')
parser.add_argument('-d', '--debug', action='store_true',
@ -51,14 +47,14 @@ def main():
# First setup logging
logging.basicConfig(
stream=sys.stdout,
#format only applies when not using -F flag for colouring
# format only applies when not using -F flag for colouring
format='%(levelname)s:%(name)s:%(funcName)s:%(message)s',
level=logging.DEBUG if args.debug else logging.INFO
level=logging.DEBUG if args.debug else logging.INFO if args.verbose else logging.WARNING
)
logger = logging.getLogger()
# we set up the logger to log as well to file
fh = logging.FileHandler('vulnwhisperer.log')
fh.setLevel(logging.DEBUG if args.debug else logging.INFO)
fh.setLevel(logging.DEBUG if args.debug else logging.INFO if args.verbose else logging.WARNING)
fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s - %(funcName)s:%(message)s", "%Y-%m-%d %H:%M:%S"))
logger.addHandler(fh)
@ -87,8 +83,7 @@ def main():
vw = vulnWhisperer(config=args.config,
profile=section,
verbose=args.verbose,
username=args.username,
password=args.password,
debug=args.debug,
source=args.source,
scanname=args.scanname)
exit_code += vw.whisper_vulnerabilities()
@ -97,8 +92,7 @@ def main():
vw = vulnWhisperer(config=args.config,
profile=args.section,
verbose=args.verbose,
username=args.username,
password=args.password,
debug=args.debug,
source=args.source,
scanname=args.scanname)
exit_code += vw.whisper_vulnerabilities()
@ -107,10 +101,8 @@ def main():
sys.exit(exit_code)
except Exception as e:
if args.verbose:
# this will remain a print since we are in the main binary
logger.error('{}'.format(str(e)))
print('ERROR: {error}'.format(error=e))
logger.error('{}'.format(str(e)))
print('ERROR: {error}'.format(error=e))
# TODO: fix this to NOT be exit 2 unless in error
close_logging_handlers(logger)
sys.exit(2)

View File

@ -9,7 +9,7 @@ password=nessus_password
write_path=/opt/VulnWhisperer/data/nessus/
db_path=/opt/VulnWhisperer/data/database
trash=false
verbose=true
verbose=false
[tenable]
enabled=true
@ -22,7 +22,7 @@ password=tenable.io_password
write_path=/opt/VulnWhisperer/data/tenable/
db_path=/opt/VulnWhisperer/data/database
trash=false
verbose=true
verbose=false
[qualys_web]
#Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
@ -48,7 +48,7 @@ username = exampleuser
password = examplepass
write_path=/opt/VulnWhisperer/data/qualys_vuln/
db_path=/opt/VulnWhisperer/data/database
verbose=true
verbose=false
[detectify]
#Reference https://developer.detectify.com/
@ -70,7 +70,7 @@ username = exampleuser
password = examplepass
write_path=/opt/VulnWhisperer/data/openvas/
db_path=/opt/VulnWhisperer/data/database
verbose=true
verbose=false
[jira]
enabled = false

View File

@ -9,7 +9,7 @@ password=nessus_password
write_path=/opt/VulnWhisperer/data/nessus/
db_path=/opt/VulnWhisperer/data/database
trash=false
verbose=true
verbose=false
[tenable]
enabled=true
@ -22,73 +22,73 @@ password=tenable.io_password
write_path=/opt/VulnWhisperer/data/tenable/
db_path=/opt/VulnWhisperer/data/database
trash=false
verbose=true
verbose=false
[qualys_web]
#Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
enabled = true
hostname = qualys_web
username = exampleuser
password = examplepass
enabled=true
hostname=qualys_web
username=exampleuser
password=examplepass
write_path=/opt/VulnWhisperer/data/qualys_web/
db_path=/opt/VulnWhisperer/data/database
verbose=true
verbose=false
# Set the maximum number of retries each connection should attempt.
#Note, this applies only to failed connections and timeouts, never to requests where the server returns a response.
max_retries = 10
max_retries=10
# Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID.
template_id = 289109
template_id=289109
[qualys_vuln]
#Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
enabled = true
hostname = qualys_vuln
username = exampleuser
password = examplepass
enabled=true
hostname=qualys_vuln
username=exampleuser
password=examplepass
write_path=/opt/VulnWhisperer/data/qualys_vuln/
db_path=/opt/VulnWhisperer/data/database
verbose=true
verbose=false
[detectify]
#Reference https://developer.detectify.com/
enabled = false
hostname = detectify
enabled=false
hostname=detectify
#username variable used as apiKey
username = exampleuser
username=exampleuser
#password variable used as secretKey
password = examplepass
password=examplepass
write_path =/opt/VulnWhisperer/data/detectify/
db_path = /opt/VulnWhisperer/data/database
verbose = true
db_path=/opt/VulnWhisperer/data/database
verbose=false
[openvas]
enabled = true
hostname = openvas
port = 4000
username = exampleuser
password = examplepass
enabled=true
hostname=openvas
port=4000
username=exampleuser
password=examplepass
write_path=/opt/VulnWhisperer/data/openvas/
db_path=/opt/VulnWhisperer/data/database
verbose=true
verbose=false
[jira]
enabled = false
hostname = jira-host
username = username
password = password
write_path = /opt/VulnWhisperer/data/jira/
db_path = /opt/VulnWhisperer/data/database
verbose = true
dns_resolv = False
enabled=false
hostname=jira-host
username=username
password=password
write_path=/opt/VulnWhisperer/data/jira/
db_path=/opt/VulnWhisperer/data/database
verbose=false
dns_resolv=False
#Sample jira report scan, will automatically be created for existent scans
#[jira.qualys_vuln.test_scan]
#source = qualys_vuln
#scan_name = Test Scan
#jira_project = PROJECT
; if multiple components, separate by "," = None
#source=qualys_vuln
#scan_name=Test Scan
#jira_project=PROJECT
; if multiple components, separate by ","=None
#components =
; minimum criticality to report (low, medium, high or critical) = None
#min_critical_to_report = high
; minimum criticality to report (low, medium, high or critical)=None
#min_critical_to_report=high

View File

@ -44,8 +44,7 @@ class NessusAPI(object):
def __init__(self, hostname=None, port=None, username=None, password=None, verbose=True, profile=None, access_key=None, secret_key=None):
self.logger = logging.getLogger('NessusAPI')
if verbose:
self.logger.setLevel(logging.DEBUG)
self.logger.setLevel(logging.DEBUG if verbose else logging.INFO)
if not all((username, password)) and not all((access_key, secret_key)):
raise Exception('ERROR: Missing username, password or API keys.')
@ -165,8 +164,6 @@ class NessusAPI(object):
report_status = self.request(self.EXPORT_STATUS.format(scan_id=scan_id, file_id=file_id), method='GET',
json_output=True)
running = report_status['status'] != 'ready'
sys.stdout.write('.')
sys.stdout.flush()
if self.profile == 'tenable' or self.api_keys:
content = self.request(self.EXPORT_FILE_DOWNLOAD.format(scan_id=scan_id, file_id=file_id), method='GET', download=True)
else:

View File

@ -31,14 +31,14 @@ class mockAPI(object):
for filename in self.get_files('{}/{}'.format(self.mock_dir, framework)):
method, resource = filename.split('_', 1)
resource = resource.replace('_', '/')
self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, method, resource))
self.logger.info('Adding mocked {} endpoint {} {}'.format(framework, method, resource))
httpretty.register_uri(
getattr(httpretty, method), 'https://{}:443/{}'.format(framework, resource),
body=open('{}/{}/{}'.format(self.mock_dir, framework, filename)).read()
)
def qualys_vuln_callback(self, request, uri, response_headers):
self.logger.debug('Simulating response for {} ({})'.format(uri, request.body))
self.logger.info('Simulating response for {} ({})'.format(uri, request.body))
if 'list' in request.parsed_body['action']:
return [200,
response_headers,
@ -56,19 +56,19 @@ class mockAPI(object):
def create_qualys_vuln_resource(self, framework):
# Create health check endpoint
self.logger.debug('Adding mocked {} endpoint GET msp/about.php'.format(framework))
self.logger.info('Adding mocked {} endpoint GET msp/about.php'.format(framework))
httpretty.register_uri(
httpretty.GET,
'https://{}:443/msp/about.php'.format(framework),
body='')
self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, 'POST', 'api/2.0/fo/scan'))
self.logger.info('Adding mocked {} endpoint {} {}'.format(framework, 'POST', 'api/2.0/fo/scan'))
httpretty.register_uri(
httpretty.POST, 'https://{}:443/api/2.0/fo/scan/'.format(framework),
body=self.qualys_vuln_callback)
def qualys_web_callback(self, request, uri, response_headers):
self.logger.debug('Simulating response for {} ({})'.format(uri, request.body))
self.logger.info('Simulating response for {} ({})'.format(uri, request.body))
report_id = request.parsed_body.split('<WasScan><id>')[1].split('<')[0]
response_body = open('{}/create_{}'.format(self.qualys_web_path, report_id)).read()
return [200, response_headers, response_body]
@ -78,19 +78,19 @@ class mockAPI(object):
if filename.startswith('POST') or filename.startswith('GET'):
method, resource = filename.split('_', 1)
resource = resource.replace('_', '/')
self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, method, resource))
self.logger.info('Adding mocked {} endpoint {} {}'.format(framework, method, resource))
httpretty.register_uri(
getattr(httpretty, method), 'https://{}:443/{}'.format(framework, resource),
body=open('{}/{}/{}'.format(self.mock_dir, framework, filename)).read()
)
self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, 'POST', 'qps/rest/3.0/create/was/report'))
self.logger.info('Adding mocked {} endpoint {} {}'.format(framework, 'POST', 'qps/rest/3.0/create/was/report'))
httpretty.register_uri(
httpretty.POST, 'https://{}:443/qps/rest/3.0/create/was/report'.format(framework),
body=self.qualys_web_callback)
def openvas_callback(self, request, uri, response_headers):
self.logger.debug('Simulating response for {} ({})'.format(uri, request.body))
self.logger.info('Simulating response for {} ({})'.format(uri, request.body))
if request.querystring['cmd'][0] in ['get_reports', 'get_report_formats']:
response_body = open('{}/{}'.format(self.openvas_path, request.querystring['cmd'][0])).read()

View File

@ -32,16 +32,11 @@ class vulnWhispererBase(object):
config=None,
db_name='report_tracker.db',
purge=False,
verbose=None,
verbose=False,
debug=False,
username=None,
password=None,
section=None,
develop=False,
):
self.logger = logging.getLogger('vulnWhispererBase')
if debug:
self.logger.setLevel(logging.DEBUG)
if self.CONFIG_SECTION is None:
raise Exception('Implementing class must define CONFIG_SECTION')
@ -66,9 +61,11 @@ class vulnWhispererBase(object):
self.password = None
self.write_path = self.config.get(self.CONFIG_SECTION, 'write_path')
self.db_path = self.config.get(self.CONFIG_SECTION, 'db_path')
self.verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
self.logger = logging.getLogger('vulnWhispererBase')
self.logger.setLevel(logging.INFO)
self.logger.info('Running {} framwork'.format(self.CONFIG_SECTION))
self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)
if self.db_name is not None:
if self.db_path:
@ -254,17 +251,17 @@ class vulnWhispererBase(object):
"""Map and transform common data values"""
self.logger.info('Start common normalisation')
self.logger.info('Normalising CVSS')
self.logger.debug('Normalising CVSS')
for cvss_version in ['cvss', 'cvss3']:
if cvss_version + '_base' in df:
self.logger.info('Normalising {} base'.format(cvss_version))
self.logger.debug('Normalising {} base'.format(cvss_version))
# CVSS = cvss_temporal or cvss_base
df[cvss_version] = df[cvss_version + '_base']
df.loc[df[cvss_version + '_temporal'] != '', cvss_version] = df[cvss_version + '_temporal']
# Combine CVSS and CVSS3 vectors
if cvss_version + '_vector' in df and cvss_version + '_temporal_vector' in df:
self.logger.info('Normalising {} vector'.format(cvss_version))
self.logger.debug('Normalising {} vector'.format(cvss_version))
df[cvss_version + '_vector'] = (
df[[cvss_version + '_vector', cvss_version + '_temporal_vector']]
.apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1)
@ -273,18 +270,17 @@ class vulnWhispererBase(object):
df.drop(cvss_version + '_temporal_vector', axis=1, inplace=True)
if cvss_version in df:
self.logger.info('Normalising {} severity'.format(cvss_version))
self.logger.debug('Normalising {} severity'.format(cvss_version))
# Map CVSS to severity name
df.loc[df[cvss_version].astype(str) == '', cvss_version] = None
df[cvss_version] = df[cvss_version].astype('float')
# df.loc[df[cvss_version].isnull(), cvss_version + '_severity'] = 'info'
df.loc[df[cvss_version] == 0, cvss_version + '_severity'] = 'info'
df.loc[df[cvss_version] == 0, cvss_version + '_severity'] = 'informational'
df.loc[(df[cvss_version] > 0) & (df[cvss_version] < 3), cvss_version + '_severity'] = 'low'
df.loc[(df[cvss_version] >= 3) & (df[cvss_version] < 6), cvss_version + '_severity'] = 'medium'
df.loc[(df[cvss_version] >= 6) & (df[cvss_version] < 9), cvss_version + '_severity'] = 'high'
df.loc[(df[cvss_version] > 9) & (df[cvss_version].notnull()), cvss_version + '_severity'] = 'critical'
self.logger.info('Creating Unique Document ID')
self.logger.debug('Creating Unique Document ID')
df['_unique'] = df.index.values
if 'history_id' in df:
df['_unique'] = df[['scan_id', 'history_id', '_unique']].apply(lambda x: '_'.join(x.astype(str)), axis=1)
@ -309,20 +305,18 @@ class vulnWhispererNessus(vulnWhispererBase):
config=None,
db_name='report_tracker.db',
purge=False,
verbose=None,
verbose=False,
debug=False,
username=None,
password=None,
profile='nessus'
):
self.CONFIG_SECTION=profile
super(vulnWhispererNessus, self).__init__(config=config)
super(vulnWhispererNessus, self).__init__(config=config, verbose=verbose, debug=debug)
self.logger = logging.getLogger('vulnWhispererNessus')
if debug:
self.logger.setLevel(logging.DEBUG)
self.port = int(self.config.get(self.CONFIG_SECTION, 'port'))
if not verbose:
verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)
self.develop = True
self.purge = purge
@ -347,7 +341,8 @@ class vulnWhispererNessus(vulnWhispererBase):
password=self.password,
profile=self.CONFIG_SECTION,
access_key=self.access_key,
secret_key=self.secret_key
secret_key=self.secret_key,
verbose=verbose,
)
self.nessus_connect = True
self.logger.info('Connected to {} on {host}:{port}'.format(self.CONFIG_SECTION, host=self.hostname,
@ -557,16 +552,15 @@ class vulnWhispererQualys(vulnWhispererBase):
config=None,
db_name='report_tracker.db',
purge=False,
verbose=None,
verbose=False,
debug=False,
username=None,
password=None,
):
super(vulnWhispererQualys, self).__init__(config=config)
super(vulnWhispererQualys, self).__init__(config=config, debug=debug)
self.logger = logging.getLogger('vulnWhispererQualys')
if debug:
self.logger.setLevel(logging.DEBUG)
if not verbose:
verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)
self.qualys_scan = qualysScanReport(config=config)
self.latest_scans = self.qualys_scan.qw.get_all_scans()
@ -732,15 +726,14 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
config=None,
db_name='report_tracker.db',
purge=False,
verbose=None,
verbose=False,
debug=False,
username=None,
password=None,
):
super(vulnWhispererOpenVAS, self).__init__(config=config)
super(vulnWhispererOpenVAS, self).__init__(config=config, debug=debug)
self.logger = logging.getLogger('vulnWhispererOpenVAS')
if debug:
self.logger.setLevel(logging.DEBUG)
if not verbose:
verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)
self.directory_check()
self.port = int(self.config.get(self.CONFIG_SECTION, 'port'))
@ -857,16 +850,15 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
config=None,
db_name='report_tracker.db',
purge=False,
verbose=None,
verbose=False,
debug=False,
username=None,
password=None,
):
super(vulnWhispererQualysVuln, self).__init__(config=config)
super(vulnWhispererQualysVuln, self).__init__(config=config, debug=debug)
self.logger = logging.getLogger('vulnWhispererQualysVuln')
if debug:
self.logger.setLevel(logging.DEBUG)
if not verbose:
verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)
self.qualys_scan = qualysVulnScan(config=config)
self.directory_check()
@ -989,17 +981,16 @@ class vulnWhispererJIRA(vulnWhispererBase):
config=None,
db_name='report_tracker.db',
purge=False,
verbose=None,
verbose=False,
debug=False,
username=None,
password=None,
):
super(vulnWhispererJIRA, self).__init__(config=config)
super(vulnWhispererJIRA, self).__init__(config=config, debug=debug)
self.logger = logging.getLogger('vulnWhispererJira')
if debug:
self.logger.setLevel(logging.DEBUG)
self.config_path = config
self.config = vwConfig(config)
if not verbose:
verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)
self.host_resolv_cache = {}
self.directory_check()
@ -1269,26 +1260,20 @@ class vulnWhisperer(object):
def __init__(self,
profile=None,
verbose=None,
username=None,
password=None,
verbose=False,
debug=False,
config=None,
source=None,
scanname=None):
self.logger = logging.getLogger('vulnWhisperer')
if verbose:
self.logger.setLevel(logging.DEBUG)
self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)
self.profile = profile
self.config = config
self.username = username
self.password = password
self.verbose = verbose
self.source = source
self.scanname = scanname
self.exit_code = 0
def whisper_vulnerabilities(self):
if self.profile == 'nessus':