From 2997e2d2b6c932392c35aeaad903ca1c30e31892 Mon Sep 17 00:00:00 2001 From: Austin Taylor Date: Wed, 27 Dec 2017 10:38:44 -0500 Subject: [PATCH] Refactored classes to be more modular, update to ini file and submodules --- bin/vuln_whisperer | 7 +- configs/frameworks_example.ini | 8 +- deps/qualysapi/qualysapi/config.py | 7 +- deps/qualysapi/qualysapi/connector.py | 12 +- deps/qualysapi/setup.py | 6 +- logstash/1000_nessus_preprocess_nessus.conf | 141 -------- logstash/9000_output_nessus.conf | 2 +- vulnwhisp/__init__.py | 2 +- vulnwhisp/frameworks/nessus.py | 2 +- vulnwhisp/frameworks/qualys.py | 83 ++--- vulnwhisp/utils/cli.py | 1 + vulnwhisp/vulnwhisp.py | 349 +++++++++++++------- 12 files changed, 285 insertions(+), 335 deletions(-) delete mode 100755 logstash/1000_nessus_preprocess_nessus.conf diff --git a/bin/vuln_whisperer b/bin/vuln_whisperer index 87a5932..603de43 100644 --- a/bin/vuln_whisperer +++ b/bin/vuln_whisperer @@ -21,20 +21,24 @@ def main(): your vulnerability scans through aggregation of historical scans.""") parser.add_argument('-c', '--config', dest='config', required=False, default='frameworks.ini', help='Path of config file', type=lambda x: isFileValid(parser, x.strip())) + parser.add_argument('-s', '--section', dest='section', required=False, + help='Section in config') parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=True, help='Prints status out to screen (defaults to True)') parser.add_argument('-u', '--username', dest='username', required=False, default=None, type=lambda x: x.strip(), help='The NESSUS username') parser.add_argument('-p', '--password', dest='password', required=False, default=None, type=lambda x: x.strip(), help='The NESSUS password') args = parser.parse_args() + try: vw = vulnWhisperer(config=args.config, + profile=args.section, verbose=args.verbose, username=args.username, password=args.password) - vw.whisper_nessus() + vw.whisper_vulnerabilities() sys.exit(1) except Exception as e: @@ -43,6 +47,5 @@ def main(): sys.exit(2) - if __name__ == '__main__': main() \ No newline at end of file diff --git a/configs/frameworks_example.ini b/configs/frameworks_example.ini index 0c8e24d..2b8d7fa 100755 --- a/configs/frameworks_example.ini +++ b/configs/frameworks_example.ini @@ -11,16 +11,18 @@ verbose=true [qualys] #Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API +enabled = true hostname = qualysapi.qg2.apps.qualys.com username = exampleuser password = examplepass write_path=/opt/vulnwhisp/qualys/ -db_path=/opt/vulnwhisp/database/ +db_path=/opt/vulnwhisp/database verbose=true -# Set the maximum number of retries each connection should attempt. Note, this applies only to failed connections and timeouts, never to requests where the server returns a response. +# Set the maximum number of retries each connection should attempt. +#Note, this applies only to failed connections and timeouts, never to requests where the server returns a response. max_retries = 10 -template_id = = 126024 +template_id = 126024 #[proxy] ; This section is optional. Leave it out if you're not using a proxy. diff --git a/deps/qualysapi/qualysapi/config.py b/deps/qualysapi/qualysapi/config.py index 19a7950..51010e5 100644 --- a/deps/qualysapi/qualysapi/config.py +++ b/deps/qualysapi/qualysapi/config.py @@ -94,8 +94,8 @@ class QualysConnectConfig: logger.error('Report Template ID Must be set and be an integer') print('Value template ID must be an integer.') exit(1) - self._cfgparse.set('qualys', 'template_id', str(self.max_retries)) - self.max_retries = int(self.max_retries) + self._cfgparse.set('qualys', 'template_id', str(self.report_template_id)) + self.report_template_id = int(self.report_template_id) # Proxy support proxy_config = proxy_url = proxy_protocol = proxy_port = proxy_username = proxy_password = None @@ -216,3 +216,6 @@ class QualysConnectConfig: def get_hostname(self): ''' Returns hostname. ''' return self._cfgparse.get('qualys', 'hostname') + + def get_template_id(self): + return self._cfgparse.get('qualys','template_id') diff --git a/deps/qualysapi/qualysapi/connector.py b/deps/qualysapi/qualysapi/connector.py index b8e92b0..1f30879 100644 --- a/deps/qualysapi/qualysapi/connector.py +++ b/deps/qualysapi/qualysapi/connector.py @@ -9,7 +9,12 @@ and requesting data from it. """ import logging import time -import urllib.parse + +try: + from urllib.parse import urlparse +except ImportError: + from urlparse import urlparse + from collections import defaultdict import requests @@ -154,7 +159,7 @@ class QGConnector(api_actions.QGActions): if api_call_endpoint in self.api_methods['was get']: return 'get' # Post calls with no payload will result in HTTPError: 415 Client Error: Unsupported Media Type. - if not data: + if data is None: # No post data. Some calls change to GET with no post data. if api_call_endpoint in self.api_methods['was no data get']: return 'get' @@ -215,7 +220,8 @@ class QGConnector(api_actions.QGActions): data = data.lstrip('?') data = data.rstrip('&') # Convert to dictionary. - data = urllib.parse.parse_qs(data) + #data = urllib.parse.parse_qs(data) + data = urlparse(data) logger.debug('Converted:\n%s' % str(data)) elif api_version in ('am', 'was', 'am2'): if type(data) == etree._Element: diff --git a/deps/qualysapi/setup.py b/deps/qualysapi/setup.py index 264b960..e16dd66 100644 --- a/deps/qualysapi/setup.py +++ b/deps/qualysapi/setup.py @@ -1,9 +1,8 @@ #!/usr/bin/env python - from __future__ import absolute_import import os import setuptools -import sys + try: from setuptools import setup except ImportError: @@ -35,7 +34,8 @@ setup(name=__pkgname__, keywords='Qualys QualysGuard API helper network security', url='https://github.com/austin-taylor/qualysapi', package_dir={'': '.'}, - packages=setuptools.find_packages(),, + #packages=setuptools.find_packages(), + packages=['qualysapi',], # package_data={'qualysapi':['LICENSE']}, # scripts=['src/scripts/qhostinfo.py', 'src/scripts/qscanhist.py', 'src/scripts/qreports.py'], long_description=read('README.md'), diff --git a/logstash/1000_nessus_preprocess_nessus.conf b/logstash/1000_nessus_preprocess_nessus.conf deleted file mode 100755 index 0f65742..0000000 --- a/logstash/1000_nessus_preprocess_nessus.conf +++ /dev/null @@ -1,141 +0,0 @@ -# Author: Austin Taylor and Justin Henderson -# Email: email@austintaylor.io -# Last Update: 05/22/2017 -# Version 0.2 -# Description: Take in nessus reports from vulnWhisperer and pumps into logstash -#Replace "filebeathost" with the name of your computer - -input { - beats { - port => 5044 - tags => "beats" - } -} - -filter { - if [beat][hostname] == "filebeathost" { - mutate { - add_tag => ["nessus"] - } - } -} - - -filter { - if "nessus" in [tags]{ - mutate { - gsub => [ - "message", "\|\|\|", " ", - "message", "\t\t", " ", - "message", " ", " ", - "message", " ", " ", - "message", " ", " " - ] - } - - csv { - columns => ["plugin_id", "cve", "cvss", "risk", "host", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"] - separator => "," - source => "message" - } - - grok { - match => { "source" => "(?[\\\:a-z A-Z_]*\\)(?[a-z-0-9\.A-Z_\-]*)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}" } - tag_on_failure => [] - } - date { - match => [ "last_updated" , "UNIX" ] - target => "@timestamp" - remove_field => ["last_updated"] - } - if [risk] == "None" { - mutate { add_field => { "risk_number" => 0 }} - } - if [risk] == "Low" { - mutate { add_field => { "risk_number" => 1 }} - } - if [risk] == "Medium" { - mutate { add_field => { "risk_number" => 2 }} - } - if [risk] == "High" { - mutate { add_field => { "risk_number" => 3 }} - } - if [risk] == "Critical" { - mutate { add_field => { "risk_number" => 4 }} - } - if [cve] == "nan" { - mutate { remove_field => [ "cve" ] } - } - if [see_also] == "nan" { - mutate { remove_field => [ "see_also" ] } - } - if [description] == "nan" { - mutate { remove_field => [ "description" ] } - } - if [plugin_output] == "nan" { - mutate { remove_field => [ "plugin_output" ] } - } - if [synopsis] == "nan" { - mutate { remove_field => [ "synopsis" ] } - } - - mutate { - remove_field => [ "message" ] - add_field => { "risk_score" => "%{cvss}" } - } - mutate { - convert => { "risk_score" => "float" } - } - - # Compensating controls - adjust risk_score - # Adobe and Java are not allowed to run in browser unless whitelisted - # Therefore, lower score by dividing by 3 (score is subjective to risk) - if [risk_score] != 0 { - if [plugin_name] =~ "Adobe" and [risk_score] > 6 or [plugin_name] =~ "Java" and [risk_score] > 6 { - ruby { - code => "event.set('risk_score', event.get('risk_score') / 3)" - } - mutate { - add_field => { "compensating_control" => "Adobe and Flash removed from browsers unless whitelisted site." } - } - } - } - - # Add tags for reporting based on assets or criticality - if [host] == "192.168.0.1" or [host] == "192.168.0.50" or [host] =~ "^192\.168\.10\." or [host] =~ "^42.42.42." { - mutate { - add_tag => [ "critical_asset" ] - } - } - if [host] =~ "^192\.168\.[45][0-9][0-9]\.1$" or [host] =~ "^192.168\.[50]\.[0-9]{1,2}\.1$"{ - mutate { - add_tag => [ "has_hipaa_data" ] - } - } - if [host] =~ "^192\.168\.[45][0-9][0-9]\." { - mutate { - add_tag => [ "hipaa_asset" ] - } - } - if [host] =~ "^192\.168\.5\." { - mutate { - add_tag => [ "pci_asset" ] - } - if [host] =~ "^10\.0\.50\." { - mutate { - add_tag => [ "web_servers" ] - } - } - } -} -} - -output { - if "nessus" in [tags] or [type] == "nessus" { - #stdout { codec => rubydebug } - elasticsearch { - hosts => [ "localhost" ] - index => "logstash-nessus-%{+YYYY.MM}" - } - } -} diff --git a/logstash/9000_output_nessus.conf b/logstash/9000_output_nessus.conf index 83bed64..3b7d27c 100755 --- a/logstash/9000_output_nessus.conf +++ b/logstash/9000_output_nessus.conf @@ -7,7 +7,7 @@ output { if "nessus" in [tags] or [type] == "nessus" { #stdout { codec => rubydebug } elasticsearch { - hosts => "localhost:19200" + hosts => "localhost:9200" index => "logstash-nessus-%{+YYYY.MM}" } } diff --git a/vulnwhisp/__init__.py b/vulnwhisp/__init__.py index ee472b8..4e5a397 100755 --- a/vulnwhisp/__init__.py +++ b/vulnwhisp/__init__.py @@ -1 +1 @@ -from utils.cli import * \ No newline at end of file +from utils.cli import bcolors \ No newline at end of file diff --git a/vulnwhisp/frameworks/nessus.py b/vulnwhisp/frameworks/nessus.py index 11a1cfb..59ab20e 100755 --- a/vulnwhisp/frameworks/nessus.py +++ b/vulnwhisp/frameworks/nessus.py @@ -35,7 +35,7 @@ class NessusAPI(object): 'Origin': self.base, 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'en-US,en;q=0.8', - 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.96 Safari/537.36', + 'User-Agent': 'VulnWhisperer for Nessus', 'Content-Type': 'application/json', 'Accept': 'application/json, text/javascript, */*; q=0.01', 'Referer': self.base, diff --git a/vulnwhisp/frameworks/qualys.py b/vulnwhisp/frameworks/qualys.py index 5dd6c24..c2b22b3 100644 --- a/vulnwhisp/frameworks/qualys.py +++ b/vulnwhisp/frameworks/qualys.py @@ -6,6 +6,7 @@ from lxml import objectify from lxml.builder import E import xml.etree.ElementTree as ET import pandas as pd +import qualysapi import qualysapi.config as qcconf import requests from requests.packages.urllib3.exceptions import InsecureRequestWarning @@ -46,6 +47,7 @@ class qualysWhisper(object): self.template_id = self.config_parse.get_template_id() except: print 'ERROR - Could not retrieve template ID' + sys.exit(2) def request( self, @@ -370,28 +372,15 @@ class qualysWebAppReport: if 'Content' not in merged_df: merged_df['Content'] = '' - merged_df['Payload #1'] = merged_df['Payload #1' - ].apply(self.cleanser) - merged_df['Request Method #1'] = merged_df['Request Method #1' - ].apply(self.cleanser) - merged_df['Request URL #1'] = merged_df['Request URL #1' - ].apply(self.cleanser) - merged_df['Request Headers #1'] = merged_df['Request Headers #1' - ].apply(self.cleanser) - merged_df['Response #1'] = merged_df['Response #1' - ].apply(self.cleanser) - merged_df['Evidence #1'] = merged_df['Evidence #1' - ].apply(self.cleanser) + columns_to_cleanse = ['Payload #1','Request Method #1','Request URL #1', + 'Request Headers #1','Response #1','Evidence #1', + 'Description','Impact','Solution','Url','Content'] + + for col in columns_to_cleanse: + merged_df[col] = merged_df[col].apply(self.cleanser) + - merged_df['Description'] = merged_df['Description' - ].apply(self.cleanser) - merged_df['Impact'] = merged_df['Impact'].apply(self.cleanser) - merged_df['Solution'] = merged_df['Solution' - ].apply(self.cleanser) - merged_df['Url'] = merged_df['Url'].apply(self.cleanser) - merged_df['Content'] = merged_df['Content'].apply(self.cleanser) merged_df = merged_df.drop(['QID_y', 'QID_x'], axis=1) - merged_df = merged_df.rename(columns={'Id': 'QID'}) try: @@ -427,49 +416,15 @@ class qualysWebAppReport: return merged_data - def whisper_webapp(self, report_id, updated_date): - """ - report_id: App ID - updated_date: Last time scan was ran for app_id - """ - vuln_ready = None - - try: - - if 'Z' in updated_date: - updated_date = self.iso_to_epoch(updated_date) - report_name = 'qualys_web_' + str(report_id) \ - + '_{last_updated}'.format(last_updated=updated_date) \ - + '.csv' - if os.path.isfile(report_name): - print('[ACTION] - File already exist! Skipping...') - pass - else: - print('[ACTION] - Generating report for %s' % report_id) - status = self.qw.create_report(report_id) - root = objectify.fromstring(status) - if root.responseCode == 'SUCCESS': - print('[INFO] - Successfully generated report for webapp: %s' \ - % report_id) - generated_report_id = root.data.Report.id - print ('[INFO] - New Report ID: %s' \ - % generated_report_id) - vuln_ready = self.process_data(generated_report_id) - - vuln_ready.to_csv(report_name, index=False, header=True) # add when timestamp occured - print('[SUCCESS] - Report written to %s' \ - % report_name) - print('[ACTION] - Removing report %s' \ - % generated_report_id) - cleaning_up = \ - self.qw.delete_report(generated_report_id) - os.remove(str(generated_report_id) + '.csv') - print('[ACTION] - Deleted report: %s' \ - % generated_report_id) - else: - print('Could not process report ID: %s' % status) - except Exception as e: - print('[ERROR] - Could not process %s - %s' % (report_id, e)) - return vuln_ready +maxInt = sys.maxsize +decrement = True + +while decrement: + decrement = False + try: + csv.field_size_limit(maxInt) + except OverflowError: + maxInt = int(maxInt/10) + decrement = True \ No newline at end of file diff --git a/vulnwhisp/utils/cli.py b/vulnwhisp/utils/cli.py index c71abcf..046fbfa 100644 --- a/vulnwhisp/utils/cli.py +++ b/vulnwhisp/utils/cli.py @@ -12,5 +12,6 @@ class bcolors: UNDERLINE = '\033[4m' INFO = '{info}[INFO]{endc}'.format(info=OKBLUE, endc=ENDC) + ACTION = '{info}[ACTION]{endc}'.format(info=OKBLUE, endc=ENDC) SUCCESS = '{green}[SUCCESS]{endc}'.format(green=OKGREEN, endc=ENDC) FAIL = '{red}[FAIL]{endc}'.format(red=FAIL, endc=ENDC) diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index 886cf2c..24ab211 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -4,8 +4,10 @@ __author__ = 'Austin Taylor' from base.config import vwConfig from frameworks.nessus import NessusAPI +from frameworks.qualys import qualysWebAppReport from utils.cli import bcolors import pandas as pd +from lxml import objectify import sys import os import io @@ -18,6 +20,9 @@ import logging class vulnWhispererBase(object): + + CONFIG_SECTION = None + def __init__( self, config=None, @@ -27,84 +32,31 @@ class vulnWhispererBase(object): debug=False, username=None, password=None, - ): - pass + section=None, + ): -class vulnWhisperer(object): - def __init__( - self, - config=None, - db_name='report_tracker.db', - purge=False, - verbose=None, - debug=False, - username=None, - password=None, - ): - self.verbose = verbose - self.nessus_connect = False - self.develop = True + if self.CONFIG_SECTION is None: + raise Exception('Implementing class must define CONFIG_SECTION') + + self.db_name = db_name self.purge = purge if config is not None: - try: - self.config = vwConfig(config_in=config) - self.nessus_enabled = self.config.getbool('nessus', - 'enabled') + self.config = vwConfig(config_in=config) + self.enabled = self.config.get(self.CONFIG_SECTION, 'enabled') + self.hostname = self.config.get(self.CONFIG_SECTION, 'hostname') + self.username = self.config.get(self.CONFIG_SECTION, 'username') + self.password = self.config.get(self.CONFIG_SECTION, 'password') + self.write_path = self.config.get(self.CONFIG_SECTION, 'write_path') + self.db_path = self.config.get(self.CONFIG_SECTION, 'db_path') + self.verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose') - if self.nessus_enabled: - self.nessus_hostname = self.config.get('nessus', - 'hostname') - self.nessus_port = self.config.get('nessus', 'port') - if password: - self.nessus_password = password - else: - self.nessus_password = self.config.get('nessus' - , 'password') - if username: - self.nessus_username = username - else: - self.nessus_username = self.config.get('nessus' - , 'username') - - self.nessus_writepath = self.config.get('nessus', - 'write_path') - self.nessus_dbpath = self.config.get('nessus', - 'db_path') - self.nessus_trash = self.config.getbool('nessus', - 'trash') - self.verbose = self.config.getbool('nessus', - 'verbose') - - try: - self.vprint('{info} Attempting to connect to nessus...'.format(info=bcolors.INFO)) - self.nessus = \ - NessusAPI(hostname=self.nessus_hostname, - port=self.nessus_port, - username=self.nessus_username, - password=self.nessus_password) - self.nessus_connect = True - self.vprint('{success} Connected to nessus on {host}:{port}'.format(success=bcolors.SUCCESS, - host=self.nessus_hostname, - port=str(self.nessus_port))) - except Exception as e: - self.vprint(e) - raise Exception( - '{fail} Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format( - config=self.config, - fail=bcolors.FAIL, e=e)) - except Exception as e: - - self.vprint('{fail} Could not properly load your config!\nReason: {e}'.format(fail=bcolors.FAIL, - e=e)) - sys.exit(0) - - if db_name is not None: - if self.nessus_dbpath: - self.database = os.path.join(self.nessus_dbpath, + if self.db_name is not None: + if self.db_path: + self.database = os.path.join(self.db_path, db_name) else: self.database = \ @@ -137,6 +89,7 @@ class vulnWhisperer(object): 'uuid', 'processed', ] + self.init() self.uuids = self.retrieve_uuids() self.processed = 0 @@ -145,11 +98,14 @@ class vulnWhisperer(object): def vprint(self, msg): if self.verbose: - print msg + print(msg) def create_table(self): self.cur.execute( - 'CREATE TABLE IF NOT EXISTS scan_history (id INTEGER PRIMARY KEY, scan_name TEXT, scan_id INTEGER, last_modified DATE, filename TEXT, download_time DATE, record_count INTEGER, source TEXT, uuid TEXT, processed INTEGER)' + 'CREATE TABLE IF NOT EXISTS scan_history (id INTEGER PRIMARY KEY,' + ' scan_name TEXT, scan_id INTEGER, last_modified DATE, filename TEXT,' + ' download_time DATE, record_count INTEGER, source TEXT,' + ' uuid TEXT, processed INTEGER)' ) self.conn.commit() @@ -168,10 +124,83 @@ class vulnWhisperer(object): return data def path_check(self, _data): - if self.nessus_writepath: - data = self.nessus_writepath + '/' + _data + if self.write_path: + data = self.write_path + '/' + _data return data + def record_insert(self, record): + self.cur.execute('insert into scan_history({table_columns}) values (?,?,?,?,?,?,?,?,?)'.format( + table_columns=', '.join(self.table_columns)), + record) + self.conn.commit() + + def retrieve_uuids(self): + """ + Retrieves UUIDs from database and checks list to determine which files need to be processed. + :return: + """ + try: + self.conn.text_factory = str + self.cur.execute('SELECT uuid FROM scan_history where source = {config_section}'.format(config_section=self.CONFIG_SECTION)) + results = frozenset([r[0] for r in self.cur.fetchall()]) + except: + results = [] + return results + +class vulnWhispererNessus(vulnWhispererBase): + + CONFIG_SECTION = 'nessus' + + def __init__( + self, + config=None, + db_name='report_tracker.db', + purge=False, + verbose=None, + debug=False, + username=None, + password=None, + ): + super(vulnWhispererNessus, self).__init__(config=config) + + self.port = int(self.config.get(self.CONFIG_NAME, 'port')) + + self.develop = True + self.purge = purge + + if config is not None: + try: + #if self.enabled: + self.nessus_port = self.config.get(self.CONFIG_SECTION, 'port') + + self.nessus_trash = self.config.getbool(self.CONFIG_SECTION, + 'trash') + + try: + self.vprint('{info} Attempting to connect to nessus...'.format(info=bcolors.INFO)) + self.nessus = \ + NessusAPI(hostname=self.hostname, + port=self.nessus_port, + username=self.username, + password=self.password) + self.nessus_connect = True + self.vprint('{success} Connected to nessus on {host}:{port}'.format(success=bcolors.SUCCESS, + host=self.hostname, + port=str(self.nessus_port))) + except Exception as e: + self.vprint(e) + raise Exception( + '{fail} Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format( + config=self.config, + fail=bcolors.FAIL, e=e)) + except Exception as e: + + self.vprint('{fail} Could not properly load your config!\nReason: {e}'.format(fail=bcolors.FAIL, + e=e)) + sys.exit(0) + + + def scan_count(self, scans, completed=False): """ @@ -206,32 +235,15 @@ class vulnWhisperer(object): ])) scan_records.append(record.copy()) except Exception as e: - + # Generates error each time nonetype is encountered. # print(e) pass if completed: - scan_records = [s for s in scan_records if s['status'] - == 'completed'] + scan_records = [s for s in scan_records if s['status'] == 'completed'] return scan_records - def record_insert(self, record): - self.cur.execute('insert into scan_history({table_columns}) values (?,?,?,?,?,?,?,?,?)'.format( - table_columns=', '.join(self.table_columns)), - record) - self.conn.commit() - - def retrieve_uuids(self): - """ - Retrieves UUIDs from database and checks list to determine which files need to be processed. - :return: - """ - - self.conn.text_factory = str - self.cur.execute('SELECT uuid FROM scan_history') - results = frozenset([r[0] for r in self.cur.fetchall()]) - return results def whisper_nessus(self): if self.nessus_connect: @@ -299,12 +311,9 @@ class vulnWhisperer(object): if status == 'completed': file_name = '%s_%s_%s_%s.%s' % (scan_name, scan_id, history_id, norm_time, 'csv') - repls = (('\\', '_'), ('/', '_'), ('/', '_'), (' ', - '_')) - file_name = reduce(lambda a, kv: a.replace(*kv), - repls, file_name) - relative_path_name = self.path_check(folder_name - + '/' + file_name) + repls = (('\\', '_'), ('/', '_'), ('/', '_'), (' ', '_')) + file_name = reduce(lambda a, kv: a.replace(*kv), repls, file_name) + relative_path_name = self.path_check(folder_name + '/' + file_name) if os.path.isfile(relative_path_name): if self.develop: @@ -335,23 +344,14 @@ class vulnWhisperer(object): self.vprint('Processing %s/%s for scan: %s' % (scan_count, len(scan_history), scan_name)) - clean_csv['CVSS'] = clean_csv['CVSS' - ].astype(str).apply(self.cleanser) - clean_csv['CVE'] = clean_csv['CVE' - ].astype(str).apply(self.cleanser) - clean_csv['Description'] = \ - clean_csv['Description' - ].astype(str).apply(self.cleanser) + columns_to_cleanse = ['CVSS','CVE','Description','Synopsis','Solution','See Also','Plugin Output'] + + for col in columns_to_cleanse: + clean_csv[col] = clean_csv[col].astype(str).apply(self.cleanser) + clean_csv['Synopsis'] = \ clean_csv['Description' ].astype(str).apply(self.cleanser) - clean_csv['Solution'] = clean_csv['Solution' - ].astype(str).apply(self.cleanser) - clean_csv['See Also'] = clean_csv['See Also' - ].astype(str).apply(self.cleanser) - clean_csv['Plugin Output'] = \ - clean_csv['Plugin Output' - ].astype(str).apply(self.cleanser) clean_csv.to_csv(relative_path_name, index=False) record_meta = ( @@ -391,8 +391,129 @@ class vulnWhisperer(object): else: self.vprint('{fail} Failed to use scanner at {host}'.format(fail=bcolors.FAIL, - host=self.nessus_hostname + ':' + host=self.hostname + ':' + self.nessus_port)) +class vulnWhispererQualys(vulnWhispererBase): + CONFIG_SECTION = 'qualys' + + def __init__( + self, + config=None, + db_name='report_tracker.db', + purge=False, + verbose=None, + debug=False, + username=None, + password=None, + ): + super(vulnWhispererQualys, self).__init__(config=config, ) + + self.qualys_web = qualysWebAppReport(config=config) + self.latest_scans = self.qualys_web.qw.get_web_app_list() + + + def whisper_webapp(self, report_id, updated_date): + """ + report_id: App ID + updated_date: Last time scan was ran for app_id + """ + vuln_ready = None + + try: + if 'Z' in updated_date: + updated_date = self.qualys_web.iso_to_epoch(updated_date) + report_name = 'qualys_web_' + str(report_id) \ + + '_{last_updated}'.format(last_updated=updated_date) \ + + '.csv' + if os.path.isfile(report_name): + print('{action} - File already exist! Skipping...'.format(action=bcolors.ACTION)) + pass + else: + print('{action} - Generating report for %s'.format(action=bcolors.ACTION) % report_id) + status = self.qualys_web.qw.create_report(report_id) + root = objectify.fromstring(status) + if root.responseCode == 'SUCCESS': + print('{info} - Successfully generated report for webapp: %s'.format(info=bcolors.INFO) \ + % report_id) + generated_report_id = root.data.Report.id + print('{info} - New Report ID: %s'.format(info=bcolors.INFO) \ + % generated_report_id) + vuln_ready = self.qualys_web.process_data(generated_report_id) + + vuln_ready.to_csv(report_name, index=False, header=True) # add when timestamp occured + print('{success} - Report written to %s'.format(success=bcolors.SUCCESS) \ + % report_name) + print('{action} - Removing report %s'.format(action=bcolors.ACTION) \ + % generated_report_id) + cleaning_up = \ + self.qualys_web.qw.delete_report(generated_report_id) + os.remove(str(generated_report_id) + '.csv') + print('{action} - Deleted report: %s'.format(action=bcolors.ACTION) \ + % generated_report_id) + else: + print('{error} Could not process report ID: %s'.format(error=bcolors.FAIL) % status) + except Exception as e: + print('{error} - Could not process %s - %s'.format(error=bcolors.FAIL) % (report_id, e)) + return vuln_ready + + def process_web_assets(self): + counter = 0 + for app in self.latest_scans.iterrows(): + counter += 1 + print('Processing %s/%s' % (counter, len(self.latest_scans))) + self.whisper_webapp(app[1]['id'], app[1]['createdDate']) + + + + + +class vulnWhisperer(object): + + def __init__(self, + profile=None, + verbose=None, + username=None, + password=None, + config=None): + + self.profile = profile + self.config = config + self.username = username + self.password = password + self.verbose = verbose + + + def whisper_vulnerabilities(self): + + if self.profile == 'nessus': + vw = vulnWhispererNessus(config=self.config, username=self.username, password=self.password, verbose=self.verbose) + vw.whisper_nessus() + + elif self.profile == 'qualys': + vw = vulnWhispererQualys(config=self.config) + vw.process_web_assets() + + + + + + + + +''' + for f in folders: + if not os.path.exists(self.path_check(f['name'])): + if f['name'] == 'Trash' and self.nessus_trash: + os.makedirs(self.path_check(f['name'])) + elif f['name'] != 'Trash': + os.makedirs(self.path_check(f['name'])) + else: + os.path.exists(self.path_check(f['name'])) + self.vprint('{info} Directory already exist for {scan} - Skipping creation'.format( + scan=self.path_check(f['name' + ]), info=bcolors.INFO)) + +''' \ No newline at end of file