Refactored classes to be more modular, update to ini file and submodules
This commit is contained in:
Austin Taylor
@ -21,20 +21,24 @@ def main():
|
||||
your vulnerability scans through aggregation of historical scans.""")
|
||||
parser.add_argument('-c', '--config', dest='config', required=False, default='frameworks.ini',
|
||||
help='Path of config file', type=lambda x: isFileValid(parser, x.strip()))
|
||||
parser.add_argument('-s', '--section', dest='section', required=False,
|
||||
help='Section in config')
|
||||
parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=True,
|
||||
help='Prints status out to screen (defaults to True)')
|
||||
parser.add_argument('-u', '--username', dest='username', required=False, default=None, type=lambda x: x.strip(), help='The NESSUS username')
|
||||
parser.add_argument('-p', '--password', dest='password', required=False, default=None, type=lambda x: x.strip(), help='The NESSUS password')
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
try:
|
||||
|
||||
vw = vulnWhisperer(config=args.config,
|
||||
profile=args.section,
|
||||
verbose=args.verbose,
|
||||
username=args.username,
|
||||
password=args.password)
|
||||
|
||||
vw.whisper_nessus()
|
||||
vw.whisper_vulnerabilities()
|
||||
sys.exit(1)
|
||||
|
||||
except Exception as e:
|
||||
@ -43,6 +47,5 @@ def main():
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -11,16 +11,18 @@ verbose=true
|
||||
|
||||
[qualys]
|
||||
#Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
|
||||
enabled = true
|
||||
hostname = qualysapi.qg2.apps.qualys.com
|
||||
username = exampleuser
|
||||
password = examplepass
|
||||
write_path=/opt/vulnwhisp/qualys/
|
||||
db_path=/opt/vulnwhisp/database/
|
||||
db_path=/opt/vulnwhisp/database
|
||||
verbose=true
|
||||
|
||||
# Set the maximum number of retries each connection should attempt. Note, this applies only to failed connections and timeouts, never to requests where the server returns a response.
|
||||
# Set the maximum number of retries each connection should attempt.
|
||||
#Note, this applies only to failed connections and timeouts, never to requests where the server returns a response.
|
||||
max_retries = 10
|
||||
template_id = = 126024
|
||||
template_id = 126024
|
||||
|
||||
#[proxy]
|
||||
; This section is optional. Leave it out if you're not using a proxy.
|
||||
|
||||
7
deps/qualysapi/qualysapi/config.py
vendored
7
deps/qualysapi/qualysapi/config.py
vendored
@ -94,8 +94,8 @@ class QualysConnectConfig:
|
||||
logger.error('Report Template ID Must be set and be an integer')
|
||||
print('Value template ID must be an integer.')
|
||||
exit(1)
|
||||
self._cfgparse.set('qualys', 'template_id', str(self.max_retries))
|
||||
self.max_retries = int(self.max_retries)
|
||||
self._cfgparse.set('qualys', 'template_id', str(self.report_template_id))
|
||||
self.report_template_id = int(self.report_template_id)
|
||||
|
||||
# Proxy support
|
||||
proxy_config = proxy_url = proxy_protocol = proxy_port = proxy_username = proxy_password = None
|
||||
@ -216,3 +216,6 @@ class QualysConnectConfig:
|
||||
def get_hostname(self):
|
||||
''' Returns hostname. '''
|
||||
return self._cfgparse.get('qualys', 'hostname')
|
||||
|
||||
def get_template_id(self):
|
||||
return self._cfgparse.get('qualys','template_id')
|
||||
|
||||
12
deps/qualysapi/qualysapi/connector.py
vendored
12
deps/qualysapi/qualysapi/connector.py
vendored
@ -9,7 +9,12 @@ and requesting data from it.
|
||||
"""
|
||||
import logging
|
||||
import time
|
||||
import urllib.parse
|
||||
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
except ImportError:
|
||||
from urlparse import urlparse
|
||||
|
||||
from collections import defaultdict
|
||||
|
||||
import requests
|
||||
@ -154,7 +159,7 @@ class QGConnector(api_actions.QGActions):
|
||||
if api_call_endpoint in self.api_methods['was get']:
|
||||
return 'get'
|
||||
# Post calls with no payload will result in HTTPError: 415 Client Error: Unsupported Media Type.
|
||||
if not data:
|
||||
if data is None:
|
||||
# No post data. Some calls change to GET with no post data.
|
||||
if api_call_endpoint in self.api_methods['was no data get']:
|
||||
return 'get'
|
||||
@ -215,7 +220,8 @@ class QGConnector(api_actions.QGActions):
|
||||
data = data.lstrip('?')
|
||||
data = data.rstrip('&')
|
||||
# Convert to dictionary.
|
||||
data = urllib.parse.parse_qs(data)
|
||||
#data = urllib.parse.parse_qs(data)
|
||||
data = urlparse(data)
|
||||
logger.debug('Converted:\n%s' % str(data))
|
||||
elif api_version in ('am', 'was', 'am2'):
|
||||
if type(data) == etree._Element:
|
||||
|
||||
6
deps/qualysapi/setup.py
vendored
6
deps/qualysapi/setup.py
vendored
@ -1,9 +1,8 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
from __future__ import absolute_import
|
||||
import os
|
||||
import setuptools
|
||||
import sys
|
||||
|
||||
try:
|
||||
from setuptools import setup
|
||||
except ImportError:
|
||||
@ -35,7 +34,8 @@ setup(name=__pkgname__,
|
||||
keywords='Qualys QualysGuard API helper network security',
|
||||
url='https://github.com/austin-taylor/qualysapi',
|
||||
package_dir={'': '.'},
|
||||
packages=setuptools.find_packages(),,
|
||||
#packages=setuptools.find_packages(),
|
||||
packages=['qualysapi',],
|
||||
# package_data={'qualysapi':['LICENSE']},
|
||||
# scripts=['src/scripts/qhostinfo.py', 'src/scripts/qscanhist.py', 'src/scripts/qreports.py'],
|
||||
long_description=read('README.md'),
|
||||
|
||||
@ -1,141 +0,0 @@
|
||||
# Author: Austin Taylor and Justin Henderson
|
||||
# Email: email@austintaylor.io
|
||||
# Last Update: 05/22/2017
|
||||
# Version 0.2
|
||||
# Description: Take in nessus reports from vulnWhisperer and pumps into logstash
|
||||
#Replace "filebeathost" with the name of your computer
|
||||
|
||||
input {
|
||||
beats {
|
||||
port => 5044
|
||||
tags => "beats"
|
||||
}
|
||||
}
|
||||
|
||||
filter {
|
||||
if [beat][hostname] == "filebeathost" {
|
||||
mutate {
|
||||
add_tag => ["nessus"]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
filter {
|
||||
if "nessus" in [tags]{
|
||||
mutate {
|
||||
gsub => [
|
||||
"message", "\|\|\|", " ",
|
||||
"message", "\t\t", " ",
|
||||
"message", " ", " ",
|
||||
"message", " ", " ",
|
||||
"message", " ", " "
|
||||
]
|
||||
}
|
||||
|
||||
csv {
|
||||
columns => ["plugin_id", "cve", "cvss", "risk", "host", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"]
|
||||
separator => ","
|
||||
source => "message"
|
||||
}
|
||||
|
||||
grok {
|
||||
match => { "source" => "(?<file_path>[\\\:a-z A-Z_]*\\)(?<scan_name>[a-z-0-9\.A-Z_\-]*)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}" }
|
||||
tag_on_failure => []
|
||||
}
|
||||
date {
|
||||
match => [ "last_updated" , "UNIX" ]
|
||||
target => "@timestamp"
|
||||
remove_field => ["last_updated"]
|
||||
}
|
||||
if [risk] == "None" {
|
||||
mutate { add_field => { "risk_number" => 0 }}
|
||||
}
|
||||
if [risk] == "Low" {
|
||||
mutate { add_field => { "risk_number" => 1 }}
|
||||
}
|
||||
if [risk] == "Medium" {
|
||||
mutate { add_field => { "risk_number" => 2 }}
|
||||
}
|
||||
if [risk] == "High" {
|
||||
mutate { add_field => { "risk_number" => 3 }}
|
||||
}
|
||||
if [risk] == "Critical" {
|
||||
mutate { add_field => { "risk_number" => 4 }}
|
||||
}
|
||||
if [cve] == "nan" {
|
||||
mutate { remove_field => [ "cve" ] }
|
||||
}
|
||||
if [see_also] == "nan" {
|
||||
mutate { remove_field => [ "see_also" ] }
|
||||
}
|
||||
if [description] == "nan" {
|
||||
mutate { remove_field => [ "description" ] }
|
||||
}
|
||||
if [plugin_output] == "nan" {
|
||||
mutate { remove_field => [ "plugin_output" ] }
|
||||
}
|
||||
if [synopsis] == "nan" {
|
||||
mutate { remove_field => [ "synopsis" ] }
|
||||
}
|
||||
|
||||
mutate {
|
||||
remove_field => [ "message" ]
|
||||
add_field => { "risk_score" => "%{cvss}" }
|
||||
}
|
||||
mutate {
|
||||
convert => { "risk_score" => "float" }
|
||||
}
|
||||
|
||||
# Compensating controls - adjust risk_score
|
||||
# Adobe and Java are not allowed to run in browser unless whitelisted
|
||||
# Therefore, lower score by dividing by 3 (score is subjective to risk)
|
||||
if [risk_score] != 0 {
|
||||
if [plugin_name] =~ "Adobe" and [risk_score] > 6 or [plugin_name] =~ "Java" and [risk_score] > 6 {
|
||||
ruby {
|
||||
code => "event.set('risk_score', event.get('risk_score') / 3)"
|
||||
}
|
||||
mutate {
|
||||
add_field => { "compensating_control" => "Adobe and Flash removed from browsers unless whitelisted site." }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
# Add tags for reporting based on assets or criticality
|
||||
if [host] == "192.168.0.1" or [host] == "192.168.0.50" or [host] =~ "^192\.168\.10\." or [host] =~ "^42.42.42." {
|
||||
mutate {
|
||||
add_tag => [ "critical_asset" ]
|
||||
}
|
||||
}
|
||||
if [host] =~ "^192\.168\.[45][0-9][0-9]\.1$" or [host] =~ "^192.168\.[50]\.[0-9]{1,2}\.1$"{
|
||||
mutate {
|
||||
add_tag => [ "has_hipaa_data" ]
|
||||
}
|
||||
}
|
||||
if [host] =~ "^192\.168\.[45][0-9][0-9]\." {
|
||||
mutate {
|
||||
add_tag => [ "hipaa_asset" ]
|
||||
}
|
||||
}
|
||||
if [host] =~ "^192\.168\.5\." {
|
||||
mutate {
|
||||
add_tag => [ "pci_asset" ]
|
||||
}
|
||||
if [host] =~ "^10\.0\.50\." {
|
||||
mutate {
|
||||
add_tag => [ "web_servers" ]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
if "nessus" in [tags] or [type] == "nessus" {
|
||||
#stdout { codec => rubydebug }
|
||||
elasticsearch {
|
||||
hosts => [ "localhost" ]
|
||||
index => "logstash-nessus-%{+YYYY.MM}"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -7,7 +7,7 @@ output {
|
||||
if "nessus" in [tags] or [type] == "nessus" {
|
||||
#stdout { codec => rubydebug }
|
||||
elasticsearch {
|
||||
hosts => "localhost:19200"
|
||||
hosts => "localhost:9200"
|
||||
index => "logstash-nessus-%{+YYYY.MM}"
|
||||
}
|
||||
}
|
||||
|
||||
@ -1 +1 @@
|
||||
from utils.cli import *
|
||||
from utils.cli import bcolors
|
||||
@ -35,7 +35,7 @@ class NessusAPI(object):
|
||||
'Origin': self.base,
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'Accept-Language': 'en-US,en;q=0.8',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.96 Safari/537.36',
|
||||
'User-Agent': 'VulnWhisperer for Nessus',
|
||||
'Content-Type': 'application/json',
|
||||
'Accept': 'application/json, text/javascript, */*; q=0.01',
|
||||
'Referer': self.base,
|
||||
|
||||
@ -6,6 +6,7 @@ from lxml import objectify
|
||||
from lxml.builder import E
|
||||
import xml.etree.ElementTree as ET
|
||||
import pandas as pd
|
||||
import qualysapi
|
||||
import qualysapi.config as qcconf
|
||||
import requests
|
||||
from requests.packages.urllib3.exceptions import InsecureRequestWarning
|
||||
@ -46,6 +47,7 @@ class qualysWhisper(object):
|
||||
self.template_id = self.config_parse.get_template_id()
|
||||
except:
|
||||
print 'ERROR - Could not retrieve template ID'
|
||||
sys.exit(2)
|
||||
|
||||
def request(
|
||||
self,
|
||||
@ -370,28 +372,15 @@ class qualysWebAppReport:
|
||||
if 'Content' not in merged_df:
|
||||
merged_df['Content'] = ''
|
||||
|
||||
merged_df['Payload #1'] = merged_df['Payload #1'
|
||||
].apply(self.cleanser)
|
||||
merged_df['Request Method #1'] = merged_df['Request Method #1'
|
||||
].apply(self.cleanser)
|
||||
merged_df['Request URL #1'] = merged_df['Request URL #1'
|
||||
].apply(self.cleanser)
|
||||
merged_df['Request Headers #1'] = merged_df['Request Headers #1'
|
||||
].apply(self.cleanser)
|
||||
merged_df['Response #1'] = merged_df['Response #1'
|
||||
].apply(self.cleanser)
|
||||
merged_df['Evidence #1'] = merged_df['Evidence #1'
|
||||
].apply(self.cleanser)
|
||||
columns_to_cleanse = ['Payload #1','Request Method #1','Request URL #1',
|
||||
'Request Headers #1','Response #1','Evidence #1',
|
||||
'Description','Impact','Solution','Url','Content']
|
||||
|
||||
for col in columns_to_cleanse:
|
||||
merged_df[col] = merged_df[col].apply(self.cleanser)
|
||||
|
||||
|
||||
merged_df['Description'] = merged_df['Description'
|
||||
].apply(self.cleanser)
|
||||
merged_df['Impact'] = merged_df['Impact'].apply(self.cleanser)
|
||||
merged_df['Solution'] = merged_df['Solution'
|
||||
].apply(self.cleanser)
|
||||
merged_df['Url'] = merged_df['Url'].apply(self.cleanser)
|
||||
merged_df['Content'] = merged_df['Content'].apply(self.cleanser)
|
||||
merged_df = merged_df.drop(['QID_y', 'QID_x'], axis=1)
|
||||
|
||||
merged_df = merged_df.rename(columns={'Id': 'QID'})
|
||||
|
||||
try:
|
||||
@ -427,49 +416,15 @@ class qualysWebAppReport:
|
||||
|
||||
return merged_data
|
||||
|
||||
def whisper_webapp(self, report_id, updated_date):
|
||||
"""
|
||||
report_id: App ID
|
||||
updated_date: Last time scan was ran for app_id
|
||||
"""
|
||||
vuln_ready = None
|
||||
|
||||
|
||||
maxInt = sys.maxsize
|
||||
decrement = True
|
||||
|
||||
while decrement:
|
||||
decrement = False
|
||||
try:
|
||||
|
||||
if 'Z' in updated_date:
|
||||
updated_date = self.iso_to_epoch(updated_date)
|
||||
report_name = 'qualys_web_' + str(report_id) \
|
||||
+ '_{last_updated}'.format(last_updated=updated_date) \
|
||||
+ '.csv'
|
||||
if os.path.isfile(report_name):
|
||||
print('[ACTION] - File already exist! Skipping...')
|
||||
pass
|
||||
else:
|
||||
print('[ACTION] - Generating report for %s' % report_id)
|
||||
status = self.qw.create_report(report_id)
|
||||
root = objectify.fromstring(status)
|
||||
if root.responseCode == 'SUCCESS':
|
||||
print('[INFO] - Successfully generated report for webapp: %s' \
|
||||
% report_id)
|
||||
generated_report_id = root.data.Report.id
|
||||
print ('[INFO] - New Report ID: %s' \
|
||||
% generated_report_id)
|
||||
vuln_ready = self.process_data(generated_report_id)
|
||||
|
||||
vuln_ready.to_csv(report_name, index=False, header=True) # add when timestamp occured
|
||||
print('[SUCCESS] - Report written to %s' \
|
||||
% report_name)
|
||||
print('[ACTION] - Removing report %s' \
|
||||
% generated_report_id)
|
||||
cleaning_up = \
|
||||
self.qw.delete_report(generated_report_id)
|
||||
os.remove(str(generated_report_id) + '.csv')
|
||||
print('[ACTION] - Deleted report: %s' \
|
||||
% generated_report_id)
|
||||
else:
|
||||
print('Could not process report ID: %s' % status)
|
||||
except Exception as e:
|
||||
print('[ERROR] - Could not process %s - %s' % (report_id, e))
|
||||
return vuln_ready
|
||||
|
||||
|
||||
csv.field_size_limit(maxInt)
|
||||
except OverflowError:
|
||||
maxInt = int(maxInt/10)
|
||||
decrement = True
|
||||
@ -12,5 +12,6 @@ class bcolors:
|
||||
UNDERLINE = '\033[4m'
|
||||
|
||||
INFO = '{info}[INFO]{endc}'.format(info=OKBLUE, endc=ENDC)
|
||||
ACTION = '{info}[ACTION]{endc}'.format(info=OKBLUE, endc=ENDC)
|
||||
SUCCESS = '{green}[SUCCESS]{endc}'.format(green=OKGREEN, endc=ENDC)
|
||||
FAIL = '{red}[FAIL]{endc}'.format(red=FAIL, endc=ENDC)
|
||||
|
||||
@ -4,8 +4,10 @@ __author__ = 'Austin Taylor'
|
||||
|
||||
from base.config import vwConfig
|
||||
from frameworks.nessus import NessusAPI
|
||||
from frameworks.qualys import qualysWebAppReport
|
||||
from utils.cli import bcolors
|
||||
import pandas as pd
|
||||
from lxml import objectify
|
||||
import sys
|
||||
import os
|
||||
import io
|
||||
@ -18,6 +20,9 @@ import logging
|
||||
|
||||
|
||||
class vulnWhispererBase(object):
|
||||
|
||||
CONFIG_SECTION = None
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config=None,
|
||||
@ -27,84 +32,31 @@ class vulnWhispererBase(object):
|
||||
debug=False,
|
||||
username=None,
|
||||
password=None,
|
||||
):
|
||||
pass
|
||||
|
||||
class vulnWhisperer(object):
|
||||
def __init__(
|
||||
self,
|
||||
config=None,
|
||||
db_name='report_tracker.db',
|
||||
purge=False,
|
||||
verbose=None,
|
||||
debug=False,
|
||||
username=None,
|
||||
password=None,
|
||||
section=None,
|
||||
):
|
||||
|
||||
self.verbose = verbose
|
||||
self.nessus_connect = False
|
||||
self.develop = True
|
||||
|
||||
if self.CONFIG_SECTION is None:
|
||||
raise Exception('Implementing class must define CONFIG_SECTION')
|
||||
|
||||
self.db_name = db_name
|
||||
self.purge = purge
|
||||
|
||||
if config is not None:
|
||||
try:
|
||||
self.config = vwConfig(config_in=config)
|
||||
self.nessus_enabled = self.config.getbool('nessus',
|
||||
'enabled')
|
||||
self.enabled = self.config.get(self.CONFIG_SECTION, 'enabled')
|
||||
self.hostname = self.config.get(self.CONFIG_SECTION, 'hostname')
|
||||
self.username = self.config.get(self.CONFIG_SECTION, 'username')
|
||||
self.password = self.config.get(self.CONFIG_SECTION, 'password')
|
||||
self.write_path = self.config.get(self.CONFIG_SECTION, 'write_path')
|
||||
self.db_path = self.config.get(self.CONFIG_SECTION, 'db_path')
|
||||
self.verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
|
||||
|
||||
if self.nessus_enabled:
|
||||
self.nessus_hostname = self.config.get('nessus',
|
||||
'hostname')
|
||||
self.nessus_port = self.config.get('nessus', 'port')
|
||||
|
||||
if password:
|
||||
self.nessus_password = password
|
||||
else:
|
||||
self.nessus_password = self.config.get('nessus'
|
||||
, 'password')
|
||||
|
||||
if username:
|
||||
self.nessus_username = username
|
||||
else:
|
||||
self.nessus_username = self.config.get('nessus'
|
||||
, 'username')
|
||||
|
||||
self.nessus_writepath = self.config.get('nessus',
|
||||
'write_path')
|
||||
self.nessus_dbpath = self.config.get('nessus',
|
||||
'db_path')
|
||||
self.nessus_trash = self.config.getbool('nessus',
|
||||
'trash')
|
||||
self.verbose = self.config.getbool('nessus',
|
||||
'verbose')
|
||||
|
||||
try:
|
||||
self.vprint('{info} Attempting to connect to nessus...'.format(info=bcolors.INFO))
|
||||
self.nessus = \
|
||||
NessusAPI(hostname=self.nessus_hostname,
|
||||
port=self.nessus_port,
|
||||
username=self.nessus_username,
|
||||
password=self.nessus_password)
|
||||
self.nessus_connect = True
|
||||
self.vprint('{success} Connected to nessus on {host}:{port}'.format(success=bcolors.SUCCESS,
|
||||
host=self.nessus_hostname,
|
||||
port=str(self.nessus_port)))
|
||||
except Exception as e:
|
||||
self.vprint(e)
|
||||
raise Exception(
|
||||
'{fail} Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format(
|
||||
config=self.config,
|
||||
fail=bcolors.FAIL, e=e))
|
||||
except Exception as e:
|
||||
|
||||
self.vprint('{fail} Could not properly load your config!\nReason: {e}'.format(fail=bcolors.FAIL,
|
||||
e=e))
|
||||
sys.exit(0)
|
||||
|
||||
if db_name is not None:
|
||||
if self.nessus_dbpath:
|
||||
self.database = os.path.join(self.nessus_dbpath,
|
||||
if self.db_name is not None:
|
||||
if self.db_path:
|
||||
self.database = os.path.join(self.db_path,
|
||||
db_name)
|
||||
else:
|
||||
self.database = \
|
||||
@ -137,6 +89,7 @@ class vulnWhisperer(object):
|
||||
'uuid',
|
||||
'processed',
|
||||
]
|
||||
|
||||
self.init()
|
||||
self.uuids = self.retrieve_uuids()
|
||||
self.processed = 0
|
||||
@ -145,11 +98,14 @@ class vulnWhisperer(object):
|
||||
|
||||
def vprint(self, msg):
|
||||
if self.verbose:
|
||||
print msg
|
||||
print(msg)
|
||||
|
||||
def create_table(self):
|
||||
self.cur.execute(
|
||||
'CREATE TABLE IF NOT EXISTS scan_history (id INTEGER PRIMARY KEY, scan_name TEXT, scan_id INTEGER, last_modified DATE, filename TEXT, download_time DATE, record_count INTEGER, source TEXT, uuid TEXT, processed INTEGER)'
|
||||
'CREATE TABLE IF NOT EXISTS scan_history (id INTEGER PRIMARY KEY,'
|
||||
' scan_name TEXT, scan_id INTEGER, last_modified DATE, filename TEXT,'
|
||||
' download_time DATE, record_count INTEGER, source TEXT,'
|
||||
' uuid TEXT, processed INTEGER)'
|
||||
)
|
||||
self.conn.commit()
|
||||
|
||||
@ -168,10 +124,83 @@ class vulnWhisperer(object):
|
||||
return data
|
||||
|
||||
def path_check(self, _data):
|
||||
if self.nessus_writepath:
|
||||
data = self.nessus_writepath + '/' + _data
|
||||
if self.write_path:
|
||||
data = self.write_path + '/' + _data
|
||||
return data
|
||||
|
||||
def record_insert(self, record):
|
||||
self.cur.execute('insert into scan_history({table_columns}) values (?,?,?,?,?,?,?,?,?)'.format(
|
||||
table_columns=', '.join(self.table_columns)),
|
||||
record)
|
||||
self.conn.commit()
|
||||
|
||||
def retrieve_uuids(self):
|
||||
"""
|
||||
Retrieves UUIDs from database and checks list to determine which files need to be processed.
|
||||
:return:
|
||||
"""
|
||||
try:
|
||||
self.conn.text_factory = str
|
||||
self.cur.execute('SELECT uuid FROM scan_history where source = {config_section}'.format(config_section=self.CONFIG_SECTION))
|
||||
results = frozenset([r[0] for r in self.cur.fetchall()])
|
||||
except:
|
||||
results = []
|
||||
return results
|
||||
|
||||
class vulnWhispererNessus(vulnWhispererBase):
|
||||
|
||||
CONFIG_SECTION = 'nessus'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config=None,
|
||||
db_name='report_tracker.db',
|
||||
purge=False,
|
||||
verbose=None,
|
||||
debug=False,
|
||||
username=None,
|
||||
password=None,
|
||||
):
|
||||
super(vulnWhispererNessus, self).__init__(config=config)
|
||||
|
||||
self.port = int(self.config.get(self.CONFIG_NAME, 'port'))
|
||||
|
||||
self.develop = True
|
||||
self.purge = purge
|
||||
|
||||
if config is not None:
|
||||
try:
|
||||
#if self.enabled:
|
||||
self.nessus_port = self.config.get(self.CONFIG_SECTION, 'port')
|
||||
|
||||
self.nessus_trash = self.config.getbool(self.CONFIG_SECTION,
|
||||
'trash')
|
||||
|
||||
try:
|
||||
self.vprint('{info} Attempting to connect to nessus...'.format(info=bcolors.INFO))
|
||||
self.nessus = \
|
||||
NessusAPI(hostname=self.hostname,
|
||||
port=self.nessus_port,
|
||||
username=self.username,
|
||||
password=self.password)
|
||||
self.nessus_connect = True
|
||||
self.vprint('{success} Connected to nessus on {host}:{port}'.format(success=bcolors.SUCCESS,
|
||||
host=self.hostname,
|
||||
port=str(self.nessus_port)))
|
||||
except Exception as e:
|
||||
self.vprint(e)
|
||||
raise Exception(
|
||||
'{fail} Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format(
|
||||
config=self.config,
|
||||
fail=bcolors.FAIL, e=e))
|
||||
except Exception as e:
|
||||
|
||||
self.vprint('{fail} Could not properly load your config!\nReason: {e}'.format(fail=bcolors.FAIL,
|
||||
e=e))
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
|
||||
def scan_count(self, scans, completed=False):
|
||||
"""
|
||||
|
||||
@ -206,32 +235,15 @@ class vulnWhisperer(object):
|
||||
]))
|
||||
scan_records.append(record.copy())
|
||||
except Exception as e:
|
||||
|
||||
# Generates error each time nonetype is encountered.
|
||||
# print(e)
|
||||
|
||||
pass
|
||||
|
||||
if completed:
|
||||
scan_records = [s for s in scan_records if s['status']
|
||||
== 'completed']
|
||||
scan_records = [s for s in scan_records if s['status'] == 'completed']
|
||||
return scan_records
|
||||
|
||||
def record_insert(self, record):
|
||||
self.cur.execute('insert into scan_history({table_columns}) values (?,?,?,?,?,?,?,?,?)'.format(
|
||||
table_columns=', '.join(self.table_columns)),
|
||||
record)
|
||||
self.conn.commit()
|
||||
|
||||
def retrieve_uuids(self):
|
||||
"""
|
||||
Retrieves UUIDs from database and checks list to determine which files need to be processed.
|
||||
:return:
|
||||
"""
|
||||
|
||||
self.conn.text_factory = str
|
||||
self.cur.execute('SELECT uuid FROM scan_history')
|
||||
results = frozenset([r[0] for r in self.cur.fetchall()])
|
||||
return results
|
||||
|
||||
def whisper_nessus(self):
|
||||
if self.nessus_connect:
|
||||
@ -299,12 +311,9 @@ class vulnWhisperer(object):
|
||||
if status == 'completed':
|
||||
file_name = '%s_%s_%s_%s.%s' % (scan_name, scan_id,
|
||||
history_id, norm_time, 'csv')
|
||||
repls = (('\\', '_'), ('/', '_'), ('/', '_'), (' ',
|
||||
'_'))
|
||||
file_name = reduce(lambda a, kv: a.replace(*kv),
|
||||
repls, file_name)
|
||||
relative_path_name = self.path_check(folder_name
|
||||
+ '/' + file_name)
|
||||
repls = (('\\', '_'), ('/', '_'), ('/', '_'), (' ', '_'))
|
||||
file_name = reduce(lambda a, kv: a.replace(*kv), repls, file_name)
|
||||
relative_path_name = self.path_check(folder_name + '/' + file_name)
|
||||
|
||||
if os.path.isfile(relative_path_name):
|
||||
if self.develop:
|
||||
@ -335,23 +344,14 @@ class vulnWhisperer(object):
|
||||
self.vprint('Processing %s/%s for scan: %s'
|
||||
% (scan_count, len(scan_history),
|
||||
scan_name))
|
||||
clean_csv['CVSS'] = clean_csv['CVSS'
|
||||
].astype(str).apply(self.cleanser)
|
||||
clean_csv['CVE'] = clean_csv['CVE'
|
||||
].astype(str).apply(self.cleanser)
|
||||
clean_csv['Description'] = \
|
||||
clean_csv['Description'
|
||||
].astype(str).apply(self.cleanser)
|
||||
columns_to_cleanse = ['CVSS','CVE','Description','Synopsis','Solution','See Also','Plugin Output']
|
||||
|
||||
for col in columns_to_cleanse:
|
||||
clean_csv[col] = clean_csv[col].astype(str).apply(self.cleanser)
|
||||
|
||||
clean_csv['Synopsis'] = \
|
||||
clean_csv['Description'
|
||||
].astype(str).apply(self.cleanser)
|
||||
clean_csv['Solution'] = clean_csv['Solution'
|
||||
].astype(str).apply(self.cleanser)
|
||||
clean_csv['See Also'] = clean_csv['See Also'
|
||||
].astype(str).apply(self.cleanser)
|
||||
clean_csv['Plugin Output'] = \
|
||||
clean_csv['Plugin Output'
|
||||
].astype(str).apply(self.cleanser)
|
||||
clean_csv.to_csv(relative_path_name,
|
||||
index=False)
|
||||
record_meta = (
|
||||
@ -391,8 +391,129 @@ class vulnWhisperer(object):
|
||||
else:
|
||||
|
||||
self.vprint('{fail} Failed to use scanner at {host}'.format(fail=bcolors.FAIL,
|
||||
host=self.nessus_hostname + ':'
|
||||
host=self.hostname + ':'
|
||||
+ self.nessus_port))
|
||||
|
||||
|
||||
class vulnWhispererQualys(vulnWhispererBase):
|
||||
|
||||
CONFIG_SECTION = 'qualys'
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
config=None,
|
||||
db_name='report_tracker.db',
|
||||
purge=False,
|
||||
verbose=None,
|
||||
debug=False,
|
||||
username=None,
|
||||
password=None,
|
||||
):
|
||||
super(vulnWhispererQualys, self).__init__(config=config, )
|
||||
|
||||
self.qualys_web = qualysWebAppReport(config=config)
|
||||
self.latest_scans = self.qualys_web.qw.get_web_app_list()
|
||||
|
||||
|
||||
def whisper_webapp(self, report_id, updated_date):
|
||||
"""
|
||||
report_id: App ID
|
||||
updated_date: Last time scan was ran for app_id
|
||||
"""
|
||||
vuln_ready = None
|
||||
|
||||
try:
|
||||
if 'Z' in updated_date:
|
||||
updated_date = self.qualys_web.iso_to_epoch(updated_date)
|
||||
report_name = 'qualys_web_' + str(report_id) \
|
||||
+ '_{last_updated}'.format(last_updated=updated_date) \
|
||||
+ '.csv'
|
||||
if os.path.isfile(report_name):
|
||||
print('{action} - File already exist! Skipping...'.format(action=bcolors.ACTION))
|
||||
pass
|
||||
else:
|
||||
print('{action} - Generating report for %s'.format(action=bcolors.ACTION) % report_id)
|
||||
status = self.qualys_web.qw.create_report(report_id)
|
||||
root = objectify.fromstring(status)
|
||||
if root.responseCode == 'SUCCESS':
|
||||
print('{info} - Successfully generated report for webapp: %s'.format(info=bcolors.INFO) \
|
||||
% report_id)
|
||||
generated_report_id = root.data.Report.id
|
||||
print('{info} - New Report ID: %s'.format(info=bcolors.INFO) \
|
||||
% generated_report_id)
|
||||
vuln_ready = self.qualys_web.process_data(generated_report_id)
|
||||
|
||||
vuln_ready.to_csv(report_name, index=False, header=True) # add when timestamp occured
|
||||
print('{success} - Report written to %s'.format(success=bcolors.SUCCESS) \
|
||||
% report_name)
|
||||
print('{action} - Removing report %s'.format(action=bcolors.ACTION) \
|
||||
% generated_report_id)
|
||||
cleaning_up = \
|
||||
self.qualys_web.qw.delete_report(generated_report_id)
|
||||
os.remove(str(generated_report_id) + '.csv')
|
||||
print('{action} - Deleted report: %s'.format(action=bcolors.ACTION) \
|
||||
% generated_report_id)
|
||||
else:
|
||||
print('{error} Could not process report ID: %s'.format(error=bcolors.FAIL) % status)
|
||||
except Exception as e:
|
||||
print('{error} - Could not process %s - %s'.format(error=bcolors.FAIL) % (report_id, e))
|
||||
return vuln_ready
|
||||
|
||||
def process_web_assets(self):
|
||||
counter = 0
|
||||
for app in self.latest_scans.iterrows():
|
||||
counter += 1
|
||||
print('Processing %s/%s' % (counter, len(self.latest_scans)))
|
||||
self.whisper_webapp(app[1]['id'], app[1]['createdDate'])
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
class vulnWhisperer(object):
|
||||
|
||||
def __init__(self,
|
||||
profile=None,
|
||||
verbose=None,
|
||||
username=None,
|
||||
password=None,
|
||||
config=None):
|
||||
|
||||
self.profile = profile
|
||||
self.config = config
|
||||
self.username = username
|
||||
self.password = password
|
||||
self.verbose = verbose
|
||||
|
||||
|
||||
def whisper_vulnerabilities(self):
|
||||
|
||||
if self.profile == 'nessus':
|
||||
vw = vulnWhispererNessus(config=self.config, username=self.username, password=self.password, verbose=self.verbose)
|
||||
vw.whisper_nessus()
|
||||
|
||||
elif self.profile == 'qualys':
|
||||
vw = vulnWhispererQualys(config=self.config)
|
||||
vw.process_web_assets()
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
'''
|
||||
for f in folders:
|
||||
if not os.path.exists(self.path_check(f['name'])):
|
||||
if f['name'] == 'Trash' and self.nessus_trash:
|
||||
os.makedirs(self.path_check(f['name']))
|
||||
elif f['name'] != 'Trash':
|
||||
os.makedirs(self.path_check(f['name']))
|
||||
else:
|
||||
os.path.exists(self.path_check(f['name']))
|
||||
self.vprint('{info} Directory already exist for {scan} - Skipping creation'.format(
|
||||
scan=self.path_check(f['name'
|
||||
]), info=bcolors.INFO))
|
||||
|
||||
'''
|
||||
Reference in New Issue
Block a user