Cyberdefense/VulnWhisperer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Jira module fully working (#104)

Browse Source 
* clean OS X .DS_Store files

* fix nessus end of line carriage, added JIRA args

* JIRA module fully working

* jira module working with nessus

* added check on already existing jira config, update README

* qualys_vm<->jira working, qualys_vm database entries with qualys_vm, improved checks

* JIRA module updates ticket's assets and comments update

* added JIRA auto-close function for resolved vulnerabitilies

* fix if components variable empty issue

* fix creation of new ticket after updating existing one

* final fixes, added extra line in template

* added vulnerability criticality as label in order to be able to filter
This commit is contained in:
Quim Montal
2018-10-12 16:30:14 +02:00
committed by Austin Taylor
parent 13bb288217
commit 4422db586d
13 changed files with 922 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
.DS_Store vendored
View File

Binary file not shown.

3
.gitignore vendored
View File

@ -100,3 +100,6 @@ ENV/
# mypy
.mypy_cache/
# Mac
.DS_Store

8
README.md
View File

@ -25,8 +25,16 @@ Currently Supports
- [ ] [Nexpose](https://www.rapid7.com/products/nexpose/)
- [ ] [Insight VM](https://www.rapid7.com/products/insightvm/)
- [ ] [NMAP](https://nmap.org/)
- [ ] [Burp Suite](https://portswigger.net/burp)
- [ ] [OWASP ZAP](https://www.zaproxy.org/)
- [ ] More to come
### Reporting Frameworks
- [X] [ELK](https://www.elastic.co/elk-stack)
- [X] [Jira](https://www.atlassian.com/software/jira)
- [ ] [Splunk](https://www.splunk.com/)
Getting Started
===============

12
bin/vuln_whisperer
View File

@ -24,6 +24,10 @@ def main():
help='Path of config file', type=lambda x: isFileValid(parser, x.strip()))
parser.add_argument('-s', '--section', dest='section', required=False,
help='Section in config')
parser.add_argument('--source', dest='source', required=False,
help='JIRA required only! Source scanner to report')
parser.add_argument('-n', '--scanname', dest='scanname', required=False,
help='JIRA required only! Scan name from scan to report')
parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=True,
help='Prints status out to screen (defaults to True)')
parser.add_argument('-u', '--username', dest='username', required=False, default=None, type=lambda x: x.strip(), help='The NESSUS username')
@ -46,7 +50,9 @@ def main():
profile=section,
verbose=args.verbose,
username=args.username,
password=args.password)
password=args.password,
source=args.source,
scanname=args.scanname)
vw.whisper_vulnerabilities()
sys.exit(1)
@ -56,7 +62,9 @@ def main():
profile=args.section,
verbose=args.verbose,
username=args.username,
password=args.password)
password=args.password,
source=args.source,
scanname=args.scanname)
vw.whisper_vulnerabilities()
sys.exit(1)

8
requirements.txt
View File

@ -1,7 +1,9 @@
pandas==0.20.3
setuptools==0.9.8
setuptools==40.4.3
pytz==2017.2
Requests==2.18.3
qualysapi==4.1.0
#qualysapi==4.1.0
lxml==4.1.1
bs4
bs4
jira
bottle

4
setup.py
View File

@ -4,7 +4,7 @@ from setuptools import setup, find_packages
setup(
name='VulnWhisperer',
version='1.5.0',
version='1.7.1',
packages=find_packages(),
url='https://github.com/austin-taylor/vulnwhisperer',
license="""MIT License
@ -26,7 +26,7 @@ setup(
SOFTWARE.""",
author='Austin Taylor',
author_email='email@austintaylor.io',
description='Vulnerability assessment framework aggregator',
description='Vulnerability Assessment Framework Aggregator',
scripts=['bin/vuln_whisperer']
)

BIN
vulnwhisp/.DS_Store vendored
View File

Binary file not shown.

47
vulnwhisp/base/config.py
View File

@ -25,6 +25,49 @@ class vwConfig(object):
enabled = []
check = ["true", "True", "1"]
for section in self.config.sections():
if self.get(section, "enabled") in check:
enabled.append(section)
try:
if self.get(section, "enabled") in check:
enabled.append(section)
except:
print "[INFO] Section {} has no option 'enabled'".format(section)
return enabled
def exists_jira_profiles(self, profiles):
# get list of profiles source_scanner.scan_name
for profile in profiles:
if not self.config.has_section(self.normalize_section(profile)):
print "[INFO] JIRA Scan Profile missing"
return False
return True
def update_jira_profiles(self, profiles):
# create JIRA profiles in the ini config file
for profile in profiles:
#IMPORTANT profile scans/results will be normalized to lower and "_" instead of spaces for ini file section
section_name = self.normalize_section(profile)
try:
self.get(section_name, "source")
print "Skipping creating of section '{}'; already exists".format(section_name)
except:
print "Creating config section for '{}'".format(section_name)
self.config.add_section(section_name)
self.config.set(section_name,'source',profile.split('.')[0])
# in case any scan name contains '.' character
self.config.set(section_name,'scan_name','.'.join(profile.split('.')[1:]))
self.config.set(section_name,'jira_project', "")
self.config.set(section_name,'; if multiple components, separate by ","')
self.config.set(section_name,'components', "")
self.config.set(section_name,'; minimum criticality to report (low, medium, high or critical)')
self.config.set(section_name,'min_critical_to_report', 'high')
# writing changes back to file
with open(self.config_in, 'w') as configfile:
self.config.write(configfile)
return
def normalize_section(self, profile):
profile = "jira.{}".format(profile.lower().replace(" ","_"))
return profile

440
vulnwhisp/frameworks/nessus.py
View File

@ -1,224 +1,224 @@
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
import pytz
from datetime import datetime
import json
import sys
import time
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
import pytz
from datetime import datetime
import json
import sys
import time
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
class NessusAPI(object):
SESSION = '/session'
FOLDERS = '/folders'
SCANS = '/scans'
SCAN_ID = SCANS + '/{scan_id}'
HOST_VULN = SCAN_ID + '/hosts/{host_id}'
PLUGINS = HOST_VULN + '/plugins/{plugin_id}'
EXPORT = SCAN_ID + '/export'
EXPORT_TOKEN_DOWNLOAD = '/scans/exports/{token_id}/download'
EXPORT_FILE_DOWNLOAD = EXPORT + '/{file_id}/download'
EXPORT_STATUS = EXPORT + '/{file_id}/status'
EXPORT_HISTORY = EXPORT + '?history_id={history_id}'
def __init__(self, hostname=None, port=None, username=None, password=None, verbose=True):
if username is None or password is None:
raise Exception('ERROR: Missing username or password.')
self.user = username
self.password = password
self.base = 'https://{hostname}:{port}'.format(hostname=hostname, port=port)
self.verbose = verbose
self.headers = {
'Origin': self.base,
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'en-US,en;q=0.8',
'User-Agent': 'VulnWhisperer for Nessus',
'Content-Type': 'application/json',
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Referer': self.base,
'X-Requested-With': 'XMLHttpRequest',
'Connection': 'keep-alive',
'X-Cookie': None
}
self.login()
self.scan_ids = self.get_scan_ids()
def vprint(self, msg):
if self.verbose:
print(msg)
def login(self):
resp = self.get_token()
if resp.status_code is 200:
self.headers['X-Cookie'] = 'token={token}'.format(token=resp.json()['token'])
else:
raise Exception('[FAIL] Could not login to Nessus')
def request(self, url, data=None, headers=None, method='POST', download=False, json=False):
if headers is None:
headers = self.headers
timeout = 0
success = False
url = self.base + url
methods = {'GET': requests.get,
'POST': requests.post,
'DELETE': requests.delete}
while (timeout <= 10) and (not success):
data = methods[method](url, data=data, headers=self.headers, verify=False)
if data.status_code == 401:
if url == self.base + self.SESSION:
break
try:
self.login()
timeout += 1
self.vprint('[INFO] Token refreshed')
except Exception as e:
self.vprint('[FAIL] Could not refresh token\nReason: %s' % e)
else:
success = True
if json:
data = data.json()
if download:
return data.content
return data
def get_token(self):
auth = '{"username":"%s", "password":"%s"}' % (self.user, self.password)
token = self.request(self.SESSION, data=auth, json=False)
return token
def logout(self):
self.request(self.SESSION, method='DELETE')
def get_folders(self):
folders = self.request(self.FOLDERS, method='GET', json=True)
return folders
def get_scans(self):
scans = self.request(self.SCANS, method='GET', json=True)
return scans
def get_scan_ids(self):
scans = self.get_scans()
scan_ids = [scan_id['id'] for scan_id in scans['scans']] if scans['scans'] else []
return scan_ids
def count_scan(self, scans, folder_id):
count = 0
for scan in scans:
if scan['folder_id'] == folder_id: count = count + 1
return count
def print_scans(self, data):
for folder in data['folders']:
print("\\{0} - ({1})\\".format(folder['name'], self.count_scan(data['scans'], folder['id'])))
for scan in data['scans']:
if scan['folder_id'] == folder['id']:
print(
"\t\"{0}\" - sid:{1} - uuid: {2}".format(scan['name'].encode('utf-8'), scan['id'], scan['uuid']))
def get_scan_details(self, scan_id):
data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json=True)
return data
def get_scan_history(self, scan_id):
data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json=True)
return data['history']
def get_scan_hosts(self, scan_id):
data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json=True)
return data['hosts']
def get_host_vulnerabilities(self, scan_id, host_id):
query = self.HOST_VULN.format(scan_id=scan_id, host_id=host_id)
data = self.request(query, method='GET', json=True)
return data
def get_plugin_info(self, scan_id, host_id, plugin_id):
query = self.PLUGINS.format(scan_id=scan_id, host_id=host_id, plugin_id=plugin_id)
data = self.request(query, method='GET', json=True)
return data
def export_scan(self, scan_id, history_id):
data = {'format': 'csv'}
query = self.EXPORT_REPORT.format(scan_id=scan_id, history_id=history_id)
req = self.request(query, data=data, method='POST')
return req
def download_scan(self, scan_id=None, history=None, export_format="", chapters="", dbpasswd="", profile=""):
running = True
counter = 0
data = {'format': export_format}
if not history:
query = self.EXPORT.format(scan_id=scan_id)
else:
query = self.EXPORT_HISTORY.format(scan_id=scan_id, history_id=history)
scan_id = str(scan_id)
req = self.request(query, data=json.dumps(data), method='POST', json=True)
try:
file_id = req['file']
token_id = req['token'] if 'token' in req else req['temp_token']
except Exception as e:
print("[ERROR] %s" % e)
print('Download for file id ' + str(file_id) + '.')
while running:
time.sleep(2)
counter += 2
report_status = self.request(self.EXPORT_STATUS.format(scan_id=scan_id, file_id=file_id), method='GET',
json=True)
running = report_status['status'] != 'ready'
sys.stdout.write(".")
sys.stdout.flush()
if counter % 60 == 0:
print("")
print("")
if profile=='tenable':
content = self.request(self.EXPORT_FILE_DOWNLOAD.format(scan_id=scan_id, file_id=file_id), method='GET', download=True)
else:
content = self.request(self.EXPORT_TOKEN_DOWNLOAD.format(token_id=token_id), method='GET', download=True)
return content
@staticmethod
def merge_dicts(self, *dict_args):
"""
Given any number of dicts, shallow copy and merge into a new dict,
precedence goes to key value pairs in latter dicts.
"""
result = {}
for dictionary in dict_args:
result.update(dictionary)
return result
def get_utc_from_local(self, date_time, local_tz=None, epoch=True):
date_time = datetime.fromtimestamp(date_time)
if local_tz is None:
local_tz = pytz.timezone('US/Central')
else:
local_tz = pytz.timezone(local_tz)
# print date_time
local_time = local_tz.normalize(local_tz.localize(date_time))
local_time = local_time.astimezone(pytz.utc)
if epoch:
naive = local_time.replace(tzinfo=None)
local_time = int((naive - datetime(1970, 1, 1)).total_seconds())
return local_time
def tz_conv(self, tz):
time_map = {'Eastern Standard Time': 'US/Eastern',
'Central Standard Time': 'US/Central',
'Pacific Standard Time': 'US/Pacific',
'None': 'US/Central'}
return time_map.get(tz, None)
class NessusAPI(object):
SESSION = '/session'
FOLDERS = '/folders'
SCANS = '/scans'
SCAN_ID = SCANS + '/{scan_id}'
HOST_VULN = SCAN_ID + '/hosts/{host_id}'
PLUGINS = HOST_VULN + '/plugins/{plugin_id}'
EXPORT = SCAN_ID + '/export'
EXPORT_TOKEN_DOWNLOAD = '/scans/exports/{token_id}/download'
EXPORT_FILE_DOWNLOAD = EXPORT + '/{file_id}/download'
EXPORT_STATUS = EXPORT + '/{file_id}/status'
EXPORT_HISTORY = EXPORT + '?history_id={history_id}'
def __init__(self, hostname=None, port=None, username=None, password=None, verbose=True):
if username is None or password is None:
raise Exception('ERROR: Missing username or password.')
self.user = username
self.password = password
self.base = 'https://{hostname}:{port}'.format(hostname=hostname, port=port)
self.verbose = verbose
self.headers = {
'Origin': self.base,
'Accept-Encoding': 'gzip, deflate, br',
'Accept-Language': 'en-US,en;q=0.8',
'User-Agent': 'VulnWhisperer for Nessus',
'Content-Type': 'application/json',
'Accept': 'application/json, text/javascript, */*; q=0.01',
'Referer': self.base,
'X-Requested-With': 'XMLHttpRequest',
'Connection': 'keep-alive',
'X-Cookie': None
}
self.login()
self.scan_ids = self.get_scan_ids()
def vprint(self, msg):
if self.verbose:
print(msg)
def login(self):
resp = self.get_token()
if resp.status_code is 200:
self.headers['X-Cookie'] = 'token={token}'.format(token=resp.json()['token'])
else:
raise Exception('[FAIL] Could not login to Nessus')
def request(self, url, data=None, headers=None, method='POST', download=False, json=False):
if headers is None:
headers = self.headers
timeout = 0
success = False
url = self.base + url
methods = {'GET': requests.get,
'POST': requests.post,
'DELETE': requests.delete}
while (timeout <= 10) and (not success):
data = methods[method](url, data=data, headers=self.headers, verify=False)
if data.status_code == 401:
if url == self.base + self.SESSION:
break
try:
self.login()
timeout += 1
self.vprint('[INFO] Token refreshed')
except Exception as e:
self.vprint('[FAIL] Could not refresh token\nReason: %s' % e)
else:
success = True
if json:
data = data.json()
if download:
return data.content
return data
def get_token(self):
auth = '{"username":"%s", "password":"%s"}' % (self.user, self.password)
token = self.request(self.SESSION, data=auth, json=False)
return token
def logout(self):
self.request(self.SESSION, method='DELETE')
def get_folders(self):
folders = self.request(self.FOLDERS, method='GET', json=True)
return folders
def get_scans(self):
scans = self.request(self.SCANS, method='GET', json=True)
return scans
def get_scan_ids(self):
scans = self.get_scans()
scan_ids = [scan_id['id'] for scan_id in scans['scans']] if scans['scans'] else []
return scan_ids
def count_scan(self, scans, folder_id):
count = 0
for scan in scans:
if scan['folder_id'] == folder_id: count = count + 1
return count
def print_scans(self, data):
for folder in data['folders']:
print("\\{0} - ({1})\\".format(folder['name'], self.count_scan(data['scans'], folder['id'])))
for scan in data['scans']:
if scan['folder_id'] == folder['id']:
print(
"\t\"{0}\" - sid:{1} - uuid: {2}".format(scan['name'].encode('utf-8'), scan['id'], scan['uuid']))
def get_scan_details(self, scan_id):
data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json=True)
return data
def get_scan_history(self, scan_id):
data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json=True)
return data['history']
def get_scan_hosts(self, scan_id):
data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json=True)
return data['hosts']
def get_host_vulnerabilities(self, scan_id, host_id):
query = self.HOST_VULN.format(scan_id=scan_id, host_id=host_id)
data = self.request(query, method='GET', json=True)
return data
def get_plugin_info(self, scan_id, host_id, plugin_id):
query = self.PLUGINS.format(scan_id=scan_id, host_id=host_id, plugin_id=plugin_id)
data = self.request(query, method='GET', json=True)
return data
def export_scan(self, scan_id, history_id):
data = {'format': 'csv'}
query = self.EXPORT_REPORT.format(scan_id=scan_id, history_id=history_id)
req = self.request(query, data=data, method='POST')
return req
def download_scan(self, scan_id=None, history=None, export_format="", chapters="", dbpasswd="", profile=""):
running = True
counter = 0
data = {'format': export_format}
if not history:
query = self.EXPORT.format(scan_id=scan_id)
else:
query = self.EXPORT_HISTORY.format(scan_id=scan_id, history_id=history)
scan_id = str(scan_id)
req = self.request(query, data=json.dumps(data), method='POST', json=True)
try:
file_id = req['file']
token_id = req['token'] if 'token' in req else req['temp_token']
except Exception as e:
print("[ERROR] %s" % e)
print('Download for file id ' + str(file_id) + '.')
while running:
time.sleep(2)
counter += 2
report_status = self.request(self.EXPORT_STATUS.format(scan_id=scan_id, file_id=file_id), method='GET',
json=True)
running = report_status['status'] != 'ready'
sys.stdout.write(".")
sys.stdout.flush()
if counter % 60 == 0:
print("")
print("")
if profile=='tenable':
content = self.request(self.EXPORT_FILE_DOWNLOAD.format(scan_id=scan_id, file_id=file_id), method='GET', download=True)
else:
content = self.request(self.EXPORT_TOKEN_DOWNLOAD.format(token_id=token_id), method='GET', download=True)
return content
@staticmethod
def merge_dicts(self, *dict_args):
"""
Given any number of dicts, shallow copy and merge into a new dict,
precedence goes to key value pairs in latter dicts.
"""
result = {}
for dictionary in dict_args:
result.update(dictionary)
return result
def get_utc_from_local(self, date_time, local_tz=None, epoch=True):
date_time = datetime.fromtimestamp(date_time)
if local_tz is None:
local_tz = pytz.timezone('US/Central')
else:
local_tz = pytz.timezone(local_tz)
# print date_time
local_time = local_tz.normalize(local_tz.localize(date_time))
local_time = local_time.astimezone(pytz.utc)
if epoch:
naive = local_time.replace(tzinfo=None)
local_time = int((naive - datetime(1970, 1, 1)).total_seconds())
return local_time
def tz_conv(self, tz):
time_map = {'Eastern Standard Time': 'US/Eastern',
'Central Standard Time': 'US/Central',
'Pacific Standard Time': 'US/Pacific',
'None': 'US/Central'}
return time_map.get(tz, None)

0
vulnwhisp/reporting/__init__.py Executable file
View File

320
vulnwhisp/reporting/jira_api.py Normal file
View File

@ -0,0 +1,320 @@
import json
from datetime import datetime, timedelta
from jira import JIRA
import requests
from bottle import template
import re
class JiraAPI(object): #NamedLogger):
__logname__="vjira"
#TODO implement logging
def __init__(self, hostname=None, username=None, password=None, debug=False, clean_obsolete=True, max_time_window=6):
#self.setup_logger(debug=debug)
if "https://" not in hostname:
hostname = "https://{}".format(hostname)
self.username = username
self.password = password
self.jira = JIRA(options={'server': hostname}, basic_auth=(self.username, self.password))
#self.logger.info("Created vjira service for {}".format(server))
self.all_tickets = []
self.JIRA_REOPEN_ISSUE = "Reopen Issue"
self.JIRA_CLOSE_ISSUE = "Close Issue"
self.max_time_tracking = max_time_window #in months
#<JIRA Resolution: name=u'Obsolete', id=u'11'>
self.JIRA_RESOLUTION_OBSOLETE = "Obsolete"
self.JIRA_RESOLUTION_FIXED = "Fixed"
self.clean_obsolete = clean_obsolete
self.template_path = 'vulnwhisp/reporting/resources/ticket.tpl'
def create_ticket(self, title, desc, project="IS", components=[], tags=[]):
labels = ['vulnerability_management']
for tag in tags:
labels.append(str(tag))
#self.logger.info("creating ticket for project {} title[20] {}".format(project, title[:20]))
#self.logger.info("project {} has a component requirement: {}".format(project, self.PROJECT_COMPONENT_TABLE[project]))
project_obj = self.jira.project(project)
components_ticket = []
for component in components:
exists = False
for c in project_obj.components:
if component == c.name:
#self.logger.debug("resolved component name {} to id {}".format(component_name, c.id)ra python)
components_ticket.append({ "id": c.id })
exists=True
if not exists:
print "[ERROR] Error creating Ticket: component {} not found".format(component)
return 0
new_issue = self.jira.create_issue(project=project,
summary=title,
description=desc,
issuetype={'name': 'Bug'},
labels=labels,
components=components_ticket)
print "[SUCCESS] Ticket {} has been created".format(new_issue)
return new_issue
#Basic JIRA Metrics
def metrics_open_tickets(self, project=None):
jql = "labels= vulnerability_management and resolution = Unresolved"
if project:
jql += " and (project='{}')".format(project)
print jql
return len(self.jira.search_issues(jql, maxResults=0))
def metrics_closed_tickets(self, project=None):
jql = "labels= vulnerability_management and NOT resolution = Unresolved"
if project:
jql += " and (project='{}')".format(project)
return len(self.jira.search_issues(jql, maxResults=0))
def sync(self, vulnerabilities, project, components=[]):
#JIRA structure of each vulnerability: [source, scan_name, title, diagnosis, consequence, solution, ips, risk, references]
print "JIRA Sync started"
# [HIGIENE] close tickets older than 6 months as obsolete
# Higiene clean up affects to all tickets created by the module, filters by label 'vulnerability_management'
if self.clean_obsolete:
self.close_obsolete_tickets()
for vuln in vulnerabilities:
# JIRA doesn't allow labels with spaces, so making sure that the scan_name doesn't have spaces
# if it has, they will be replaced by "_"
if " " in vuln['scan_name']:
vuln['scan_name'] = "_".join(vuln['scan_name'].split(" "))
exists = False
to_update = False
ticketid = ""
ticket_assets = []
exists, to_update, ticketid, ticket_assets = self.check_vuln_already_exists(vuln)
if exists:
# If ticket "resolved" -> reopen, as vulnerability is still existent
self.reopen_ticket(ticketid)
continue
elif to_update:
self.ticket_update_assets(vuln, ticketid, ticket_assets)
continue
try:
tpl = template(self.template_path, vuln)
except Exception as e:
print e
return 0
self.create_ticket(title=vuln['title'], desc=tpl, project=project, components=components, tags=[vuln['source'], vuln['scan_name'], 'vulnerability', vuln['risk']])
self.close_fixed_tickets(vulnerabilities)
# we reinitialize so the next sync redoes the query with their specific variables
self.all_tickets = []
return True
def check_vuln_already_exists(self, vuln):
# we need to return if the vulnerability has already been reported and the ID of the ticket for further processing
#function returns array [duplicated(bool), update(bool), ticketid, ticket_assets]
title = vuln['title']
labels = [vuln['source'], vuln['scan_name'], 'vulnerability_management', 'vulnerability']
#list(set()) to remove duplicates
assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", ",".join(vuln['ips']))))
if not self.all_tickets:
print "Retrieving all JIRA tickets with the following tags {}".format(labels)
# we want to check all JIRA tickets, to include tickets moved to other queues
# will exclude tickets older than 6 months, old tickets will get closed for higiene and recreated if still vulnerable
jql = "{} AND NOT labels=advisory AND created >=startOfMonth(-{})".format(" AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
self.all_tickets = self.jira.search_issues(jql, maxResults=0)
#WARNING: function IGNORES DUPLICATES, after finding a "duplicate" will just return it exists
#it wont iterate over the rest of tickets looking for other possible duplicates/similar issues
print "Comparing Vulnerabilities to created tickets"
for index in range(len(self.all_tickets)-1):
checking_ticketid, checking_title, checking_assets = self.ticket_get_unique_fields(self.all_tickets[index])
if title == checking_title:
difference = list(set(assets).symmetric_difference(checking_assets))
#to check intersection - set(assets) & set(checking_assets)
if difference:
print "Asset mismatch, ticket to update. TickedID: {}".format(checking_ticketid)
return False, True, checking_ticketid, checking_assets #this will automatically validate
else:
print "Confirmed duplicated. TickedID: {}".format(checking_ticketid)
return True, False, checking_ticketid, [] #this will automatically validate
return False, False, "", []
def ticket_get_unique_fields(self, ticket):
title = ticket.raw.get('fields', {}).get('summary').encode("ascii").strip()
ticketid = ticket.key.encode("ascii")
try:
affected_assets_section = ticket.raw.get('fields', {}).get('description').encode("ascii").split("{panel:title=Affected Assets}")[1].split("{panel}")[0]
assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", affected_assets_section)))
except:
print "[ERROR] Ticket IPs regex failed. Ticket ID: {}".format(ticketid)
assets = []
return ticketid, title, assets
def ticket_update_assets(self, vuln, ticketid, ticket_assets):
# correct description will always be in the vulnerability to report, only needed to update description to new one
print "Ticket {} exists, UPDATE requested".format(ticketid)
if self.is_ticket_resolved(self.jira.issue(ticketid)):
self.reopen_ticket(ticketid)
try:
tpl = template(self.template_path, vuln)
except Exception as e:
print e
return 0
ticket_obj = self.jira.issue(ticketid)
ticket_obj.update()
assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", ",".join(vuln['ips']))))
difference = list(set(assets).symmetric_difference(ticket_assets))
comment = ''
#put a comment with the assets that have been added/removed
for asset in difference:
if asset in assets:
comment += "Asset {} have been added to the ticket as vulnerability *has been newly detected*.\n".format(asset)
elif asset in ticket_assets:
comment += "Asset {} have been removed from the ticket as vulnerability *has been resolved*.\n".format(asset)
ticket_obj.fields.labels.append('updated')
try:
ticket_obj.update(description=tpl, comment=comment, fields={"labels":ticket_obj.fields.labels})
print "Ticket {} updated successfully".format(ticketid)
except:
print "[ERROR] Error while trying up update ticket {}".format(ticketid)
return 0
def close_fixed_tickets(self, vulnerabilities):
# close tickets which vulnerabilities have been resolved and are still open
found_vulns = []
for vuln in vulnerabilities:
found_vulns.append(vuln['title'])
comment = '''This ticket is being closed as it appears that the vulnerability no longer exists.
If the vulnerability reappears, a new ticket will be opened.'''
for ticket in self.all_tickets:
if ticket.raw['fields']['summary'].strip() in found_vulns:
print "Ticket {} is still vulnerable".format(ticket)
continue
print "Ticket {} is no longer vulnerable".format(ticket)
self.close_ticket(ticket, self.JIRA_RESOLUTION_FIXED, comment)
return 0
def is_ticket_reopenable(self, ticket_obj):
transitions = self.jira.transitions(ticket_obj)
for transition in transitions:
if transition.get('name') == self.JIRA_REOPEN_ISSUE:
#print "ticket is reopenable"
return True
print "[ERROR] Ticket can't be opened. Check Jira transitions."
return False
def is_ticket_closeable(self, ticket_obj):
transitions = self.jira.transitions(ticket_obj)
for transition in transitions:
if transition.get('name') == self.JIRA_CLOSE_ISSUE:
return True
print "[ERROR] Ticket can't closed. Check Jira transitions."
return False
def is_ticket_resolved(self, ticket_obj):
#Checks if a ticket is resolved or not
if ticket_obj is not None:
if ticket_obj.raw['fields'].get('resolution') is not None:
if ticket_obj.raw['fields'].get('resolution').get('name') != 'Unresolved':
print "Checked ticket {} is already closed".format(ticket_obj)
#logger.info("ticket {} is closed".format(ticketid))
return True
print "Checked ticket {} is already open".format(ticket_obj)
return False
def is_risk_accepted(self, ticket_obj):
if ticket_obj is not None:
if ticket_obj.raw['fields'].get('labels') is not None:
labels = ticket_obj.raw['fields'].get('labels')
print labels
if "risk_accepted" in labels:
print "Ticket {} accepted risk, will be ignored".format(ticket_obj)
return True
elif "server_decomission" in labels:
print "Ticket {} server decomissioned, will be ignored".format(ticket_obj)
return True
print "Ticket {} risk has not been accepted".format(ticket_obj)
return False
def reopen_ticket(self, ticketid):
print "Ticket {} exists, REOPEN requested".format(ticketid)
# this will reopen a ticket by ticketid
ticket_obj = self.jira.issue(ticketid)
if self.is_ticket_resolved(ticket_obj):
#print "ticket is resolved"
if not self.is_risk_accepted(ticket_obj):
try:
if self.is_ticket_reopenable(ticket_obj):
comment = '''This ticket has been reopened due to the vulnerability not having been fixed (if multiple assets are affected, all need to be fixed; if the server is down, lastest known vulnerability might be the one reported).
In the case of the team accepting the risk and wanting to close the ticket, please add the label "*risk_accepted*" to the ticket before closing it.
If server has been decomissioned, please add the label "*server_decomission*" to the ticket before closing it.
If you have further doubts, please contact the Security Team.'''
error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_REOPEN_ISSUE, comment = comment)
print "[SUCCESS] ticket {} reopened successfully".format(ticketid)
#logger.info("ticket {} reopened successfully".format(ticketid))
return 1
except Exception as e:
# continue with ticket data so that a new ticket is created in place of the "lost" one
print "[ERROR] error reopening ticket {}: {}".format(ticketid, e)
#logger.error("error reopening ticket {}: {}".format(ticketid, e))
return 0
return 0
def close_ticket(self, ticketid, resolution, comment):
# this will close a ticket by ticketid
print "Ticket {} exists, CLOSE requested".format(ticketid)
ticket_obj = self.jira.issue(ticketid)
if not self.is_ticket_resolved(ticket_obj):
try:
if self.is_ticket_closeable(ticket_obj):
error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_CLOSE_ISSUE, comment = comment, resolution = {"name": resolution })
print "[SUCCESS] ticket {} closed successfully".format(ticketid)
#logger.info("ticket {} reopened successfully".format(ticketid))
return 1
except Exception as e:
# continue with ticket data so that a new ticket is created in place of the "lost" one
print "[ERROR] error closing ticket {}: {}".format(ticketid, e)
#logger.error("error closing ticket {}: {}".format(ticketid, e))
return 0
return 0
def close_obsolete_tickets(self):
# Close tickets older than 6 months, vulnerabilities not solved will get created a new ticket
print "Closing obsolete tickets older than {} months".format(self.max_time_tracking)
jql = "labels=vulnerability_management AND created <startOfMonth(-{}) and resolution=Unresolved".format(self.max_time_tracking)
tickets_to_close = self.jira.search_issues(jql, maxResults=0)
comment = '''This ticket is being closed for hygiene, as it is more than 6 months old.
If the vulnerability still exists, a new ticket will be opened.'''
for ticket in tickets_to_close:
self.close_ticket(ticket, self.JIRA_RESOLUTION_OBSOLETE, comment)
return 0
def project_exists(self, project):
try:
self.jira.project(project)
return True
except:
return False
return False

32
vulnwhisp/reporting/resources/ticket.tpl Normal file
View File

@ -0,0 +1,32 @@
{panel:title=Description}
{{ !diagnosis}}
{panel}
{panel:title=Consequence}
{{ !consequence}}
{panel}
{panel:title=Solution}
{{ !solution}}
{panel}
{panel:title=Affected Assets}
% for ip in ips:
* {{ip}}
% end
{panel}
{panel:title=References}
% for ref in references:
* {{ref}}
% end
{panel}
Please do not delete or modify the ticket assigned tags or title, as they are used to be synced. If the ticket ceases to be recognised, a new ticket will raise.
In the case of the team accepting the risk and wanting to close the ticket, please add the label "*risk_accepted*" to the ticket before closing it.
If server has been decomissioned, please add the label "*server_decomission*" to the ticket before closing it.

280
vulnwhisp/vulnwhisp.py
View File

@ -7,6 +7,7 @@ from frameworks.nessus import NessusAPI
from frameworks.qualys import qualysScanReport
from frameworks.qualys_vuln import qualysVulnScan
from frameworks.openvas import OpenVAS_API
from reporting.jira_api import JiraAPI
from utils.cli import bcolors
import pandas as pd
from lxml import objectify
@ -15,6 +16,7 @@ import os
import io
import time
import sqlite3
import json
# TODO Create logging option which stores data about scan
@ -49,7 +51,10 @@ class vulnWhispererBase(object):
if config is not None:
self.config = vwConfig(config_in=config)
self.enabled = self.config.get(self.CONFIG_SECTION, 'enabled')
try:
self.enabled = self.config.get(self.CONFIG_SECTION, 'enabled')
except:
self.enabled = False
self.hostname = self.config.get(self.CONFIG_SECTION, 'hostname')
self.username = self.config.get(self.CONFIG_SECTION, 'username')
self.password = self.config.get(self.CONFIG_SECTION, 'password')
@ -173,7 +178,46 @@ class vulnWhispererBase(object):
self.vprint('{info} Directory already exist for {scan} - Skipping creation'.format(
scan=self.write_path, info=bcolors.INFO))
def get_latest_results(self, source, scan_name):
try:
self.conn.text_factory = str
self.cur.execute('SELECT filename FROM scan_history WHERE source="{}" AND scan_name="{}" ORDER BY id DESC LIMIT 1;'.format(source, scan_name))
#should always return just one filename
results = [r[0] for r in self.cur.fetchall()][0]
except:
results = []
return results
return True
def get_scan_profiles(self):
# Returns a list of source.scan_name elements from the database
# we get the list of sources
try:
self.conn.text_factory = str
self.cur.execute('SELECT DISTINCT source FROM scan_history;')
sources = [r[0] for r in self.cur.fetchall()]
except:
sources = []
self.vprint("{fail} Process failed at executing 'SELECT DISTINCT source FROM scan_history;'".format(fail=bcolors.FAIL))
results = []
# we get the list of scans within each source
for source in sources:
scan_names = []
try:
self.conn.text_factory = str
self.cur.execute("SELECT DISTINCT scan_name FROM scan_history WHERE source='{}';".format(source))
scan_names = [r[0] for r in self.cur.fetchall()]
for scan in scan_names:
results.append('{}.{}'.format(source,scan))
except:
scan_names = []
return results
class vulnWhispererNessus(vulnWhispererBase):
@ -753,7 +797,7 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
class vulnWhispererQualysVuln(vulnWhispererBase):
CONFIG_SECTION = 'qualys'
CONFIG_SECTION = 'qualys_vuln'
COLUMN_MAPPING = {'cvss_base': 'cvss',
'cvss3_base': 'cvss3',
'cve_id': 'cve',
@ -875,6 +919,224 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
return 0
class vulnWhispererJIRA(vulnWhispererBase):
CONFIG_SECTION = 'jira'
def __init__(
self,
config=None,
db_name='report_tracker.db',
purge=False,
verbose=None,
debug=False,
username=None,
password=None,
):
super(vulnWhispererJIRA, self).__init__(config=config)
self.config_path = config
self.config = vwConfig(config)
if config is not None:
try:
self.vprint('{info} Attempting to connect to jira...'.format(info=bcolors.INFO))
self.jira = \
JiraAPI(hostname=self.hostname,
username=self.username,
password=self.password)
self.jira_connect = True
self.vprint('{success} Connected to jira on {host}'.format(success=bcolors.SUCCESS,
host=self.hostname))
except Exception as e:
self.vprint(e)
raise Exception(
'{fail} Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format(
config=self.config.config_in,
fail=bcolors.FAIL, e=e))
sys.exit(1)
profiles = []
profiles = self.get_scan_profiles()
if not self.config.exists_jira_profiles(profiles):
self.config.update_jira_profiles(profiles)
self.vprint("{info} Jira profiles have been created in {config}, please fill the variables before rerunning the module.".format(info=bcolors.INFO ,config=self.config_path))
sys.exit(0)
def get_env_variables(self, source, scan_name):
# function returns an array with [jira_project, jira_components, datafile_path]
#Jira variables
jira_section = self.config.normalize_section("{}.{}".format(source,scan_name))
project = self.config.get(jira_section,'jira_project')
if project == "":
self.vprint('{fail} JIRA project is missing on the configuration file!'.format(fail=bcolors.FAIL))
sys.exit(0)
# check that project actually exists
if not self.jira.project_exists(project):
self.vprint("{fail} JIRA project '{project}' doesn't exist!".format(fail=bcolors.FAIL, project=project))
sys.exit(0)
components = self.config.get(jira_section,'components').split(',')
#cleaning empty array from ''
if not components[0]:
components = []
#datafile path
filename = self.get_latest_results(source, scan_name)
# search data files under user specified directory
for root, dirnames, filenames in os.walk(vwConfig(self.config_path).get(source,'write_path')):
if filename in filenames:
fullpath = "{}/{}".format(root,filename)
if not fullpath:
self.vprint('{error} - Scan file path "{scan_name}" for source "{source}" has not been found.'.format(error=bcolors.FAIL, scan_name=scan_name, source=source))
return 0
return project, components, fullpath
def parse_nessus_vulnerabilities(self, fullpath, source, scan_name):
vulnerabilities = []
# we need to parse the CSV
excluded_risks = ['None','Low']
df = pd.read_csv(fullpath, delimiter=',')
#nessus fields we want - ['Host','Protocol','Port', 'Name', 'Synopsis', 'Description', 'Solution', 'See Also']
for index in range(len(df)):
# filtering vulnerabilities by criticality, discarding low risk
if df.loc[index]['Risk'] in excluded_risks:
continue
if not vulnerabilities or df.loc[index]['Name'] not in [entry['title'] for entry in vulnerabilities]:
vuln = {}
#vulnerabilities should have all the info for creating all JIRA labels
vuln['source'] = source
vuln['scan_name'] = scan_name
#vulnerability variables
vuln['title'] = df.loc[index]['Name']
vuln['diagnosis'] = df.loc[index]['Synopsis'].replace('\\n',' ')
vuln['consequence'] = df.loc[index]['Description'].replace('\\n',' ')
vuln['solution'] = df.loc[index]['Solution'].replace('\\n',' ')
vuln['ips'] = []
vuln['ips'].append("{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], df.loc[index]['Port']))
vuln['risk'] = df.loc[index]['Risk'].lower()
# Nessus "nan" value gets automatically casted to float by python
if not (type(df.loc[index]['See Also']) is float):
vuln['references'] = df.loc[index]['See Also'].split("\\n")
else:
vuln['references'] = []
vulnerabilities.append(vuln)
else:
# grouping assets by vulnerability to open on single ticket, as each asset has its own nessus entry
for vuln in vulnerabilities:
if vuln['title'] == df.loc[index]['Name']:
vuln['ips'].append("{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], df.loc[index]['Port']))
return vulnerabilities
def parse_qualys_vuln_vulnerabilities(self, fullpath, source, scan_name):
#parsing of the qualys vulnerabilities schema
#parse json
vulnerabilities = []
minimum_criticality_reported = 4
risks = ['info', 'low', 'medium', 'high', 'critical']
data=[json.loads(line) for line in open(fullpath).readlines()]
#qualys fields we want - []
for index in range(len(data)):
if int(data[index]['risk']) < 4:
continue
if not vulnerabilities or data[index]['plugin_name'] not in [entry['title'] for entry in vulnerabilities]:
vuln = {}
#vulnerabilities should have all the info for creating all JIRA labels
vuln['source'] = source
vuln['scan_name'] = scan_name
#vulnerability variables
vuln['title'] = data[index]['plugin_name']
vuln['diagnosis'] = data[index]['threat'].replace('\\n',' ')
vuln['consequence'] = data[index]['impact'].replace('\\n',' ')
vuln['solution'] = data[index]['solution'].replace('\\n',' ')
vuln['ips'] = []
#TODO ADDED DNS RESOLUTION FROM QUALYS! \n SEPARATORS INSTEAD OF \\n!
vuln['ips'].append("{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index])))
#different risk system than Nessus!
vuln['risk'] = risks[int(data[index]['risk'])-1]
# Nessus "nan" value gets automatically casted to float by python
if not (type(data[index]['vendor_reference']) is float or data[index]['vendor_reference'] == None):
vuln['references'] = data[index]['vendor_reference'].split("\\n")
else:
vuln['references'] = []
vulnerabilities.append(vuln)
else:
# grouping assets by vulnerability to open on single ticket, as each asset has its own nessus entry
for vuln in vulnerabilities:
if vuln['title'] == data[index]['plugin_name']:
vuln['ips'].append("{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index])))
return vulnerabilities
def get_asset_fields(self, vuln):
values = {}
values['ip'] = vuln['ip']
values['protocol'] = vuln['protocol']
values['port'] = vuln['port']
values['dns'] = vuln['dns']
for key in values.keys():
if not values[key]:
values[key] = 'N/A'
return values
def parse_vulnerabilities(self, fullpath, source, scan_name):
#TODO: SINGLE LOCAL SAVE FORMAT FOR ALL SCANNERS
#JIRA standard vuln format - ['source', 'scan_name', 'title', 'diagnosis', 'consequence', 'solution', 'ips', 'references']
return 0
def jira_sync(self, source, scan_name):
project, components, fullpath = self.get_env_variables(source, scan_name)
vulnerabilities = []
#***Nessus parsing***
if source == "nessus":
vulnerabilities = self.parse_nessus_vulnerabilities(fullpath, source, scan_name)
#***Qualys VM parsing***
if source == "qualys_vuln":
vulnerabilities = self.parse_qualys_vuln_vulnerabilities(fullpath, source, scan_name)
#***JIRA sync***
if vulnerabilities:
self.vprint('{info} {source} data has been successfuly parsed'.format(info=bcolors.INFO, source=source.upper()))
self.vprint('{info} Starting JIRA sync'.format(info=bcolors.INFO))
self.jira.sync(vulnerabilities, project, components)
else:
self.vprint("{fail} Vulnerabilities from {source} has not been parsed! Exiting...".format(fail=bcolors.FAIL, source=source))
sys.exit(0)
return True
class vulnWhisperer(object):
def __init__(self,
@ -882,13 +1144,17 @@ class vulnWhisperer(object):
verbose=None,
username=None,
password=None,
config=None):
config=None,
source=None,
scanname=None):
self.profile = profile
self.config = config
self.username = username
self.password = password
self.verbose = verbose
self.source = source
self.scanname = scanname
def whisper_vulnerabilities(self):
@ -920,3 +1186,11 @@ class vulnWhisperer(object):
elif self.profile == 'qualys_vuln':
vw = vulnWhispererQualysVuln(config=self.config)
vw.process_vuln_scans()
elif self.profile == 'jira':
#first we check config fields are created, otherwise we create them
vw = vulnWhispererJIRA(config=self.config)
if not (self.source and self.scanname):
print('{error} - Source scanner and scan name needed!'.format(error=bcolors.FAIL))
return 0
vw.jira_sync(self.source, self.scanname)