Cyberdefense/VulnWhisperer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Jira extras (#120)

Browse Source 
* changing config template paths for qualys

* Update frameworks_example.ini

Will leave for now qualys local folder as "qualys" instead of changing to one for each module, as like this it will still be compatible with the current logstash and we will be able to update master to drop the qualysapi fork once the new version is uploaded to PyPI repository.
PR from qualysapi repo has already been merged, so the only missing is the upload to PyPI.

* initialize variable fullpath to avoid break

* fix get latest scan entry from db and ignore 'potential' not verified vulns

* added host resolv + cache to speed already resolved, jira logging

* make sure that vulnerability criticality appears as a label on ticket + automatic actions

* jira bulk report of scans, fix on nessus logging, jira time resolution and list all ticket reported assets

* added jira ticket data download + change default time window from 6 to 12 months

* small fixes

* jira logstash files

* fix variable confusion (thx Travis :)
This commit is contained in:
Quim Montal
2018-11-08 09:24:24 +01:00
committed by GitHub
parent 0b571799dc
commit 8bd3c5cab9
8 changed files with 185 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

18
vulnwhisp/base/config.py
View File

@ -25,17 +25,17 @@ class vwConfig(object):
self.logger.debug('Calling getbool for {}:{}'.format(section, option))
return self.config.getboolean(section, option)
def get_enabled(self):
enabled = []
def get_sections_with_attribute(self, attribute):
sections = []
# TODO: does this not also need the "yes" case?
check = ["true", "True", "1"]
for section in self.config.sections():
try:
if self.get(section, "enabled") in check:
enabled.append(section)
if self.get(section, attribute) in check:
sections.append(section)
except:
self.logger.error("Section {} has no option 'enabled'".format(section))
return enabled
self.logger.warn("Section {} has no option '{}'".format(section, attribute))
return sections
def exists_jira_profiles(self, profiles):
# get list of profiles source_scanner.scan_name
@ -62,11 +62,13 @@ class vwConfig(object):
self.config.set(section_name,'source',profile.split('.')[0])
# in case any scan name contains '.' character
self.config.set(section_name,'scan_name','.'.join(profile.split('.')[1:]))
self.config.set(section_name,'jira_project', "")
self.config.set(section_name,'jira_project', '')
self.config.set(section_name,'; if multiple components, separate by ","')
self.config.set(section_name,'components', "")
self.config.set(section_name,'components', '')
self.config.set(section_name,'; minimum criticality to report (low, medium, high or critical)')
self.config.set(section_name,'min_critical_to_report', 'high')
self.config.set(section_name,'; automatically report, boolean value ')
self.config.set(section_name,'autoreport', 'false')
# TODO: try/catch this
# writing changes back to file

2
vulnwhisp/frameworks/nessus.py
View File

@ -69,7 +69,7 @@ class NessusAPI(object):
success = False
url = self.base + url
self.logging.debug('Requesting to url {}'.format(url))
self.logger.debug('Requesting to url {}'.format(url))
methods = {'GET': requests.get,
'POST': requests.post,
'DELETE': requests.delete}

98
vulnwhisp/reporting/jira_api.py
View File

@ -1,5 +1,6 @@
import json
from datetime import datetime, timedelta
import os
from datetime import datetime, date, timedelta
from jira import JIRA
import requests
@ -8,7 +9,7 @@ from bottle import template
import re
class JiraAPI(object):
def __init__(self, hostname=None, username=None, password=None, debug=False, clean_obsolete=True, max_time_window=6):
def __init__(self, hostname=None, username=None, password=None, path="", debug=False, clean_obsolete=True, max_time_window=12):
self.logger = logging.getLogger('JiraAPI')
if debug:
self.logger.setLevel(logging.DEBUG)
@ -28,7 +29,11 @@ class JiraAPI(object):
self.JIRA_RESOLUTION_FIXED = "Fixed"
self.clean_obsolete = clean_obsolete
self.template_path = 'vulnwhisp/reporting/resources/ticket.tpl'
if path:
self.download_tickets(path)
else:
self.logger.warn("No local path specified, skipping Jira ticket download.")
def create_ticket(self, title, desc, project="IS", components=[], tags=[]):
labels = ['vulnerability_management']
for tag in tags:
@ -56,7 +61,7 @@ class JiraAPI(object):
labels=labels,
components=components_ticket)
self.logger.info("Ticket {} has been created".format(new_issue))
self.logger.info("Ticket {} created successfully".format(new_issue))
return new_issue
#Basic JIRA Metrics
@ -77,7 +82,7 @@ class JiraAPI(object):
#JIRA structure of each vulnerability: [source, scan_name, title, diagnosis, consequence, solution, ips, risk, references]
self.logger.info("JIRA Sync started")
# [HIGIENE] close tickets older than 6 months as obsolete
# [HIGIENE] close tickets older than 12 months as obsolete
# Higiene clean up affects to all tickets created by the module, filters by label 'vulnerability_management'
if self.clean_obsolete:
self.close_obsolete_tickets()
@ -97,9 +102,11 @@ class JiraAPI(object):
if exists:
# If ticket "resolved" -> reopen, as vulnerability is still existent
self.reopen_ticket(ticketid)
self.add_label(ticketid, vuln['risk'])
continue
elif to_update:
self.ticket_update_assets(vuln, ticketid, ticket_assets)
self.add_label(ticketid, vuln['risk'])
continue
try:
@ -125,7 +132,7 @@ class JiraAPI(object):
if not self.all_tickets:
self.logger.info("Retrieving all JIRA tickets with the following tags {}".format(labels))
# we want to check all JIRA tickets, to include tickets moved to other queues
# will exclude tickets older than 6 months, old tickets will get closed for higiene and recreated if still vulnerable
# will exclude tickets older than 12 months, old tickets will get closed for higiene and recreated if still vulnerable
jql = "{} AND NOT labels=advisory AND created >=startOfMonth(-{})".format(" AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
self.all_tickets = self.jira.search_issues(jql, maxResults=0)
@ -139,7 +146,7 @@ class JiraAPI(object):
difference = list(set(assets).symmetric_difference(checking_assets))
#to check intersection - set(assets) & set(checking_assets)
if difference:
self.logger.info("Asset mismatch, ticket to update. TickedID: {}".format(checking_ticketid))
self.logger.info("Asset mismatch, ticket to update. Ticket ID: {}".format(checking_ticketid))
return False, True, checking_ticketid, checking_assets #this will automatically validate
else:
self.logger.info("Confirmed duplicated. TickedID: {}".format(checking_ticketid))
@ -158,6 +165,28 @@ class JiraAPI(object):
return ticketid, title, assets
def get_ticket_reported_assets(self, ticket):
#[METRICS] return a list with all the affected assets for that vulnerability (including already resolved ones)
return list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b",str(self.jira.issue(ticket).raw))))
def get_resolution_time(self, ticket):
#get time a ticket took to be resolved
ticket_obj = self.jira.issue(ticket)
if self.is_ticket_resolved(ticket_obj):
ticket_data = ticket_obj.raw.get('fields')
#dates follow format '2018-11-06T10:36:13.849+0100'
created = [int(x) for x in ticket_data['created'].split('.')[0].replace('T', '-').replace(':','-').split('-')]
resolved =[int(x) for x in ticket_data['resolutiondate'].split('.')[0].replace('T', '-').replace(':','-').split('-')]
start = datetime(created[0],created[1],created[2],created[3],created[4],created[5])
end = datetime(resolved[0],resolved[1],resolved[2],resolved[3],resolved[4],resolved[5])
return (end-start).days
else:
self.logger.error("Ticket {ticket} is not resolved, can't calculate resolution time".format(ticket=ticket))
return False
def ticket_update_assets(self, vuln, ticketid, ticket_assets):
# correct description will always be in the vulnerability to report, only needed to update description to new one
self.logger.info("Ticket {} exists, UPDATE requested".format(ticketid))
@ -182,15 +211,28 @@ class JiraAPI(object):
comment += "Asset {} have been added to the ticket as vulnerability *has been newly detected*.\n".format(asset)
elif asset in ticket_assets:
comment += "Asset {} have been removed from the ticket as vulnerability *has been resolved*.\n".format(asset)
ticket_obj.fields.labels.append('updated')
try:
ticket_obj.update(description=tpl, comment=comment, fields={"labels":ticket_obj.fields.labels})
self.logger.info("Ticket {} updated successfully".format(ticketid))
self.add_label(ticketid, 'updated')
except:
self.logger.error("Error while trying up update ticket {}".format(ticketid))
return 0
def add_label(self, ticketid, label):
ticket_obj = self.jira.issue(ticketid)
if label not in ticket_obj.fields.labels:
ticket_obj.fields.labels.append(label)
try:
ticket_obj.update(fields={"labels":ticket_obj.fields.labels})
self.logger.info("Added label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
except:
self.logger.error("Error while trying to add label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
return 0
def close_fixed_tickets(self, vulnerabilities):
# close tickets which vulnerabilities have been resolved and are still open
found_vulns = []
@ -232,7 +274,7 @@ class JiraAPI(object):
if ticket_obj.raw['fields'].get('resolution') is not None:
if ticket_obj.raw['fields'].get('resolution').get('name') != 'Unresolved':
self.logger.debug("Checked ticket {} is already closed".format(ticket_obj))
self.logger.info("ticket {} is closed".format(ticket_obj.id))
self.logger.info("Ticket {} is closed".format(ticket_obj))
return True
self.logger.debug("Checked ticket {} is already open".format(ticket_obj))
return False
@ -265,7 +307,8 @@ class JiraAPI(object):
If server has been decomissioned, please add the label "*server_decomission*" to the ticket before closing it.
If you have further doubts, please contact the Security Team.'''
error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_REOPEN_ISSUE, comment = comment)
self.logger.info("ticket {} reopened successfully".format(ticketid))
self.logger.info("Ticket {} reopened successfully".format(ticketid))
self.add_label(ticketid, 'reopened')
return 1
except Exception as e:
# continue with ticket data so that a new ticket is created in place of the "lost" one
@ -280,8 +323,10 @@ class JiraAPI(object):
if not self.is_ticket_resolved(ticket_obj):
try:
if self.is_ticket_closeable(ticket_obj):
#need to add the label before closing the ticket
self.add_label(ticketid, 'closed')
error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_CLOSE_ISSUE, comment = comment, resolution = {"name": resolution })
self.logger.info("ticket {} reopened successfully".format(ticketid))
self.logger.info("Ticket {} closed successfully".format(ticketid))
return 1
except Exception as e:
# continue with ticket data so that a new ticket is created in place of the "lost" one
@ -291,12 +336,12 @@ class JiraAPI(object):
return 0
def close_obsolete_tickets(self):
# Close tickets older than 6 months, vulnerabilities not solved will get created a new ticket
# Close tickets older than 12 months, vulnerabilities not solved will get created a new ticket
self.logger.info("Closing obsolete tickets older than {} months".format(self.max_time_tracking))
jql = "labels=vulnerability_management AND created <startOfMonth(-{}) and resolution=Unresolved".format(self.max_time_tracking)
tickets_to_close = self.jira.search_issues(jql, maxResults=0)
comment = '''This ticket is being closed for hygiene, as it is more than 6 months old.
comment = '''This ticket is being closed for hygiene, as it is more than 12 months old.
If the vulnerability still exists, a new ticket will be opened.'''
for ticket in tickets_to_close:
@ -312,3 +357,28 @@ class JiraAPI(object):
return False
return False
def download_tickets(self, path):
#saves all tickets locally, local snapshot of vulnerability_management ticktes
#check if file already exists
check_date = str(date.today())
fname = '{}jira_{}.json'.format(path, check_date)
if os.path.isfile(fname):
self.logger.info("File {} already exists, skipping ticket download".format(fname))
return True
try:
self.logger.info("Saving locally tickets from the last {} months".format(self.max_time_tracking))
jql = "labels=vulnerability_management AND created >=startOfMonth(-{})".format(self.max_time_tracking)
tickets_data = self.jira.search_issues(jql, maxResults=0)
#end of line needed, as writelines() doesn't add it automatically, otherwise one big line
to_save = [json.dumps(ticket.raw.get('fields'))+"\n" for ticket in tickets_data]
with open(fname, 'w') as outfile:
outfile.writelines(to_save)
self.logger.info("Tickets saved succesfully.")
return True
except Exception as e:
self.logger.error("Tickets could not be saved locally: {}.".format(e))
return False

54
vulnwhisp/vulnwhisp.py
View File

@ -17,6 +17,7 @@ import time
import sqlite3
import json
import logging
import socket
class vulnWhispererBase(object):
@ -172,7 +173,7 @@ class vulnWhispererBase(object):
def get_latest_results(self, source, scan_name):
try:
self.conn.text_factory = str
self.cur.execute('SELECT filename FROM scan_history WHERE source="{}" AND scan_name="{}" ORDER BY id DESC LIMIT 1;'.format(source, scan_name))
self.cur.execute('SELECT filename FROM scan_history WHERE source="{}" AND scan_name="{}" ORDER BY last_modified DESC LIMIT 1;'.format(source, scan_name))
#should always return just one filename
results = [r[0] for r in self.cur.fetchall()][0]
except:
@ -915,7 +916,8 @@ class vulnWhispererJIRA(vulnWhispererBase):
self.logger.setLevel(logging.DEBUG)
self.config_path = config
self.config = vwConfig(config)
self.host_resolv_cache = {}
self.directory_check()
if config is not None:
try:
@ -923,7 +925,8 @@ class vulnWhispererJIRA(vulnWhispererBase):
self.jira = \
JiraAPI(hostname=self.hostname,
username=self.username,
password=self.password)
password=self.password,
path=self.config.get('jira','write_path'))
self.jira_connect = True
self.logger.info('Connected to jira on {host}'.format(host=self.hostname))
except Exception as e:
@ -971,6 +974,7 @@ class vulnWhispererJIRA(vulnWhispererBase):
#datafile path
filename = self.get_latest_results(source, scan_name)
fullpath = ""
# search data files under user specified directory
for root, dirnames, filenames in os.walk(vwConfig(self.config_path).get(source,'write_path')):
@ -979,7 +983,7 @@ class vulnWhispererJIRA(vulnWhispererBase):
if not fullpath:
self.logger.error('Scan file path "{scan_name}" for source "{source}" has not been found.'.format(scan_name=scan_name, source=source))
return 0
sys.exit(1)
return project, components, fullpath, min_critical
@ -1045,6 +1049,10 @@ class vulnWhispererJIRA(vulnWhispererBase):
for index in range(len(data)):
if int(data[index]['risk']) < min_risk:
continue
elif data[index]['type'] == 'Practice' or data[index]['type'] == 'Ig':
self.logger.debug("Vulnerability '{vuln}' ignored, as it is 'Practice/Potential', not verified.".format(vuln=data[index]['plugin_name']))
continue
if not vulnerabilities or data[index]['plugin_name'] not in [entry['title'] for entry in vulnerabilities]:
vuln = {}
@ -1083,7 +1091,23 @@ class vulnWhispererJIRA(vulnWhispererBase):
values['ip'] = vuln['ip']
values['protocol'] = vuln['protocol']
values['port'] = vuln['port']
values['dns'] = vuln['dns']
values['dns'] = ''
if vuln['dns']:
values['dns'] = vuln['dns']
else:
if values['ip'] in self.host_resolv_cache.keys():
self.logger.debug("Hostname from {ip} cached, retrieving from cache.".format(ip=values['ip']))
values['dns'] = self.host_resolv_cache[values['ip']]
else:
self.logger.debug("No hostname, trying to resolve {ip}'s hostname.".format(ip=values['ip']))
try:
values['dns'] = socket.gethostbyaddr(vuln['ip'])[0]
self.host_resolv_cache[values['ip']] = values['dns']
self.logger.debug("Hostname found: {hostname}.".format(hostname=values['dns']))
except:
self.host_resolv_cache[values['ip']] = ''
self.logger.debug("Hostname not found for: {ip}.".format(ip=values['ip']))
for key in values.keys():
if not values[key]:
values[key] = 'N/A'
@ -1098,6 +1122,7 @@ class vulnWhispererJIRA(vulnWhispererBase):
def jira_sync(self, source, scan_name):
self.logger.info("Jira Sync triggered for source '{source}' and scan '{scan_name}'".format(source=source, scan_name=scan_name))
project, components, fullpath, min_critical = self.get_env_variables(source, scan_name)
@ -1123,6 +1148,14 @@ class vulnWhispererJIRA(vulnWhispererBase):
return True
def sync_all(self):
autoreport_sections = self.config.get_sections_with_attribute('autoreport')
if autoreport_sections:
for scan in autoreport_sections:
self.jira_sync(self.config.get(scan, 'source'), self.config.get(scan, 'scan_name'))
return True
return False
class vulnWhisperer(object):
@ -1181,6 +1214,11 @@ class vulnWhisperer(object):
#first we check config fields are created, otherwise we create them
vw = vulnWhispererJIRA(config=self.config)
if not (self.source and self.scanname):
self.logger.error('Source scanner and scan name needed!')
return 0
vw.jira_sync(self.source, self.scanname)
self.logger.info('No source/scan_name selected, all enabled scans will be synced')
success = vw.sync_all()
if not success:
self.logger.error('All scans sync failed!')
self.logger.error('Source scanner and scan name needed!')
return 0
else:
vw.jira_sync(self.source, self.scanname)