From 8bd3c5cab99395ecc11da9ee9133a24da0ad6de5 Mon Sep 17 00:00:00 2001 From: Quim Montal Date: Thu, 8 Nov 2018 09:24:24 +0100 Subject: [PATCH] Jira extras (#120) * changing config template paths for qualys * Update frameworks_example.ini Will leave for now qualys local folder as "qualys" instead of changing to one for each module, as like this it will still be compatible with the current logstash and we will be able to update master to drop the qualysapi fork once the new version is uploaded to PyPI repository. PR from qualysapi repo has already been merged, so the only missing is the upload to PyPI. * initialize variable fullpath to avoid break * fix get latest scan entry from db and ignore 'potential' not verified vulns * added host resolv + cache to speed already resolved, jira logging * make sure that vulnerability criticality appears as a label on ticket + automatic actions * jira bulk report of scans, fix on nessus logging, jira time resolution and list all ticket reported assets * added jira ticket data download + change default time window from 6 to 12 months * small fixes * jira logstash files * fix variable confusion (thx Travis :) --- bin/vuln_whisperer | 2 +- docker-compose.yml | 1 + docker/4000_jira.conf | 21 +++++++ logstash/4000_jira.conf | 21 +++++++ vulnwhisp/base/config.py | 18 +++--- vulnwhisp/frameworks/nessus.py | 2 +- vulnwhisp/reporting/jira_api.py | 98 ++++++++++++++++++++++++++++----- vulnwhisp/vulnwhisp.py | 54 +++++++++++++++--- 8 files changed, 185 insertions(+), 32 deletions(-) create mode 100755 docker/4000_jira.conf create mode 100644 logstash/4000_jira.conf diff --git a/bin/vuln_whisperer b/bin/vuln_whisperer index 36b6d6d..d94b5ce 100644 --- a/bin/vuln_whisperer +++ b/bin/vuln_whisperer @@ -55,7 +55,7 @@ def main(): \nExample vuln_whisperer -c config.ini -s nessus')) logger.info('No section was specified, vulnwhisperer will scrape enabled modules from the config file.') config = vwConfig(config_in=args.config) - enabled_sections = config.get_enabled() + enabled_sections = config.get_sections_with_attribute('enabled') for section in enabled_sections: vw = vulnWhisperer(config=args.config, diff --git a/docker-compose.yml b/docker-compose.yml index 66451a3..a0c186f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -46,6 +46,7 @@ services: - ./docker/1000_nessus_process_file.conf:/usr/share/logstash/pipeline/1000_nessus_process_file.conf - ./docker/2000_qualys_web_scans.conf:/usr/share/logstash/pipeline/2000_qualys_web_scans.conf - ./docker/3000_openvas.conf:/usr/share/logstash/pipeline/3000_openvas.conf + - ./docker/4000_jira.conf:/usr/share/logstash/pipeline/4000_jira.conf - ./docker/logstash.yml:/usr/share/logstash/config/logstash.yml - ./data/:/opt/VulnWhisperer/data environment: diff --git a/docker/4000_jira.conf b/docker/4000_jira.conf new file mode 100755 index 0000000..a9f4966 --- /dev/null +++ b/docker/4000_jira.conf @@ -0,0 +1,21 @@ +# Description: Take in jira tickets from vulnWhisperer and pumps into logstash + +input { + file { + path => "/opt/Vulnwhisperer/jira/*.json" + type => json + codec => json + start_position => "beginning" + tags => [ "jira" ] + } +} + +output { + if "jira" in [tags] { + stdout { codec => rubydebug } + elasticsearch { + hosts => [ "vulnwhisp-es1.local:9200" ] + index => "logstash-vulnwhisperer-%{+YYYY.MM}" + } + } +} diff --git a/logstash/4000_jira.conf b/logstash/4000_jira.conf new file mode 100644 index 0000000..e4106c7 --- /dev/null +++ b/logstash/4000_jira.conf @@ -0,0 +1,21 @@ +# Description: Take in jira tickets from vulnWhisperer and pumps into logstash + +input { + file { + path => "/opt/vulnwhisperer/jira/*.json" + type => json + codec => json + start_position => "beginning" + tags => [ "jira" ] + } +} + +output { + if "jira" in [tags] { + stdout { codec => rubydebug } + elasticsearch { + hosts => [ "localhost:9200" ] + index => "logstash-vulnwhisperer-%{+YYYY.MM}" + } + } +} diff --git a/vulnwhisp/base/config.py b/vulnwhisp/base/config.py index 832a1b9..04cbb2a 100644 --- a/vulnwhisp/base/config.py +++ b/vulnwhisp/base/config.py @@ -25,17 +25,17 @@ class vwConfig(object): self.logger.debug('Calling getbool for {}:{}'.format(section, option)) return self.config.getboolean(section, option) - def get_enabled(self): - enabled = [] + def get_sections_with_attribute(self, attribute): + sections = [] # TODO: does this not also need the "yes" case? check = ["true", "True", "1"] for section in self.config.sections(): try: - if self.get(section, "enabled") in check: - enabled.append(section) + if self.get(section, attribute) in check: + sections.append(section) except: - self.logger.error("Section {} has no option 'enabled'".format(section)) - return enabled + self.logger.warn("Section {} has no option '{}'".format(section, attribute)) + return sections def exists_jira_profiles(self, profiles): # get list of profiles source_scanner.scan_name @@ -62,11 +62,13 @@ class vwConfig(object): self.config.set(section_name,'source',profile.split('.')[0]) # in case any scan name contains '.' character self.config.set(section_name,'scan_name','.'.join(profile.split('.')[1:])) - self.config.set(section_name,'jira_project', "") + self.config.set(section_name,'jira_project', '') self.config.set(section_name,'; if multiple components, separate by ","') - self.config.set(section_name,'components', "") + self.config.set(section_name,'components', '') self.config.set(section_name,'; minimum criticality to report (low, medium, high or critical)') self.config.set(section_name,'min_critical_to_report', 'high') + self.config.set(section_name,'; automatically report, boolean value ') + self.config.set(section_name,'autoreport', 'false') # TODO: try/catch this # writing changes back to file diff --git a/vulnwhisp/frameworks/nessus.py b/vulnwhisp/frameworks/nessus.py index 3e7a2ad..09ce24e 100755 --- a/vulnwhisp/frameworks/nessus.py +++ b/vulnwhisp/frameworks/nessus.py @@ -69,7 +69,7 @@ class NessusAPI(object): success = False url = self.base + url - self.logging.debug('Requesting to url {}'.format(url)) + self.logger.debug('Requesting to url {}'.format(url)) methods = {'GET': requests.get, 'POST': requests.post, 'DELETE': requests.delete} diff --git a/vulnwhisp/reporting/jira_api.py b/vulnwhisp/reporting/jira_api.py index 9316b53..a869a50 100644 --- a/vulnwhisp/reporting/jira_api.py +++ b/vulnwhisp/reporting/jira_api.py @@ -1,5 +1,6 @@ import json -from datetime import datetime, timedelta +import os +from datetime import datetime, date, timedelta from jira import JIRA import requests @@ -8,7 +9,7 @@ from bottle import template import re class JiraAPI(object): - def __init__(self, hostname=None, username=None, password=None, debug=False, clean_obsolete=True, max_time_window=6): + def __init__(self, hostname=None, username=None, password=None, path="", debug=False, clean_obsolete=True, max_time_window=12): self.logger = logging.getLogger('JiraAPI') if debug: self.logger.setLevel(logging.DEBUG) @@ -28,7 +29,11 @@ class JiraAPI(object): self.JIRA_RESOLUTION_FIXED = "Fixed" self.clean_obsolete = clean_obsolete self.template_path = 'vulnwhisp/reporting/resources/ticket.tpl' - + if path: + self.download_tickets(path) + else: + self.logger.warn("No local path specified, skipping Jira ticket download.") + def create_ticket(self, title, desc, project="IS", components=[], tags=[]): labels = ['vulnerability_management'] for tag in tags: @@ -56,7 +61,7 @@ class JiraAPI(object): labels=labels, components=components_ticket) - self.logger.info("Ticket {} has been created".format(new_issue)) + self.logger.info("Ticket {} created successfully".format(new_issue)) return new_issue #Basic JIRA Metrics @@ -77,7 +82,7 @@ class JiraAPI(object): #JIRA structure of each vulnerability: [source, scan_name, title, diagnosis, consequence, solution, ips, risk, references] self.logger.info("JIRA Sync started") - # [HIGIENE] close tickets older than 6 months as obsolete + # [HIGIENE] close tickets older than 12 months as obsolete # Higiene clean up affects to all tickets created by the module, filters by label 'vulnerability_management' if self.clean_obsolete: self.close_obsolete_tickets() @@ -97,9 +102,11 @@ class JiraAPI(object): if exists: # If ticket "resolved" -> reopen, as vulnerability is still existent self.reopen_ticket(ticketid) + self.add_label(ticketid, vuln['risk']) continue elif to_update: self.ticket_update_assets(vuln, ticketid, ticket_assets) + self.add_label(ticketid, vuln['risk']) continue try: @@ -125,7 +132,7 @@ class JiraAPI(object): if not self.all_tickets: self.logger.info("Retrieving all JIRA tickets with the following tags {}".format(labels)) # we want to check all JIRA tickets, to include tickets moved to other queues - # will exclude tickets older than 6 months, old tickets will get closed for higiene and recreated if still vulnerable + # will exclude tickets older than 12 months, old tickets will get closed for higiene and recreated if still vulnerable jql = "{} AND NOT labels=advisory AND created >=startOfMonth(-{})".format(" AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking) self.all_tickets = self.jira.search_issues(jql, maxResults=0) @@ -139,7 +146,7 @@ class JiraAPI(object): difference = list(set(assets).symmetric_difference(checking_assets)) #to check intersection - set(assets) & set(checking_assets) if difference: - self.logger.info("Asset mismatch, ticket to update. TickedID: {}".format(checking_ticketid)) + self.logger.info("Asset mismatch, ticket to update. Ticket ID: {}".format(checking_ticketid)) return False, True, checking_ticketid, checking_assets #this will automatically validate else: self.logger.info("Confirmed duplicated. TickedID: {}".format(checking_ticketid)) @@ -158,6 +165,28 @@ class JiraAPI(object): return ticketid, title, assets + def get_ticket_reported_assets(self, ticket): + #[METRICS] return a list with all the affected assets for that vulnerability (including already resolved ones) + return list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b",str(self.jira.issue(ticket).raw)))) + + def get_resolution_time(self, ticket): + #get time a ticket took to be resolved + ticket_obj = self.jira.issue(ticket) + if self.is_ticket_resolved(ticket_obj): + ticket_data = ticket_obj.raw.get('fields') + #dates follow format '2018-11-06T10:36:13.849+0100' + created = [int(x) for x in ticket_data['created'].split('.')[0].replace('T', '-').replace(':','-').split('-')] + resolved =[int(x) for x in ticket_data['resolutiondate'].split('.')[0].replace('T', '-').replace(':','-').split('-')] + + start = datetime(created[0],created[1],created[2],created[3],created[4],created[5]) + end = datetime(resolved[0],resolved[1],resolved[2],resolved[3],resolved[4],resolved[5]) + + return (end-start).days + else: + self.logger.error("Ticket {ticket} is not resolved, can't calculate resolution time".format(ticket=ticket)) + + return False + def ticket_update_assets(self, vuln, ticketid, ticket_assets): # correct description will always be in the vulnerability to report, only needed to update description to new one self.logger.info("Ticket {} exists, UPDATE requested".format(ticketid)) @@ -182,15 +211,28 @@ class JiraAPI(object): comment += "Asset {} have been added to the ticket as vulnerability *has been newly detected*.\n".format(asset) elif asset in ticket_assets: comment += "Asset {} have been removed from the ticket as vulnerability *has been resolved*.\n".format(asset) - - ticket_obj.fields.labels.append('updated') + try: ticket_obj.update(description=tpl, comment=comment, fields={"labels":ticket_obj.fields.labels}) self.logger.info("Ticket {} updated successfully".format(ticketid)) + self.add_label(ticketid, 'updated') except: self.logger.error("Error while trying up update ticket {}".format(ticketid)) return 0 + def add_label(self, ticketid, label): + ticket_obj = self.jira.issue(ticketid) + + if label not in ticket_obj.fields.labels: + ticket_obj.fields.labels.append(label) + + try: + ticket_obj.update(fields={"labels":ticket_obj.fields.labels}) + self.logger.info("Added label {label} to ticket {ticket}".format(label=label, ticket=ticketid)) + except: + self.logger.error("Error while trying to add label {label} to ticket {ticket}".format(label=label, ticket=ticketid)) + return 0 + def close_fixed_tickets(self, vulnerabilities): # close tickets which vulnerabilities have been resolved and are still open found_vulns = [] @@ -232,7 +274,7 @@ class JiraAPI(object): if ticket_obj.raw['fields'].get('resolution') is not None: if ticket_obj.raw['fields'].get('resolution').get('name') != 'Unresolved': self.logger.debug("Checked ticket {} is already closed".format(ticket_obj)) - self.logger.info("ticket {} is closed".format(ticket_obj.id)) + self.logger.info("Ticket {} is closed".format(ticket_obj)) return True self.logger.debug("Checked ticket {} is already open".format(ticket_obj)) return False @@ -265,7 +307,8 @@ class JiraAPI(object): If server has been decomissioned, please add the label "*server_decomission*" to the ticket before closing it. If you have further doubts, please contact the Security Team.''' error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_REOPEN_ISSUE, comment = comment) - self.logger.info("ticket {} reopened successfully".format(ticketid)) + self.logger.info("Ticket {} reopened successfully".format(ticketid)) + self.add_label(ticketid, 'reopened') return 1 except Exception as e: # continue with ticket data so that a new ticket is created in place of the "lost" one @@ -280,8 +323,10 @@ class JiraAPI(object): if not self.is_ticket_resolved(ticket_obj): try: if self.is_ticket_closeable(ticket_obj): + #need to add the label before closing the ticket + self.add_label(ticketid, 'closed') error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_CLOSE_ISSUE, comment = comment, resolution = {"name": resolution }) - self.logger.info("ticket {} reopened successfully".format(ticketid)) + self.logger.info("Ticket {} closed successfully".format(ticketid)) return 1 except Exception as e: # continue with ticket data so that a new ticket is created in place of the "lost" one @@ -291,12 +336,12 @@ class JiraAPI(object): return 0 def close_obsolete_tickets(self): - # Close tickets older than 6 months, vulnerabilities not solved will get created a new ticket + # Close tickets older than 12 months, vulnerabilities not solved will get created a new ticket self.logger.info("Closing obsolete tickets older than {} months".format(self.max_time_tracking)) jql = "labels=vulnerability_management AND created