Cyberdefense/VulnWhisperer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

modernize python2 to python3 applied

Browse Source
This commit is contained in:
Quim
2020-03-03 08:48:00 +01:00
parent 4974be02b4
commit 97de805e0c
10 changed files with 1546 additions and 1497 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
bin/vuln_whisperer
View File

@ -93,7 +93,7 @@ def main():
scanname=args.scanname)
exit_code += vw.whisper_vulnerabilities()
except Exception as e:
logger.error("VulnWhisperer was unable to perform the processing on '{}'".format(args.source))
logger.error("VulnWhisperer was unable to perform the processing on '{}'".format(section))
else:
logger.info('Running vulnwhisperer for section {}'.format(args.section))
vw = vulnWhisperer(config=args.config,

1
setup.py
View File

@ -1,5 +1,6 @@
#!/usr/bin/env python
from __future__ import absolute_import
from setuptools import setup, find_packages
setup(

3
vulnwhisp/base/config.py
View File

@ -1,3 +1,4 @@
from __future__ import absolute_import
import sys
import logging
@ -5,7 +6,7 @@ import logging
if sys.version_info > (3, 0):
import configparser as cp
else:
import ConfigParser as cp
import six.moves.configparser as cp
class vwConfig(object):

1
vulnwhisp/frameworks/nessus.py
View File

@ -1,3 +1,4 @@
from __future__ import absolute_import
import json
import logging
import sys

1
vulnwhisp/frameworks/openvas.py
View File

@ -1,5 +1,6 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import absolute_import
__author__ = 'Austin Taylor'
import datetime as dt

1
vulnwhisp/frameworks/qualys_vuln.py
View File

@ -1,5 +1,6 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import absolute_import
__author__ = 'Nathan Young'
import logging

3
vulnwhisp/frameworks/qualys_web.py
View File

@ -1,5 +1,8 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from six.moves import range
from functools import reduce
__author__ = 'Austin Taylor'
from lxml import objectify

115
vulnwhisp/reporting/jira_api.py
View File

@ -1,15 +1,18 @@
from __future__ import absolute_import
import json
import os
from datetime import datetime, date, timedelta
from datetime import datetime, date
from jira import JIRA
import requests
import logging
from bottle import template
import re
from six.moves import range
class JiraAPI(object):
def __init__(self, hostname=None, username=None, password=None, path="", debug=False, clean_obsolete=True, max_time_window=12, decommission_time_window=3):
def __init__(self, hostname=None, username=None, password=None, path="", debug=False, clean_obsolete=True,
max_time_window=12, decommission_time_window=3):
self.logger = logging.getLogger('JiraAPI')
if debug:
self.logger.setLevel(logging.DEBUG)
@ -41,10 +44,15 @@ class JiraAPI(object):
# deletes the tag "server_decommission" from those tickets closed <=3 months ago
self.decommission_cleanup()
self.jira_still_vulnerable_comment = '''This ticket has been reopened due to the vulnerability not having been fixed (if multiple assets are affected, all need to be fixed; if the server is down, lastest known vulnerability might be the one reported).
- In the case of the team accepting the risk and wanting to close the ticket, please add the label "*risk_accepted*" to the ticket before closing it.
- If server has been decommissioned, please add the label "*server_decommission*" to the ticket before closing it.
- If when checking the vulnerability it looks like a false positive, _+please elaborate in a comment+_ and add the label "*false_positive*" before closing it; we will review it and report it to the vendor.
self.jira_still_vulnerable_comment = '''This ticket has been reopened due to the vulnerability not having been \
fixed (if multiple assets are affected, all need to be fixed; if the server is down, lastest known \
vulnerability might be the one reported).
- In the case of the team accepting the risk and wanting to close the ticket, please add the label \
"*risk_accepted*" to the ticket before closing it.
- If server has been decommissioned, please add the label "*server_decommission*" to the ticket before closing \
it.
- If when checking the vulnerability it looks like a false positive, _+please elaborate in a comment+_ and add \
the label "*false_positive*" before closing it; we will review it and report it to the vendor.
If you have further doubts, please contact the Security Team.'''
@ -91,13 +99,15 @@ class JiraAPI(object):
return len(self.jira.search_issues(jql, maxResults=0))
def metrics_closed_tickets(self, project=None):
jql = "labels= vulnerability_management and NOT resolution = Unresolved AND created >=startOfMonth(-{})".format(self.max_time_tracking)
jql = "labels= vulnerability_management and NOT resolution = Unresolved AND created >=startOfMonth(-{})".format(
self.max_time_tracking)
if project:
jql += " and (project='{}')".format(project)
return len(self.jira.search_issues(jql, maxResults=0))
def sync(self, vulnerabilities, project, components=[]):
#JIRA structure of each vulnerability: [source, scan_name, title, diagnosis, consequence, solution, ips, risk, references]
# JIRA structure of each vulnerability: [source, scan_name, title, diagnosis, consequence, solution,
# ips, risk, references]
self.logger.info("JIRA Sync started")
for vuln in vulnerabilities:
@ -106,7 +116,8 @@ class JiraAPI(object):
if " " in vuln['scan_name']:
vuln['scan_name'] = "_".join(vuln['scan_name'].split(" "))
# we exclude from the vulnerabilities to report those assets that already exist with *risk_accepted*/*server_decommission*
# we exclude from the vulnerabilities to report those assets that already exist
# with *risk_accepted*/*server_decommission*
vuln = self.exclude_accepted_assets(vuln)
# make sure after exclusion of risk_accepted assets there are still assets
@ -131,13 +142,17 @@ class JiraAPI(object):
# create local text file with assets, attach it to ticket
if len(vuln['ips']) > self.max_ips_ticket:
attachment_contents = vuln['ips']
vuln['ips'] = ["Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(assets = len(attachment_contents))]
vuln['ips'] = [
"Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(
assets=len(attachment_contents))]
try:
tpl = template(self.template_path, vuln)
except Exception as e:
self.logger.error('Exception templating: {}'.format(str(e)))
return 0
self.create_ticket(title=vuln['title'], desc=tpl, project=project, components=components, tags=[vuln['source'], vuln['scan_name'], 'vulnerability', vuln['risk']], attachment_contents = attachment_contents)
self.create_ticket(title=vuln['title'], desc=tpl, project=project, components=components,
tags=[vuln['source'], vuln['scan_name'], 'vulnerability', vuln['risk']],
attachment_contents=attachment_contents)
else:
self.logger.info("Ignoring vulnerability as all assets are already reported in a risk_accepted ticket")
@ -153,7 +168,8 @@ class JiraAPI(object):
labels = [vuln['source'], vuln['scan_name'], 'vulnerability_management', 'vulnerability']
if not self.excluded_tickets:
jql = "{} AND labels in (risk_accepted,server_decommission, false_positive) AND NOT labels=advisory AND created >=startOfMonth(-{})".format(" AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
jql = "{} AND labels in (risk_accepted,server_decommission, false_positive) AND NOT labels=advisory AND created >=startOfMonth(-{})".format(
" AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
self.excluded_tickets = self.jira.search_issues(jql, maxResults=0)
title = vuln['title']
@ -163,7 +179,8 @@ class JiraAPI(object):
assets_to_exclude = []
tickets_excluded_assets = []
for index in range(len(self.excluded_tickets)):
checking_ticketid, checking_title, checking_assets = self.ticket_get_unique_fields(self.excluded_tickets[index])
checking_ticketid, checking_title, checking_assets = self.ticket_get_unique_fields(
self.excluded_tickets[index])
if title.encode('ascii') == checking_title.encode('ascii'):
if checking_assets:
# checking_assets is a list, we add to our full list for later delete all assets
@ -172,7 +189,8 @@ class JiraAPI(object):
if assets_to_exclude:
assets_to_remove = []
self.logger.warn("Vulnerable Assets seen on an already existing risk_accepted Jira ticket: {}".format(', '.join(tickets_excluded_assets)))
self.logger.warn("Vulnerable Assets seen on an already existing risk_accepted Jira ticket: {}".format(
', '.join(tickets_excluded_assets)))
self.logger.debug("Original assets: {}".format(vuln['ips']))
# assets in vulnerability have the structure "ip - hostname - port", so we need to match by partial
for exclusion in assets_to_exclude:
@ -180,7 +198,9 @@ class JiraAPI(object):
# and we don't want it to affect the rest of the processing (otherwise, it would miss the asset right after the removed one)
for index in range(len(vuln['ips']))[::-1]:
if exclusion == vuln['ips'][index].split(" - ")[0]:
self.logger.debug("Deleting asset {} from vulnerability {}, seen in risk_accepted.".format(vuln['ips'][index], title))
self.logger.debug(
"Deleting asset {} from vulnerability {}, seen in risk_accepted.".format(vuln['ips'][index],
title))
vuln['ips'].pop(index)
self.logger.debug("Modified assets: {}".format(vuln['ips']))
@ -202,7 +222,8 @@ class JiraAPI(object):
self.logger.info("Retrieving all JIRA tickets with the following tags {}".format(labels))
# we want to check all JIRA tickets, to include tickets moved to other queues
# will exclude tickets older than 12 months, old tickets will get closed for higiene and recreated if still vulnerable
jql = "{} AND NOT labels=advisory AND created >=startOfMonth(-{})".format(" AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
jql = "{} AND NOT labels=advisory AND created >=startOfMonth(-{})".format(
" AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
self.all_tickets = self.jira.search_issues(jql, maxResults=0)
@ -212,7 +233,8 @@ class JiraAPI(object):
for index in range(len(self.all_tickets)):
checking_ticketid, checking_title, checking_assets = self.ticket_get_unique_fields(self.all_tickets[index])
# added "not risk_accepted", as if it is risk_accepted, we will create a new ticket excluding the accepted assets
if title.encode('ascii') == checking_title.encode('ascii') and not self.is_risk_accepted(self.jira.issue(checking_ticketid)):
if title.encode('ascii') == checking_title.encode('ascii') and not self.is_risk_accepted(
self.jira.issue(checking_ticketid)):
difference = list(set(assets).symmetric_difference(checking_assets))
# to check intersection - set(assets) & set(checking_assets)
if difference:
@ -239,9 +261,12 @@ class JiraAPI(object):
# structure the text to have the same structure as the assets from the attachment
affected_assets = ""
try:
affected_assets = ticket.raw.get('fields', {}).get('description').encode("ascii").split("{panel:title=Affected Assets}")[1].split("{panel}")[0].replace('\n','').replace(' * ','\n').replace('\n', '', 1)
affected_assets = \
ticket.raw.get('fields', {}).get('description').encode("ascii").split("{panel:title=Affected Assets}")[
1].split("{panel}")[0].replace('\n', '').replace(' * ', '\n').replace('\n', '', 1)
except Exception as e:
self.logger.error("Unable to process the Ticket's 'Affected Assets'. Ticket ID: {}. Reason: {}".format(ticket, e))
self.logger.error(
"Unable to process the Ticket's 'Affected Assets'. Ticket ID: {}. Reason: {}".format(ticket, e))
if affected_assets:
if _raw:
@ -280,7 +305,8 @@ class JiraAPI(object):
affected_assets = self.jira.attachment(attachment_id).get()
except Exception as e:
self.logger.error("Failed to get assets from ticket attachment. Ticket ID: {}. Reason: {}".format(ticket, e))
self.logger.error(
"Failed to get assets from ticket attachment. Ticket ID: {}. Reason: {}".format(ticket, e))
if affected_assets:
if _raw:
@ -353,8 +379,10 @@ class JiraAPI(object):
if self.is_ticket_resolved(ticket_obj):
ticket_data = ticket_obj.raw.get('fields')
# dates follow format '2018-11-06T10:36:13.849+0100'
created = [int(x) for x in ticket_data['created'].split('.')[0].replace('T', '-').replace(':','-').split('-')]
resolved =[int(x) for x in ticket_data['resolutiondate'].split('.')[0].replace('T', '-').replace(':','-').split('-')]
created = [int(x) for x in
ticket_data['created'].split('.')[0].replace('T', '-').replace(':', '-').split('-')]
resolved = [int(x) for x in
ticket_data['resolutiondate'].split('.')[0].replace('T', '-').replace(':', '-').split('-')]
start = datetime(created[0], created[1], created[2], created[3], created[4], created[5])
end = datetime(resolved[0], resolved[1], resolved[2], resolved[3], resolved[4], resolved[5])
@ -405,7 +433,9 @@ class JiraAPI(object):
attachment_contents = []
if len(vuln['ips']) > self.max_ips_ticket:
attachment_contents = vuln['ips']
vuln['ips'] = ["Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(assets = len(attachment_contents))]
vuln['ips'] = [
"Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(
assets=len(attachment_contents))]
# fill the ticket description template
try:
@ -425,7 +455,8 @@ class JiraAPI(object):
self.logger.info("Ticket {} updated successfully".format(ticketid))
self.add_label(ticketid, 'updated')
except Exception as e:
self.logger.error("Error while trying up update ticket {ticketid}.\nReason: {e}".format(ticketid = ticketid, e=e))
self.logger.error(
"Error while trying up update ticket {ticketid}.\nReason: {e}".format(ticketid=ticketid, e=e))
return 0
def add_label(self, ticketid, label):
@ -437,8 +468,9 @@ class JiraAPI(object):
try:
ticket_obj.update(fields={"labels": ticket_obj.fields.labels})
self.logger.info("Added label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
except:
self.logger.error("Error while trying to add label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
except Exception as e:
self.logger.error(
"Error while trying to add label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
return 0
@ -451,8 +483,9 @@ class JiraAPI(object):
try:
ticket_obj.update(fields={"labels": ticket_obj.fields.labels})
self.logger.info("Removed label {label} from ticket {ticket}".format(label=label, ticket=ticketid))
except:
self.logger.error("Error while trying to remove label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
except Exception as e:
self.logger.error("Error while trying to remove label {label} to ticket {ticket}".format(label=label,
ticket=ticketid))
else:
self.logger.error("Error: label {label} not in ticket {ticket}".format(label=label, ticket=ticketid))
@ -478,7 +511,6 @@ class JiraAPI(object):
self.close_ticket(ticket, self.JIRA_RESOLUTION_FIXED, comment)
return 0
def is_ticket_reopenable(self, ticket_obj):
transitions = self.jira.transitions(ticket_obj)
for transition in transitions:
@ -507,7 +539,6 @@ class JiraAPI(object):
self.logger.debug("Checked ticket {} is already open".format(ticket_obj))
return False
def is_risk_accepted(self, ticket_obj):
if ticket_obj is not None:
if ticket_obj.raw['fields'].get('labels') is not None:
@ -533,7 +564,8 @@ class JiraAPI(object):
if (not self.is_risk_accepted(ticket_obj) or ignore_labels):
try:
if self.is_ticket_reopenable(ticket_obj):
error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_REOPEN_ISSUE, comment = comment)
error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_REOPEN_ISSUE,
comment=comment)
self.logger.info("Ticket {} reopened successfully".format(ticketid))
if not ignore_labels:
self.add_label(ticketid, 'reopened')
@ -553,7 +585,8 @@ class JiraAPI(object):
if self.is_ticket_closeable(ticket_obj):
# need to add the label before closing the ticket
self.add_label(ticketid, 'closed')
error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_CLOSE_ISSUE, comment = comment, resolution = {"name": resolution })
error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_CLOSE_ISSUE,
comment=comment, resolution={"name": resolution})
self.logger.info("Ticket {} closed successfully".format(ticketid))
return 1
except Exception as e:
@ -566,7 +599,8 @@ class JiraAPI(object):
def close_obsolete_tickets(self):
# Close tickets older than 12 months, vulnerabilities not solved will get created a new ticket
self.logger.info("Closing obsolete tickets older than {} months".format(self.max_time_tracking))
jql = "labels=vulnerability_management AND NOT labels=advisory AND created <startOfMonth(-{}) and resolution=Unresolved".format(self.max_time_tracking)
jql = "labels=vulnerability_management AND NOT labels=advisory AND created <startOfMonth(-{}) and resolution=Unresolved".format(
self.max_time_tracking)
tickets_to_close = self.jira.search_issues(jql, maxResults=0)
comment = '''This ticket is being closed for hygiene, as it is more than {} months old.
@ -597,7 +631,8 @@ class JiraAPI(object):
return True
try:
self.logger.info("Saving locally tickets from the last {} months".format(self.max_time_tracking))
jql = "labels=vulnerability_management AND NOT labels=advisory AND created >=startOfMonth(-{})".format(self.max_time_tracking)
jql = "labels=vulnerability_management AND NOT labels=advisory AND created >=startOfMonth(-{})".format(
self.max_time_tracking)
tickets_data = self.jira.search_issues(jql, maxResults=0)
# TODO process tickets, creating a new field called "_metadata" with all the affected assets well structured
@ -621,7 +656,6 @@ class JiraAPI(object):
assets_json = self.parse_asset_to_json(assets)
_metadata["affected_hosts"].append(assets_json)
temp_ticket = ticket.raw.get('fields')
temp_ticket['_metadata'] = _metadata
@ -646,13 +680,16 @@ class JiraAPI(object):
closed already for more than x months (default is 3 months) in order to clean solved issues
for statistics purposes
'''
self.logger.info("Deleting 'server_decommission' tag from tickets closed more than {} months ago".format(self.max_decommission_time))
self.logger.info("Deleting 'server_decommission' tag from tickets closed more than {} months ago".format(
self.max_decommission_time))
jql = "labels=vulnerability_management AND labels=server_decommission and resolutiondate <=startOfMonth(-{})".format(self.max_decommission_time)
jql = "labels=vulnerability_management AND labels=server_decommission and resolutiondate <=startOfMonth(-{})".format(
self.max_decommission_time)
decommissioned_tickets = self.jira.search_issues(jql, maxResults=0)
comment = '''This ticket is having deleted the *server_decommission* tag, as it is more than {} months old and is expected to already have been decommissioned.
If that is not the case and the vulnerability still exists, the vulnerability will be opened again.'''.format(self.max_decommission_time)
If that is not the case and the vulnerability still exists, the vulnerability will be opened again.'''.format(
self.max_decommission_time)
for ticket in decommissioned_tickets:
# we open first the ticket, as we want to make sure the process is not blocked due to

1
vulnwhisp/test/mock.py
View File

@ -1,3 +1,4 @@
from __future__ import absolute_import
import os
import logging
import httpretty

17
vulnwhisp/vulnwhisp.py
View File

@ -1,13 +1,16 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
from __future__ import absolute_import
from six.moves import range
from functools import reduce
__author__ = 'Austin Taylor'
from base.config import vwConfig
from frameworks.nessus import NessusAPI
from frameworks.qualys_web import qualysScanReport
from frameworks.qualys_vuln import qualysVulnScan
from frameworks.openvas import OpenVAS_API
from reporting.jira_api import JiraAPI
from .base.config import vwConfig
from .frameworks.nessus import NessusAPI
from .frameworks.qualys_web import qualysScanReport
from .frameworks.qualys_vuln import qualysVulnScan
from .frameworks.openvas import OpenVAS_API
from .reporting.jira_api import JiraAPI
import pandas as pd
from lxml import objectify
import sys
@ -1200,7 +1203,7 @@ class vulnWhispererJIRA(vulnWhispererBase):
if vuln['dns']:
values['dns'] = vuln['dns']
else:
if values['ip'] in self.host_resolv_cache.keys():
if values['ip'] in list(self.host_resolv_cache.keys()):
self.logger.debug("Hostname from {ip} cached, retrieving from cache.".format(ip=values['ip']))
values['dns'] = self.host_resolv_cache[values['ip']]
else: