Merge pull request #206 from HASecuritySolutions/jira_ticket_download_attachment_data
Jira ticket download attachment data
This commit is contained in:
Quim Montal
@ -83,14 +83,17 @@ def main():
|
||||
enabled_sections = config.get_sections_with_attribute('enabled')
|
||||
|
||||
for section in enabled_sections:
|
||||
vw = vulnWhisperer(config=args.config,
|
||||
profile=section,
|
||||
verbose=args.verbose,
|
||||
username=args.username,
|
||||
password=args.password,
|
||||
source=args.source,
|
||||
scanname=args.scanname)
|
||||
exit_code += vw.whisper_vulnerabilities()
|
||||
try:
|
||||
vw = vulnWhisperer(config=args.config,
|
||||
profile=section,
|
||||
verbose=args.verbose,
|
||||
username=args.username,
|
||||
password=args.password,
|
||||
source=args.source,
|
||||
scanname=args.scanname)
|
||||
exit_code += vw.whisper_vulnerabilities()
|
||||
except Exception as e:
|
||||
logger.error("VulnWhisperer was unable to perform the processing on '{}'".format(args.source))
|
||||
else:
|
||||
logger.info('Running vulnwhisperer for section {}'.format(args.section))
|
||||
vw = vulnWhisperer(config=args.config,
|
||||
|
||||
@ -226,32 +226,44 @@ class JiraAPI(object):
|
||||
def ticket_get_unique_fields(self, ticket):
|
||||
title = ticket.raw.get('fields', {}).get('summary').encode("ascii").strip()
|
||||
ticketid = ticket.key.encode("ascii")
|
||||
assets = []
|
||||
try:
|
||||
affected_assets_section = ticket.raw.get('fields', {}).get('description').encode("ascii").split("{panel:title=Affected Assets}")[1].split("{panel}")[0]
|
||||
assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", affected_assets_section)))
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error("Ticket IPs regex failed. Ticket ID: {}. Reason: {}".format(ticketid, e))
|
||||
assets = []
|
||||
|
||||
try:
|
||||
if not assets:
|
||||
#check if attachment, if so, get assets from attachment
|
||||
affected_assets_section = self.check_ips_attachment(ticket)
|
||||
if affected_assets_section:
|
||||
assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", affected_assets_section)))
|
||||
except Exception as e:
|
||||
self.logger.error("Ticket IPs Attachment regex failed. Ticket ID: {}. Reason: {}".format(ticketid, e))
|
||||
|
||||
assets = self.get_assets_from_description(ticket)
|
||||
if not assets:
|
||||
#check if attachment, if so, get assets from attachment
|
||||
assets = self.get_assets_from_attachment(ticket)
|
||||
|
||||
return ticketid, title, assets
|
||||
|
||||
def check_ips_attachment(self, ticket):
|
||||
affected_assets_section = []
|
||||
def get_assets_from_description(self, ticket, _raw = False):
|
||||
# Get the assets as a string "host - protocol/port - hostname" separated by "\n"
|
||||
# structure the text to have the same structure as the assets from the attachment
|
||||
affected_assets = ""
|
||||
try:
|
||||
affected_assets = ticket.raw.get('fields', {}).get('description').encode("ascii").split("{panel:title=Affected Assets}")[1].split("{panel}")[0].replace('\n','').replace(' * ','\n').replace('\n', '', 1)
|
||||
except Exception as e:
|
||||
self.logger.error("Unable to process the Ticket's 'Affected Assets'. Ticket ID: {}. Reason: {}".format(ticket, e))
|
||||
|
||||
if affected_assets:
|
||||
if _raw:
|
||||
# from line 406 check if the text in the panel corresponds to having added an attachment
|
||||
if "added as an attachment" in affected_assets:
|
||||
return False
|
||||
return affected_assets
|
||||
|
||||
try:
|
||||
# if _raw is not true, we return only the IPs of the affected assets
|
||||
return list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", affected_assets)))
|
||||
except Exception as e:
|
||||
self.logger.error("Ticket IPs regex failed. Ticket ID: {}. Reason: {}".format(ticket, e))
|
||||
return False
|
||||
|
||||
def get_assets_from_attachment(self, ticket, _raw = False):
|
||||
# Get the assets as a string "host - protocol/port - hostname" separated by "\n"
|
||||
affected_assets = []
|
||||
try:
|
||||
fields = self.jira.issue(ticket.key).raw.get('fields', {})
|
||||
attachments = fields.get('attachment', {})
|
||||
affected_assets_section = ""
|
||||
affected_assets = ""
|
||||
#we will make sure we get the latest version of the file
|
||||
latest = ''
|
||||
attachment_id = ''
|
||||
@ -265,12 +277,43 @@ class JiraAPI(object):
|
||||
if latest < item.get('created'):
|
||||
latest = item.get('created')
|
||||
attachment_id = item.get('id')
|
||||
affected_assets_section = self.jira.attachment(attachment_id).get()
|
||||
affected_assets = self.jira.attachment(attachment_id).get()
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error("Failed to get assets from ticket attachment. Ticket ID: {}. Reason: {}".format(ticket, e))
|
||||
|
||||
return affected_assets_section
|
||||
if affected_assets:
|
||||
if _raw:
|
||||
return affected_assets
|
||||
|
||||
try:
|
||||
# if _raw is not true, we return only the IPs of the affected assets
|
||||
affected_assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", affected_assets)))
|
||||
return affected_assets
|
||||
except Exception as e:
|
||||
self.logger.error("Ticket IPs Attachment regex failed. Ticket ID: {}. Reason: {}".format(ticket, e))
|
||||
|
||||
return False
|
||||
|
||||
def parse_asset_to_json(self, asset):
|
||||
hostname, protocol, port = "", "", ""
|
||||
asset_info = asset.split(" - ")
|
||||
ip = asset_info[0]
|
||||
proto_port = asset_info[1]
|
||||
# in case there is some case where hostname is not reported at all
|
||||
if len(asset_info) == 3:
|
||||
hostname = asset_info[2]
|
||||
if proto_port != "N/A/N/A":
|
||||
protocol, port = proto_port.split("/")
|
||||
|
||||
asset_dict = {
|
||||
"host": ip,
|
||||
"protocol": protocol,
|
||||
"port": port,
|
||||
"hostname": hostname
|
||||
}
|
||||
|
||||
return asset_dict
|
||||
|
||||
def clean_old_attachments(self, ticket):
|
||||
fields = ticket.raw.get('fields')
|
||||
@ -522,7 +565,7 @@ class JiraAPI(object):
|
||||
def close_obsolete_tickets(self):
|
||||
# Close tickets older than 12 months, vulnerabilities not solved will get created a new ticket
|
||||
self.logger.info("Closing obsolete tickets older than {} months".format(self.max_time_tracking))
|
||||
jql = "labels=vulnerability_management AND created <startOfMonth(-{}) and resolution=Unresolved".format(self.max_time_tracking)
|
||||
jql = "labels=vulnerability_management AND NOT labels=advisory AND created <startOfMonth(-{}) and resolution=Unresolved".format(self.max_time_tracking)
|
||||
tickets_to_close = self.jira.search_issues(jql, maxResults=0)
|
||||
|
||||
comment = '''This ticket is being closed for hygiene, as it is more than {} months old.
|
||||
@ -553,8 +596,35 @@ class JiraAPI(object):
|
||||
return True
|
||||
try:
|
||||
self.logger.info("Saving locally tickets from the last {} months".format(self.max_time_tracking))
|
||||
jql = "labels=vulnerability_management AND created >=startOfMonth(-{})".format(self.max_time_tracking)
|
||||
jql = "labels=vulnerability_management AND NOT labels=advisory AND created >=startOfMonth(-{})".format(self.max_time_tracking)
|
||||
tickets_data = self.jira.search_issues(jql, maxResults=0)
|
||||
|
||||
#TODO process tickets, creating a new field called "_metadata" with all the affected assets well structured
|
||||
# for future processing in ELK/Splunk; this includes downloading attachments with assets and processing them
|
||||
|
||||
processed_tickets = []
|
||||
|
||||
for ticket in tickets_data:
|
||||
assets = self.get_assets_from_description(ticket, _raw=True)
|
||||
if not assets:
|
||||
# check if attachment, if so, get assets from attachment
|
||||
assets = self.get_assets_from_attachment(ticket, _raw=True)
|
||||
# process the affected assets to save them as json structure on a new field from the JSON
|
||||
_metadata = {"affected_hosts": []}
|
||||
if assets:
|
||||
if "\n" in assets:
|
||||
for asset in assets.split("\n"):
|
||||
assets_json = self.parse_asset_to_json(asset)
|
||||
_metadata["affected_hosts"].append(assets_json)
|
||||
else:
|
||||
assets_json = self.parse_asset_to_json(assets)
|
||||
_metadata["affected_hosts"].append(assets_json)
|
||||
|
||||
|
||||
temp_ticket = ticket.raw.get('fields')
|
||||
temp_ticket['_metadata'] = _metadata
|
||||
|
||||
processed_tickets.append(temp_ticket)
|
||||
|
||||
#end of line needed, as writelines() doesn't add it automatically, otherwise one big line
|
||||
to_save = [json.dumps(ticket.raw.get('fields'))+"\n" for ticket in tickets_data]
|
||||
@ -566,7 +636,7 @@ class JiraAPI(object):
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error("Tickets could not be saved locally: {}.".format(e))
|
||||
|
||||
|
||||
return False
|
||||
|
||||
def decommission_cleanup(self):
|
||||
|
||||
@ -204,6 +204,7 @@ class vulnWhispererBase(object):
|
||||
def get_latest_results(self, source, scan_name):
|
||||
processed = 0
|
||||
results = []
|
||||
reported = ""
|
||||
|
||||
try:
|
||||
self.conn.text_factory = str
|
||||
@ -222,6 +223,7 @@ class vulnWhispererBase(object):
|
||||
|
||||
except Exception as e:
|
||||
self.logger.error("Error when getting latest results from {}.{} : {}".format(source, scan_name, e))
|
||||
|
||||
return results, reported
|
||||
|
||||
def get_scan_profiles(self):
|
||||
@ -317,7 +319,8 @@ class vulnWhispererNessus(vulnWhispererBase):
|
||||
e=e))
|
||||
except Exception as e:
|
||||
self.logger.error('Could not properly load your config!\nReason: {e}'.format(e=e))
|
||||
sys.exit(1)
|
||||
return False
|
||||
#sys.exit(1)
|
||||
|
||||
|
||||
|
||||
@ -573,8 +576,11 @@ class vulnWhispererQualys(vulnWhispererBase):
|
||||
self.logger = logging.getLogger('vulnWhispererQualys')
|
||||
if debug:
|
||||
self.logger.setLevel(logging.DEBUG)
|
||||
|
||||
self.qualys_scan = qualysScanReport(config=config)
|
||||
try:
|
||||
self.qualys_scan = qualysScanReport(config=config)
|
||||
except Exception as e:
|
||||
self.logger.error("Unable to establish connection with Qualys scanner. Reason: {}".format(e))
|
||||
return False
|
||||
self.latest_scans = self.qualys_scan.qw.get_all_scans()
|
||||
self.directory_check()
|
||||
self.scans_to_process = None
|
||||
@ -745,10 +751,14 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
|
||||
self.develop = True
|
||||
self.purge = purge
|
||||
self.scans_to_process = None
|
||||
self.openvas_api = OpenVAS_API(hostname=self.hostname,
|
||||
port=self.port,
|
||||
username=self.username,
|
||||
password=self.password)
|
||||
try:
|
||||
self.openvas_api = OpenVAS_API(hostname=self.hostname,
|
||||
port=self.port,
|
||||
username=self.username,
|
||||
password=self.password)
|
||||
except Exception as e:
|
||||
self.logger.error("Unable to establish connection with OpenVAS scanner. Reason: {}".format(e))
|
||||
return False
|
||||
|
||||
def whisper_reports(self, output_format='json', launched_date=None, report_id=None, cleanup=True):
|
||||
report = None
|
||||
@ -859,8 +869,11 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
|
||||
self.logger = logging.getLogger('vulnWhispererQualysVuln')
|
||||
if debug:
|
||||
self.logger.setLevel(logging.DEBUG)
|
||||
|
||||
self.qualys_scan = qualysVulnScan(config=config)
|
||||
try:
|
||||
self.qualys_scan = qualysVulnScan(config=config)
|
||||
except Exception as e:
|
||||
self.logger.error("Unable to create connection with Qualys. Reason: {}".format(e))
|
||||
return False
|
||||
self.directory_check()
|
||||
self.scans_to_process = None
|
||||
|
||||
@ -871,7 +884,7 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
|
||||
scan_reference=None,
|
||||
output_format='json',
|
||||
cleanup=True):
|
||||
launched_date
|
||||
|
||||
if 'Z' in launched_date:
|
||||
launched_date = self.qualys_scan.utils.iso_to_epoch(launched_date)
|
||||
report_name = 'qualys_vuln_' + report_id.replace('/','_') \
|
||||
@ -1007,7 +1020,8 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
raise Exception(
|
||||
'Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format(
|
||||
config=self.config.config_in, e=e))
|
||||
sys.exit(1)
|
||||
return False
|
||||
#sys.exit(1)
|
||||
|
||||
profiles = []
|
||||
profiles = self.get_scan_profiles()
|
||||
@ -1259,7 +1273,10 @@ class vulnWhispererJIRA(vulnWhispererBase):
|
||||
|
||||
if autoreport_sections:
|
||||
for scan in autoreport_sections:
|
||||
self.jira_sync(self.config.get(scan, 'source'), self.config.get(scan, 'scan_name'))
|
||||
try:
|
||||
self.jira_sync(self.config.get(scan, 'source'), self.config.get(scan, 'scan_name'))
|
||||
except Exception as e:
|
||||
self.logger.error("VulnWhisperer wasn't able to report the vulnerabilities from the '{}'s source".format(self.config.get(scan, 'source')))
|
||||
return True
|
||||
return False
|
||||
|
||||
@ -1292,36 +1309,42 @@ class vulnWhisperer(object):
|
||||
if self.profile == 'nessus':
|
||||
vw = vulnWhispererNessus(config=self.config,
|
||||
profile=self.profile)
|
||||
self.exit_code += vw.whisper_nessus()
|
||||
if vw:
|
||||
self.exit_code += vw.whisper_nessus()
|
||||
|
||||
elif self.profile == 'qualys_web':
|
||||
vw = vulnWhispererQualys(config=self.config)
|
||||
self.exit_code += vw.process_web_assets()
|
||||
if vw:
|
||||
self.exit_code += vw.process_web_assets()
|
||||
|
||||
elif self.profile == 'openvas':
|
||||
vw_openvas = vulnWhispererOpenVAS(config=self.config)
|
||||
self.exit_code += vw_openvas.process_openvas_scans()
|
||||
if vw:
|
||||
self.exit_code += vw_openvas.process_openvas_scans()
|
||||
|
||||
elif self.profile == 'tenable':
|
||||
vw = vulnWhispererNessus(config=self.config,
|
||||
profile=self.profile)
|
||||
self.exit_code += vw.whisper_nessus()
|
||||
if vw:
|
||||
self.exit_code += vw.whisper_nessus()
|
||||
|
||||
elif self.profile == 'qualys_vuln':
|
||||
vw = vulnWhispererQualysVuln(config=self.config)
|
||||
self.exit_code += vw.process_vuln_scans()
|
||||
if vw:
|
||||
self.exit_code += vw.process_vuln_scans()
|
||||
|
||||
elif self.profile == 'jira':
|
||||
#first we check config fields are created, otherwise we create them
|
||||
vw = vulnWhispererJIRA(config=self.config)
|
||||
if not (self.source and self.scanname):
|
||||
self.logger.info('No source/scan_name selected, all enabled scans will be synced')
|
||||
success = vw.sync_all()
|
||||
if not success:
|
||||
self.logger.error('All scans sync failed!')
|
||||
self.logger.error('Source scanner and scan name needed!')
|
||||
return 0
|
||||
else:
|
||||
vw.jira_sync(self.source, self.scanname)
|
||||
if vw:
|
||||
if not (self.source and self.scanname):
|
||||
self.logger.info('No source/scan_name selected, all enabled scans will be synced')
|
||||
success = vw.sync_all()
|
||||
if not success:
|
||||
self.logger.error('All scans sync failed!')
|
||||
self.logger.error('Source scanner and scan name needed!')
|
||||
return 0
|
||||
else:
|
||||
vw.jira_sync(self.source, self.scanname)
|
||||
|
||||
return self.exit_code
|
||||
|
||||
Reference in New Issue
Block a user