Merge pull request #206 from HASecuritySolutions/jira_ticket_download_attachment_data

Jira ticket download attachment data

bin/vuln_whisperer

@@ -83,14 +83,17 @@ def main():
             enabled_sections = config.get_sections_with_attribute('enabled')
 
             for section in enabled_sections:
-                vw = vulnWhisperer(config=args.config,
+                try:
-                                   profile=section,
+                    vw = vulnWhisperer(config=args.config,
-                                   verbose=args.verbose,
+                                       profile=section,
-                                   username=args.username,
+                                       verbose=args.verbose,
-                                   password=args.password,
+                                       username=args.username,
-                                   source=args.source,
+                                       password=args.password,
-                                   scanname=args.scanname)
+                                       source=args.source,
-                exit_code += vw.whisper_vulnerabilities()
+                                       scanname=args.scanname)
+                    exit_code += vw.whisper_vulnerabilities()
+                except Exception as e:
+                    logger.error("VulnWhisperer was unable to perform the processing on '{}'".format(args.source))
         else:
             logger.info('Running vulnwhisperer for section {}'.format(args.section))
             vw = vulnWhisperer(config=args.config,

vulnwhisp/reporting/jira_api.py

@@ -226,32 +226,44 @@ class JiraAPI(object):
     def ticket_get_unique_fields(self, ticket):
         title = ticket.raw.get('fields', {}).get('summary').encode("ascii").strip()
         ticketid = ticket.key.encode("ascii")
-        assets = []
+
-        try:
+        assets = self.get_assets_from_description(ticket)
-            affected_assets_section = ticket.raw.get('fields', {}).get('description').encode("ascii").split("{panel:title=Affected Assets}")[1].split("{panel}")[0]
+        if not assets:
-            assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", affected_assets_section)))
+            #check if attachment, if so, get assets from attachment
-        
+            assets = self.get_assets_from_attachment(ticket)
-        except Exception as e:
-            self.logger.error("Ticket IPs regex failed. Ticket ID: {}. Reason: {}".format(ticketid, e))
-            assets = []
-        
-        try:     
-            if not assets:
-                #check if attachment, if so, get assets from attachment
-                affected_assets_section = self.check_ips_attachment(ticket)
-                if affected_assets_section:
-                    assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", affected_assets_section)))
-        except Exception as e:
-            self.logger.error("Ticket IPs Attachment regex failed. Ticket ID: {}. Reason: {}".format(ticketid, e))
                 
         return ticketid, title, assets
 
-    def check_ips_attachment(self, ticket):
+    def get_assets_from_description(self, ticket, _raw = False):
-        affected_assets_section = []
+        # Get the assets as a string "host - protocol/port - hostname" separated by "\n"
+        # structure the text to have the same structure as the assets from the attachment
+        affected_assets = ""
+        try:
+            affected_assets = ticket.raw.get('fields', {}).get('description').encode("ascii").split("{panel:title=Affected Assets}")[1].split("{panel}")[0].replace('\n','').replace(' * ','\n').replace('\n', '', 1)
+        except Exception as e:
+            self.logger.error("Unable to process the Ticket's 'Affected Assets'. Ticket ID: {}. Reason: {}".format(ticket, e))
+
+        if affected_assets:
+            if _raw:
+                # from line 406 check if the text in the panel corresponds to having added an attachment
+                if "added as an attachment" in affected_assets:
+                    return False
+                return affected_assets
+
+            try:
+                # if _raw is not true, we return only the IPs of the affected assets
+                return list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", affected_assets)))
+            except Exception as e:
+                self.logger.error("Ticket IPs regex failed. Ticket ID: {}. Reason: {}".format(ticket, e))
+        return False
+
+    def get_assets_from_attachment(self, ticket, _raw = False):
+        # Get the assets as a string "host - protocol/port - hostname" separated by "\n"
+        affected_assets = []
         try:
             fields = self.jira.issue(ticket.key).raw.get('fields', {})
             attachments = fields.get('attachment', {})
-            affected_assets_section = ""
+            affected_assets = ""
             #we will make sure we get the latest version of the file
             latest = ''
             attachment_id = ''
@@ -265,12 +277,43 @@ class JiraAPI(object):
                             if latest < item.get('created'):
                                 latest = item.get('created')         
                                 attachment_id = item.get('id') 
-            affected_assets_section = self.jira.attachment(attachment_id).get()
+            affected_assets = self.jira.attachment(attachment_id).get()
 
         except Exception as e:
             self.logger.error("Failed to get assets from ticket attachment. Ticket ID: {}. Reason: {}".format(ticket, e))
 
-        return affected_assets_section
+        if affected_assets:
+            if _raw:
+                return affected_assets
+
+            try:
+                # if _raw is not true, we return only the IPs of the affected assets
+                affected_assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", affected_assets)))
+                return affected_assets
+            except Exception as e:
+                self.logger.error("Ticket IPs Attachment regex failed. Ticket ID: {}. Reason: {}".format(ticket, e))
+
+        return False
+
+    def parse_asset_to_json(self, asset):
+        hostname, protocol, port = "", "", ""
+        asset_info = asset.split(" - ")
+        ip = asset_info[0]
+        proto_port = asset_info[1]
+        # in case there is some case where hostname is not reported at all
+        if len(asset_info) == 3:
+            hostname = asset_info[2]
+        if proto_port != "N/A/N/A":
+            protocol, port = proto_port.split("/")
+
+        asset_dict = {
+            "host": ip,
+            "protocol": protocol,
+            "port": port,
+            "hostname": hostname
+        }
+
+        return asset_dict
 
     def clean_old_attachments(self, ticket):
         fields = ticket.raw.get('fields')
@@ -522,7 +565,7 @@ class JiraAPI(object):
     def close_obsolete_tickets(self):
         # Close tickets older than 12 months, vulnerabilities not solved will get created a new ticket 
         self.logger.info("Closing obsolete tickets older than {} months".format(self.max_time_tracking))
-        jql = "labels=vulnerability_management AND created <startOfMonth(-{}) and resolution=Unresolved".format(self.max_time_tracking)
+        jql = "labels=vulnerability_management AND NOT labels=advisory AND created <startOfMonth(-{}) and resolution=Unresolved".format(self.max_time_tracking)
         tickets_to_close = self.jira.search_issues(jql, maxResults=0)
         
         comment = '''This ticket is being closed for hygiene, as it is more than {} months old.
@@ -553,8 +596,35 @@ class JiraAPI(object):
             return True
         try:
             self.logger.info("Saving locally tickets from the last {} months".format(self.max_time_tracking))
-            jql = "labels=vulnerability_management AND created >=startOfMonth(-{})".format(self.max_time_tracking)
+            jql = "labels=vulnerability_management AND NOT labels=advisory AND created >=startOfMonth(-{})".format(self.max_time_tracking)
             tickets_data = self.jira.search_issues(jql, maxResults=0)
+
+            #TODO process tickets, creating a new field called "_metadata" with all the affected assets well structured
+            # for future processing in ELK/Splunk; this includes downloading attachments with assets and processing them
+
+            processed_tickets = []
+
+            for ticket in tickets_data:
+                assets = self.get_assets_from_description(ticket, _raw=True)
+                if not assets:
+                    # check if attachment, if so, get assets from attachment
+                    assets = self.get_assets_from_attachment(ticket, _raw=True)
+                # process the affected assets to save them as json structure on a new field from the JSON
+                _metadata = {"affected_hosts": []}
+                if assets:
+                    if "\n" in assets:
+                        for asset in assets.split("\n"):
+                            assets_json = self.parse_asset_to_json(asset)
+                            _metadata["affected_hosts"].append(assets_json)
+                    else:
+                        assets_json = self.parse_asset_to_json(assets)
+                        _metadata["affected_hosts"].append(assets_json)
+
+
+                temp_ticket = ticket.raw.get('fields')
+                temp_ticket['_metadata'] = _metadata
+
+                processed_tickets.append(temp_ticket)
             
             #end of line needed, as writelines() doesn't add it automatically, otherwise one big line
             to_save = [json.dumps(ticket.raw.get('fields'))+"\n" for ticket in tickets_data]
@@ -566,7 +636,7 @@ class JiraAPI(object):
 
         except Exception as e:
             self.logger.error("Tickets could not be saved locally: {}.".format(e))
-        
+
         return False 
 
     def decommission_cleanup(self):

vulnwhisp/vulnwhisp.py

@@ -204,6 +204,7 @@ class vulnWhispererBase(object):
     def get_latest_results(self, source, scan_name):
         processed = 0
         results = []
+        reported = ""
 
         try:
             self.conn.text_factory = str
@@ -222,6 +223,7 @@ class vulnWhispererBase(object):
 
         except Exception as e:
             self.logger.error("Error when getting latest results from {}.{} : {}".format(source, scan_name, e))
+
         return results, reported
 
     def get_scan_profiles(self):
@@ -317,7 +319,8 @@ class vulnWhispererNessus(vulnWhispererBase):
                             e=e))
             except Exception as e:
                 self.logger.error('Could not properly load your config!\nReason: {e}'.format(e=e))
-                sys.exit(1)
+                return False
+                #sys.exit(1)
 
 
 
@@ -573,8 +576,11 @@ class vulnWhispererQualys(vulnWhispererBase):
         self.logger = logging.getLogger('vulnWhispererQualys')
         if debug:
             self.logger.setLevel(logging.DEBUG)
-
+        try:
-        self.qualys_scan = qualysScanReport(config=config)
+            self.qualys_scan = qualysScanReport(config=config)
+        except Exception as e:
+            self.logger.error("Unable to establish connection with Qualys scanner. Reason: {}".format(e))
+            return False
         self.latest_scans = self.qualys_scan.qw.get_all_scans()
         self.directory_check()
         self.scans_to_process = None
@@ -745,10 +751,14 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
         self.develop = True
         self.purge = purge
         self.scans_to_process = None
-        self.openvas_api = OpenVAS_API(hostname=self.hostname,
+        try:
-                                       port=self.port,
+            self.openvas_api = OpenVAS_API(hostname=self.hostname,
-                                       username=self.username,
+                                           port=self.port,
-                                       password=self.password)
+                                           username=self.username,
+                                           password=self.password)
+        except Exception as e:
+            self.logger.error("Unable to establish connection with OpenVAS scanner. Reason: {}".format(e))
+            return False
 
     def whisper_reports(self, output_format='json', launched_date=None, report_id=None, cleanup=True):
         report = None
@@ -859,8 +869,11 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
         self.logger = logging.getLogger('vulnWhispererQualysVuln')
         if debug:
             self.logger.setLevel(logging.DEBUG)
-
+        try:
-        self.qualys_scan = qualysVulnScan(config=config)
+            self.qualys_scan = qualysVulnScan(config=config)
+        except Exception as e:
+            self.logger.error("Unable to create connection with Qualys. Reason: {}".format(e))
+            return False
         self.directory_check()
         self.scans_to_process = None
 
@@ -871,7 +884,7 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
                         scan_reference=None,
                         output_format='json',
                         cleanup=True):
-            launched_date
+
             if 'Z' in launched_date:
                 launched_date = self.qualys_scan.utils.iso_to_epoch(launched_date)
             report_name = 'qualys_vuln_' + report_id.replace('/','_') \
@@ -1007,7 +1020,8 @@ class vulnWhispererJIRA(vulnWhispererBase):
                 raise Exception(
                     'Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format(
                         config=self.config.config_in, e=e))
-                sys.exit(1)
+                return False
+                #sys.exit(1)
 
         profiles = []
         profiles = self.get_scan_profiles()
@@ -1259,7 +1273,10 @@ class vulnWhispererJIRA(vulnWhispererBase):
 
         if autoreport_sections:
             for scan in autoreport_sections:
-                self.jira_sync(self.config.get(scan, 'source'), self.config.get(scan, 'scan_name'))
+                try:
+                    self.jira_sync(self.config.get(scan, 'source'), self.config.get(scan, 'scan_name'))
+                except Exception as e:
+                    self.logger.error("VulnWhisperer wasn't able to report the vulnerabilities from the '{}'s source".format(self.config.get(scan, 'source')))
             return True
         return False
 
@@ -1292,36 +1309,42 @@ class vulnWhisperer(object):
         if self.profile == 'nessus':
             vw = vulnWhispererNessus(config=self.config,
                                      profile=self.profile)
-            self.exit_code += vw.whisper_nessus()
+            if vw:
+                self.exit_code += vw.whisper_nessus()
 
         elif self.profile == 'qualys_web':
             vw = vulnWhispererQualys(config=self.config)
-            self.exit_code += vw.process_web_assets()
+            if vw:
+                self.exit_code += vw.process_web_assets()
 
         elif self.profile == 'openvas':
             vw_openvas = vulnWhispererOpenVAS(config=self.config)
-            self.exit_code += vw_openvas.process_openvas_scans()
+            if vw:
+                self.exit_code += vw_openvas.process_openvas_scans()
 
         elif self.profile == 'tenable':
             vw = vulnWhispererNessus(config=self.config,
                                      profile=self.profile)
-            self.exit_code += vw.whisper_nessus()
+            if vw:
+                self.exit_code += vw.whisper_nessus()
 
         elif self.profile == 'qualys_vuln':
             vw = vulnWhispererQualysVuln(config=self.config)
-            self.exit_code += vw.process_vuln_scans()
+            if vw:
+                self.exit_code += vw.process_vuln_scans()
 
         elif self.profile == 'jira':
             #first we check config fields are created, otherwise we create them
             vw = vulnWhispererJIRA(config=self.config)
-            if not (self.source and self.scanname):
+            if vw:
-                self.logger.info('No source/scan_name selected, all enabled scans will be synced')
+                if not (self.source and self.scanname):
-                success = vw.sync_all()
+                    self.logger.info('No source/scan_name selected, all enabled scans will be synced')
-                if not success:
+                    success = vw.sync_all()
-                    self.logger.error('All scans sync failed!')
+                    if not success:
-                    self.logger.error('Source scanner and scan name needed!')
+                        self.logger.error('All scans sync failed!')
-                    return 0
+                        self.logger.error('Source scanner and scan name needed!')
-            else:
+                        return 0
-                vw.jira_sync(self.source, self.scanname)
+                else:
+                    vw.jira_sync(self.source, self.scanname)
 
         return self.exit_code