modernize python2 to python3 applied

bin/vuln_whisperer

@@ -93,7 +93,7 @@ def main():
                                        scanname=args.scanname)
                     exit_code += vw.whisper_vulnerabilities()
                 except Exception as e:
-                    logger.error("VulnWhisperer was unable to perform the processing on '{}'".format(args.source))
+                    logger.error("VulnWhisperer was unable to perform the processing on '{}'".format(section))
         else:
             logger.info('Running vulnwhisperer for section {}'.format(args.section))
             vw = vulnWhisperer(config=args.config,

setup.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
+from __future__ import absolute_import
 from setuptools import setup, find_packages
 
 setup(

vulnwhisp/base/config.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import sys
 import logging
 
@@ -5,7 +6,7 @@ import logging
 if sys.version_info > (3, 0):
     import configparser as cp
 else:
-    import ConfigParser as cp
+    import six.moves.configparser as cp
 
 
 class vwConfig(object):

vulnwhisp/frameworks/nessus.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import json
 import logging
 import sys

vulnwhisp/frameworks/openvas.py

@@ -1,5 +1,6 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import
 __author__ = 'Austin Taylor'
 
 import datetime as dt

vulnwhisp/frameworks/qualys_vuln.py

@@ -1,5 +1,6 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import
 __author__ = 'Nathan Young'
 
 import logging

vulnwhisp/frameworks/qualys_web.py

@@ -1,5 +1,8 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import
+from six.moves import range
+from functools import reduce
 __author__ = 'Austin Taylor'
 
 from lxml import objectify

vulnwhisp/reporting/jira_api.py

@@ -1,15 +1,18 @@
+from __future__ import absolute_import
 import json
 import os
-from datetime import datetime, date, timedelta
+from datetime import datetime, date
 
 from jira import JIRA
-import requests
 import logging
 from bottle import template
 import re
+from six.moves import range
+
 
 class JiraAPI(object):
-    def __init__(self, hostname=None, username=None, password=None, path="", debug=False, clean_obsolete=True, max_time_window=12, decommission_time_window=3):
+    def __init__(self, hostname=None, username=None, password=None, path="", debug=False, clean_obsolete=True,
+                 max_time_window=12, decommission_time_window=3):
         self.logger = logging.getLogger('JiraAPI')
         if debug:
             self.logger.setLevel(logging.DEBUG)
@@ -41,10 +44,15 @@ class JiraAPI(object):
         # deletes the tag "server_decommission" from those tickets closed <=3 months ago
         self.decommission_cleanup()
 
-        self.jira_still_vulnerable_comment = '''This ticket has been reopened due to the vulnerability not having been fixed (if multiple assets are affected, all need to be fixed; if the server is down, lastest known vulnerability might be the one reported).
+        self.jira_still_vulnerable_comment = '''This ticket has been reopened due to the vulnerability not having been \
-        - In the case of the team accepting the risk and wanting to close the ticket, please add the label "*risk_accepted*" to the ticket before closing it.
+        fixed (if multiple assets are affected, all need to be fixed; if the server is down, lastest known \
-        - If server has been decommissioned, please add the label "*server_decommission*" to the ticket before closing it.
+        vulnerability might be the one reported).
-        - If when checking the vulnerability it looks like a false positive, _+please elaborate in a comment+_ and add the label "*false_positive*" before closing it; we will review it and report it to the vendor.
+        - In the case of the team accepting the risk and wanting to close the ticket, please add the label \
+        "*risk_accepted*" to the ticket before closing it.
+        - If server has been decommissioned, please add the label "*server_decommission*" to the ticket before closing \
+        it.
+        - If when checking the vulnerability it looks like a false positive, _+please elaborate in a comment+_ and add \
+         the label "*false_positive*" before closing it; we will review it and report it to the vendor.
         
         If you have further doubts, please contact the Security Team.'''
 
@@ -91,13 +99,15 @@ class JiraAPI(object):
         return len(self.jira.search_issues(jql, maxResults=0))
 
     def metrics_closed_tickets(self, project=None):
-        jql = "labels= vulnerability_management and NOT resolution = Unresolved AND created >=startOfMonth(-{})".format(self.max_time_tracking) 
+        jql = "labels= vulnerability_management and NOT resolution = Unresolved AND created >=startOfMonth(-{})".format(
+            self.max_time_tracking)
         if project:
             jql += " and (project='{}')".format(project)
         return len(self.jira.search_issues(jql, maxResults=0))
 
     def sync(self, vulnerabilities, project, components=[]):
-        #JIRA structure of each vulnerability: [source, scan_name, title, diagnosis, consequence, solution, ips, risk, references]
+        # JIRA structure of each vulnerability: [source, scan_name, title, diagnosis, consequence, solution,
+        # ips, risk, references]
         self.logger.info("JIRA Sync started")
 
         for vuln in vulnerabilities:
@@ -106,7 +116,8 @@ class JiraAPI(object):
             if " " in vuln['scan_name']:
                 vuln['scan_name'] = "_".join(vuln['scan_name'].split(" "))
 
-            # we exclude from the vulnerabilities to report those assets that already exist with *risk_accepted*/*server_decommission*
+            # we exclude from the vulnerabilities to report those assets that already exist
+            # with *risk_accepted*/*server_decommission*
             vuln = self.exclude_accepted_assets(vuln)
 
             # make sure after exclusion of risk_accepted assets there are still assets
@@ -131,13 +142,17 @@ class JiraAPI(object):
                 # create local text file with assets, attach it to ticket
                 if len(vuln['ips']) > self.max_ips_ticket:
                     attachment_contents = vuln['ips']
-                    vuln['ips'] = ["Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(assets = len(attachment_contents))]
+                    vuln['ips'] = [
+                        "Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(
+                            assets=len(attachment_contents))]
                 try:
                     tpl = template(self.template_path, vuln)
                 except Exception as e:
                     self.logger.error('Exception templating: {}'.format(str(e)))
                     return 0
-                self.create_ticket(title=vuln['title'], desc=tpl, project=project, components=components, tags=[vuln['source'], vuln['scan_name'], 'vulnerability', vuln['risk']], attachment_contents = attachment_contents)
+                self.create_ticket(title=vuln['title'], desc=tpl, project=project, components=components,
+                                   tags=[vuln['source'], vuln['scan_name'], 'vulnerability', vuln['risk']],
+                                   attachment_contents=attachment_contents)
             else:
                 self.logger.info("Ignoring vulnerability as all assets are already reported in a risk_accepted ticket")
 
@@ -153,7 +168,8 @@ class JiraAPI(object):
         labels = [vuln['source'], vuln['scan_name'], 'vulnerability_management', 'vulnerability']
 
         if not self.excluded_tickets:
-            jql = "{} AND labels in (risk_accepted,server_decommission, false_positive) AND NOT labels=advisory AND created >=startOfMonth(-{})".format(" AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
+            jql = "{} AND labels in (risk_accepted,server_decommission, false_positive) AND NOT labels=advisory AND created >=startOfMonth(-{})".format(
+                " AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
             self.excluded_tickets = self.jira.search_issues(jql, maxResults=0)
 
         title = vuln['title']
@@ -163,7 +179,8 @@ class JiraAPI(object):
         assets_to_exclude = []
         tickets_excluded_assets = []
         for index in range(len(self.excluded_tickets)):
-            checking_ticketid, checking_title, checking_assets = self.ticket_get_unique_fields(self.excluded_tickets[index])
+            checking_ticketid, checking_title, checking_assets = self.ticket_get_unique_fields(
+                self.excluded_tickets[index])
             if title.encode('ascii') == checking_title.encode('ascii'):
                 if checking_assets:
                     # checking_assets is a list, we add to our full list for later delete all assets
@@ -172,7 +189,8 @@ class JiraAPI(object):
 
         if assets_to_exclude:
             assets_to_remove = []
-            self.logger.warn("Vulnerable Assets seen on an already existing risk_accepted Jira ticket: {}".format(', '.join(tickets_excluded_assets)))
+            self.logger.warn("Vulnerable Assets seen on an already existing risk_accepted Jira ticket: {}".format(
+                ', '.join(tickets_excluded_assets)))
             self.logger.debug("Original assets: {}".format(vuln['ips']))
             # assets in vulnerability have the structure "ip - hostname - port", so we need to match by partial
             for exclusion in assets_to_exclude:
@@ -180,7 +198,9 @@ class JiraAPI(object):
                 # and we don't want it to affect the rest of the processing (otherwise, it would miss the asset right after the removed one)
                 for index in range(len(vuln['ips']))[::-1]:
                     if exclusion == vuln['ips'][index].split(" - ")[0]:
-                        self.logger.debug("Deleting asset {} from vulnerability {}, seen in risk_accepted.".format(vuln['ips'][index], title))
+                        self.logger.debug(
+                            "Deleting asset {} from vulnerability {}, seen in risk_accepted.".format(vuln['ips'][index],
+                                                                                                     title))
                         vuln['ips'].pop(index)
             self.logger.debug("Modified assets: {}".format(vuln['ips']))
 
@@ -202,7 +222,8 @@ class JiraAPI(object):
             self.logger.info("Retrieving all JIRA tickets with the following tags {}".format(labels))
             # we want to check all JIRA tickets, to include tickets moved to other queues
             # will exclude tickets older than 12 months, old tickets will get closed for higiene and recreated if still vulnerable
-            jql = "{} AND NOT labels=advisory AND created >=startOfMonth(-{})".format(" AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
+            jql = "{} AND NOT labels=advisory AND created >=startOfMonth(-{})".format(
+                " AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
 
             self.all_tickets = self.jira.search_issues(jql, maxResults=0)
 
@@ -212,7 +233,8 @@ class JiraAPI(object):
         for index in range(len(self.all_tickets)):
             checking_ticketid, checking_title, checking_assets = self.ticket_get_unique_fields(self.all_tickets[index])
             # added "not risk_accepted", as if it is risk_accepted, we will create a new ticket excluding the accepted assets
-            if title.encode('ascii') == checking_title.encode('ascii') and not self.is_risk_accepted(self.jira.issue(checking_ticketid)): 
+            if title.encode('ascii') == checking_title.encode('ascii') and not self.is_risk_accepted(
+                    self.jira.issue(checking_ticketid)):
                 difference = list(set(assets).symmetric_difference(checking_assets))
                 # to check intersection - set(assets) & set(checking_assets)
                 if difference:
@@ -239,9 +261,12 @@ class JiraAPI(object):
         # structure the text to have the same structure as the assets from the attachment
         affected_assets = ""
         try:
-            affected_assets = ticket.raw.get('fields', {}).get('description').encode("ascii").split("{panel:title=Affected Assets}")[1].split("{panel}")[0].replace('\n','').replace(' * ','\n').replace('\n', '', 1)
+            affected_assets = \
+            ticket.raw.get('fields', {}).get('description').encode("ascii").split("{panel:title=Affected Assets}")[
+                1].split("{panel}")[0].replace('\n', '').replace(' * ', '\n').replace('\n', '', 1)
         except Exception as e:
-            self.logger.error("Unable to process the Ticket's 'Affected Assets'. Ticket ID: {}. Reason: {}".format(ticket, e))
+            self.logger.error(
+                "Unable to process the Ticket's 'Affected Assets'. Ticket ID: {}. Reason: {}".format(ticket, e))
 
         if affected_assets:
             if _raw:
@@ -280,7 +305,8 @@ class JiraAPI(object):
             affected_assets = self.jira.attachment(attachment_id).get()
 
         except Exception as e:
-            self.logger.error("Failed to get assets from ticket attachment. Ticket ID: {}. Reason: {}".format(ticket, e))
+            self.logger.error(
+                "Failed to get assets from ticket attachment. Ticket ID: {}. Reason: {}".format(ticket, e))
 
         if affected_assets:
             if _raw:
@@ -353,8 +379,10 @@ class JiraAPI(object):
         if self.is_ticket_resolved(ticket_obj):
             ticket_data = ticket_obj.raw.get('fields')
             # dates follow format '2018-11-06T10:36:13.849+0100'
-            created = [int(x) for x in ticket_data['created'].split('.')[0].replace('T', '-').replace(':','-').split('-')]
+            created = [int(x) for x in
-            resolved =[int(x) for x in ticket_data['resolutiondate'].split('.')[0].replace('T', '-').replace(':','-').split('-')]
+                       ticket_data['created'].split('.')[0].replace('T', '-').replace(':', '-').split('-')]
+            resolved = [int(x) for x in
+                        ticket_data['resolutiondate'].split('.')[0].replace('T', '-').replace(':', '-').split('-')]
 
             start = datetime(created[0], created[1], created[2], created[3], created[4], created[5])
             end = datetime(resolved[0], resolved[1], resolved[2], resolved[3], resolved[4], resolved[5])
@@ -405,7 +433,9 @@ class JiraAPI(object):
         attachment_contents = []
         if len(vuln['ips']) > self.max_ips_ticket:
             attachment_contents = vuln['ips']
-            vuln['ips'] = ["Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(assets = len(attachment_contents))]
+            vuln['ips'] = [
+                "Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(
+                    assets=len(attachment_contents))]
 
         # fill the ticket description template
         try:
@@ -425,7 +455,8 @@ class JiraAPI(object):
             self.logger.info("Ticket {} updated successfully".format(ticketid))
             self.add_label(ticketid, 'updated')
         except Exception as e:
-            self.logger.error("Error while trying up update ticket {ticketid}.\nReason: {e}".format(ticketid = ticketid, e=e))
+            self.logger.error(
+                "Error while trying up update ticket {ticketid}.\nReason: {e}".format(ticketid=ticketid, e=e))
         return 0
 
     def add_label(self, ticketid, label):
@@ -437,8 +468,9 @@ class JiraAPI(object):
             try:
                 ticket_obj.update(fields={"labels": ticket_obj.fields.labels})
                 self.logger.info("Added label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
-            except:
+            except Exception as e:
-                self.logger.error("Error while trying to add label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
+                self.logger.error(
+                    "Error while trying to add label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
 
         return 0
 
@@ -451,8 +483,9 @@ class JiraAPI(object):
             try:
                 ticket_obj.update(fields={"labels": ticket_obj.fields.labels})
                 self.logger.info("Removed label {label} from ticket {ticket}".format(label=label, ticket=ticketid))
-            except:
+            except Exception as e:
-                self.logger.error("Error while trying to remove label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
+                self.logger.error("Error while trying to remove label {label} to ticket {ticket}".format(label=label,
+                                                                                                         ticket=ticketid))
         else:
             self.logger.error("Error: label {label} not in ticket {ticket}".format(label=label, ticket=ticketid))
 
@@ -478,7 +511,6 @@ class JiraAPI(object):
             self.close_ticket(ticket, self.JIRA_RESOLUTION_FIXED, comment)
         return 0
 
-
     def is_ticket_reopenable(self, ticket_obj):
         transitions = self.jira.transitions(ticket_obj)
         for transition in transitions:
@@ -507,7 +539,6 @@ class JiraAPI(object):
         self.logger.debug("Checked ticket {} is already open".format(ticket_obj))
         return False
 
-
     def is_risk_accepted(self, ticket_obj):
         if ticket_obj is not None:
             if ticket_obj.raw['fields'].get('labels') is not None:
@@ -533,7 +564,8 @@ class JiraAPI(object):
             if (not self.is_risk_accepted(ticket_obj) or ignore_labels):
                 try:
                     if self.is_ticket_reopenable(ticket_obj):
-                        error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_REOPEN_ISSUE, comment = comment)
+                        error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_REOPEN_ISSUE,
+                                                           comment=comment)
                         self.logger.info("Ticket {} reopened successfully".format(ticketid))
                         if not ignore_labels:
                             self.add_label(ticketid, 'reopened')
@@ -553,7 +585,8 @@ class JiraAPI(object):
                 if self.is_ticket_closeable(ticket_obj):
                     # need to add the label before closing the ticket
                     self.add_label(ticketid, 'closed')
-                    error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_CLOSE_ISSUE, comment = comment, resolution = {"name": resolution })
+                    error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_CLOSE_ISSUE,
+                                                       comment=comment, resolution={"name": resolution})
                     self.logger.info("Ticket {} closed successfully".format(ticketid))
                     return 1
             except Exception as e:
@@ -566,7 +599,8 @@ class JiraAPI(object):
     def close_obsolete_tickets(self):
         # Close tickets older than 12 months, vulnerabilities not solved will get created a new ticket 
         self.logger.info("Closing obsolete tickets older than {} months".format(self.max_time_tracking))
-        jql = "labels=vulnerability_management AND NOT labels=advisory AND created <startOfMonth(-{}) and resolution=Unresolved".format(self.max_time_tracking)
+        jql = "labels=vulnerability_management AND NOT labels=advisory AND created <startOfMonth(-{}) and resolution=Unresolved".format(
+            self.max_time_tracking)
         tickets_to_close = self.jira.search_issues(jql, maxResults=0)
 
         comment = '''This ticket is being closed for hygiene, as it is more than {} months old.
@@ -597,7 +631,8 @@ class JiraAPI(object):
             return True
         try:
             self.logger.info("Saving locally tickets from the last {} months".format(self.max_time_tracking))
-            jql = "labels=vulnerability_management AND NOT labels=advisory AND created >=startOfMonth(-{})".format(self.max_time_tracking)
+            jql = "labels=vulnerability_management AND NOT labels=advisory AND created >=startOfMonth(-{})".format(
+                self.max_time_tracking)
             tickets_data = self.jira.search_issues(jql, maxResults=0)
 
             # TODO process tickets, creating a new field called "_metadata" with all the affected assets well structured
@@ -621,7 +656,6 @@ class JiraAPI(object):
                         assets_json = self.parse_asset_to_json(assets)
                         _metadata["affected_hosts"].append(assets_json)
 
-
                 temp_ticket = ticket.raw.get('fields')
                 temp_ticket['_metadata'] = _metadata
 
@@ -646,13 +680,16 @@ class JiraAPI(object):
         closed already for more than x months (default is 3 months) in order to clean solved issues
         for statistics purposes
         '''
-        self.logger.info("Deleting 'server_decommission' tag from tickets closed more than {} months ago".format(self.max_decommission_time))
+        self.logger.info("Deleting 'server_decommission' tag from tickets closed more than {} months ago".format(
+            self.max_decommission_time))
 
-        jql = "labels=vulnerability_management AND labels=server_decommission and resolutiondate <=startOfMonth(-{})".format(self.max_decommission_time)
+        jql = "labels=vulnerability_management AND labels=server_decommission and resolutiondate <=startOfMonth(-{})".format(
+            self.max_decommission_time)
         decommissioned_tickets = self.jira.search_issues(jql, maxResults=0)
 
         comment = '''This ticket is having deleted the *server_decommission* tag, as it is more than {} months old and is expected to already have been decommissioned.
-        If that is not the case and the vulnerability still exists, the vulnerability will be opened again.'''.format(self.max_decommission_time)
+        If that is not the case and the vulnerability still exists, the vulnerability will be opened again.'''.format(
+            self.max_decommission_time)
 
         for ticket in decommissioned_tickets:
             # we open first the ticket, as we want to make sure the process is not blocked due to

vulnwhisp/test/mock.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import os
 import logging
 import httpretty

vulnwhisp/vulnwhisp.py

@@ -1,13 +1,16 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import
+from six.moves import range
+from functools import reduce
 __author__ = 'Austin Taylor'
 
-from base.config import vwConfig
+from .base.config import vwConfig
-from frameworks.nessus import NessusAPI
+from .frameworks.nessus import NessusAPI
-from frameworks.qualys_web import qualysScanReport
+from .frameworks.qualys_web import qualysScanReport
-from frameworks.qualys_vuln import qualysVulnScan
+from .frameworks.qualys_vuln import qualysVulnScan
-from frameworks.openvas import OpenVAS_API
+from .frameworks.openvas import OpenVAS_API
-from reporting.jira_api import JiraAPI
+from .reporting.jira_api import JiraAPI
 import pandas as pd
 from lxml import objectify
 import sys
@@ -1200,7 +1203,7 @@ class vulnWhispererJIRA(vulnWhispererBase):
             if vuln['dns']:
                 values['dns'] = vuln['dns']
             else:
-                if values['ip'] in self.host_resolv_cache.keys():
+                if values['ip'] in list(self.host_resolv_cache.keys()):
                     self.logger.debug("Hostname from {ip} cached, retrieving from cache.".format(ip=values['ip']))
                     values['dns'] = self.host_resolv_cache[values['ip']]
                 else: