Merge pull request #230 from nfalke-/2to3

2to3

bin/vuln_whisperer

@@ -93,7 +93,7 @@ def main():
                                        scanname=args.scanname)
                     exit_code += vw.whisper_vulnerabilities()
                 except Exception as e:
-                    logger.error("VulnWhisperer was unable to perform the processing on '{}'".format(args.source))
+                    logger.error("VulnWhisperer was unable to perform the processing on '{}'".format(section))
         else:
             logger.info('Running vulnwhisperer for section {}'.format(args.section))
             vw = vulnWhisperer(config=args.config,

configs/test.ini

@@ -6,8 +6,8 @@ access_key=
 secret_key=
 username=nessus_username
 password=nessus_password
-write_path=/opt/VulnWhisperer/data/nessus/
-db_path=/opt/VulnWhisperer/data/database
+write_path=/tmp/VulnWhisperer/data/nessus/
+db_path=/tmp/VulnWhisperer/data/database
 trash=false
 verbose=true
 
@@ -19,8 +19,8 @@ access_key=
 secret_key=
 username=tenable.io_username
 password=tenable.io_password
-write_path=/opt/VulnWhisperer/data/tenable/
-db_path=/opt/VulnWhisperer/data/database
+write_path=/tmp/VulnWhisperer/data/tenable/
+db_path=/tmp/VulnWhisperer/data/database
 trash=false
 verbose=true
 
@@ -30,8 +30,8 @@ enabled = false
 hostname = qualys_web
 username = exampleuser
 password = examplepass
-write_path=/opt/VulnWhisperer/data/qualys_web/
-db_path=/opt/VulnWhisperer/data/database
+write_path=/tmp/VulnWhisperer/data/qualys_web/
+db_path=/tmp/VulnWhisperer/data/database
 verbose=true
 
 # Set the maximum number of retries each connection should attempt.
@@ -46,8 +46,8 @@ enabled = true
 hostname = qualys_vuln
 username = exampleuser
 password = examplepass
-write_path=/opt/VulnWhisperer/data/qualys_vuln/
-db_path=/opt/VulnWhisperer/data/database
+write_path=/tmp/VulnWhisperer/data/qualys_vuln/
+db_path=/tmp/VulnWhisperer/data/database
 verbose=true
 
 [detectify]
@@ -58,8 +58,8 @@ hostname = detectify
 username = exampleuser
 #password variable used as secretKey
 password = examplepass
-write_path =/opt/VulnWhisperer/data/detectify/
-db_path = /opt/VulnWhisperer/data/database
+write_path =/tmp/VulnWhisperer/data/detectify/
+db_path = /tmp/VulnWhisperer/data/database
 verbose = true
 
 [openvas]
@@ -68,8 +68,8 @@ hostname = openvas
 port = 4000
 username = exampleuser
 password = examplepass
-write_path=/opt/VulnWhisperer/data/openvas/
-db_path=/opt/VulnWhisperer/data/database
+write_path=/tmp/VulnWhisperer/data/openvas/
+db_path=/tmp/VulnWhisperer/data/database
 verbose=true
 
 [jira]
@@ -77,8 +77,8 @@ enabled = false
 hostname = jira-host
 username = username
 password = password
-write_path = /opt/VulnWhisperer/data/jira/
-db_path = /opt/VulnWhisperer/data/database
+write_path = /tmp/VulnWhisperer/data/jira/
+db_path = /tmp/VulnWhisperer/data/database
 verbose = true
 dns_resolv = False
 

setup.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
+from __future__ import absolute_import
 from setuptools import setup, find_packages
 
 setup(

vulnwhisp/base/config.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import sys
 import logging
 
@@ -5,7 +6,7 @@ import logging
 if sys.version_info > (3, 0):
     import configparser as cp
 else:
-    import ConfigParser as cp
+    import six.moves.configparser as cp
 
 
 class vwConfig(object):

vulnwhisp/frameworks/nessus.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import json
 import logging
 import sys

vulnwhisp/frameworks/openvas.py

@@ -1,5 +1,6 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import
 __author__ = 'Austin Taylor'
 
 import datetime as dt

vulnwhisp/frameworks/qualys_vuln.py

@@ -1,5 +1,6 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import
 __author__ = 'Nathan Young'
 
 import logging
@@ -18,9 +19,9 @@ class qualysWhisperAPI(object):
         self.logger = logging.getLogger('qualysWhisperAPI')
         self.config = config
         try:
-            self.qgc = qualysapi.connect(config, 'qualys_vuln')
+            self.qgc = qualysapi.connect(config_file=config, section='qualys_vuln')
             # Fail early if we can't make a request or auth is incorrect
-            self.qgc.request('about.php')
+            # self.qgc.request('about.php')
             self.logger.info('Connected to Qualys at {}'.format(self.qgc.server))
         except Exception as e:
             self.logger.error('Could not connect to Qualys: {}'.format(str(e)))

vulnwhisp/frameworks/qualys_web.py

@@ -1,5 +1,8 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import
+from six.moves import range
+from functools import reduce
 __author__ = 'Austin Taylor'
 
 from lxml import objectify
@@ -14,24 +17,16 @@ import os
 import csv
 import logging
 import dateutil.parser as dp
+csv.field_size_limit(sys.maxsize)
 
 
 class qualysWhisperAPI(object):
-    COUNT_WEBAPP = '/count/was/webapp'
     COUNT_WASSCAN = '/count/was/wasscan'
     DELETE_REPORT = '/delete/was/report/{report_id}'
-    GET_WEBAPP_DETAILS = '/get/was/webapp/{was_id}'
-    QPS_REST_3 = '/qps/rest/3.0'
-    REPORT_DETAILS = '/get/was/report/{report_id}'
     REPORT_STATUS = '/status/was/report/{report_id}'
     REPORT_CREATE = '/create/was/report'
     REPORT_DOWNLOAD = '/download/was/report/{report_id}'
-    SCAN_DETAILS = '/get/was/wasscan/{scan_id}'
-    SCAN_DOWNLOAD = '/download/was/wasscan/{scan_id}'
-    SEARCH_REPORTS = '/search/was/report'
-    SEARCH_WEB_APPS = '/search/was/webapp'
     SEARCH_WAS_SCAN = '/search/was/wasscan'
-    VERSION = '/qps/rest/portal/version'
 
     def __init__(self, config=None):
         self.logger = logging.getLogger('qualysWhisperAPI')
@@ -41,10 +36,6 @@ class qualysWhisperAPI(object):
             self.logger.info('Connected to Qualys at {}'.format(self.qgc.server))
         except Exception as e:
             self.logger.error('Could not connect to Qualys: {}'.format(str(e)))
-        self.headers = {
-            #"content-type": "text/xml"}
-            "Accept" : "application/json",
-            "Content-Type": "application/json"}
         self.config_parse = qcconf.QualysConnectConfig(config, 'qualys_web')
         try:
             self.template_id = self.config_parse.get_template_id()
@@ -69,14 +60,8 @@ class qualysWhisperAPI(object):
 
     def generate_scan_result_XML(self, limit=1000, offset=1, status='FINISHED'):
         report_xml = E.ServiceRequest(
-            E.filters(
-                E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status
-                           ),
-            ),
-            E.preferences(
-                E.startFromOffset(str(offset)),
-                E.limitResults(str(limit))
-            ),
+            E.filters(E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status)),
+            E.preferences(E.startFromOffset(str(offset)), E.limitResults(str(limit))),
         )
         return report_xml
 
@@ -115,8 +100,10 @@ class qualysWhisperAPI(object):
                 if i % limit == 0:
                     if (total - i) < limit:
                         qualys_api_limit = total - i
-                    self.logger.info('Making a request with a limit of {} at offset {}'.format((str(qualys_api_limit)), str(i + 1)))
-                    scan_info = self.get_scan_info(limit=qualys_api_limit, offset=i + 1, status=status)
+                    self.logger.info('Making a request with a limit of {} at offset {}'
+                            .format((str(qualys_api_limit)), str(i + 1)))
+                    scan_info = self.get_scan_info(
+                            limit=qualys_api_limit, offset=i + 1, status=status)
                     _records.append(scan_info)
             self.logger.debug('Converting XML to DataFrame')
             dataframes = [self.xml_parser(xml) for xml in _records]
@@ -133,7 +120,8 @@ class qualysWhisperAPI(object):
         return self.qgc.request(self.REPORT_STATUS.format(report_id=report_id))
 
     def download_report(self, report_id):
-        return self.qgc.request(self.REPORT_DOWNLOAD.format(report_id=report_id))
+        return self.qgc.request(
+                self.REPORT_DOWNLOAD.format(report_id=report_id), http_method='get')
 
     def generate_scan_report_XML(self, scan_id):
         """Generates a CSV report for an asset based on template defined in .ini file"""
@@ -145,20 +133,8 @@ class qualysWhisperAPI(object):
                     E.format('CSV'),
                     #type is not needed, as the template already has it
                     E.type('WAS_SCAN_REPORT'),
-                    E.template(
-                        E.id(self.template_id)
-                    ),
-                    E.config(
-                        E.scanReport(
-                            E.target(
-                                E.scans(
-                                    E.WasScan(
-                                        E.id(scan_id)
-                                    )
-                                ),
-                            ),
-                        ),
-                    )
+                    E.template(E.id(self.template_id)),
+                    E.config(E.scanReport(E.target(E.scans(E.WasScan(E.id(scan_id))))))
                 )
             )
         )
@@ -175,95 +151,14 @@ class qualysWhisperAPI(object):
     def delete_report(self, report_id):
         return self.qgc.request(self.DELETE_REPORT.format(report_id=report_id))
 
-
-class qualysReportFields:
-    CATEGORIES = ['VULNERABILITY',
-                  'SENSITIVECONTENT',
-                  'INFORMATION_GATHERED']
-
-    # URL Vulnerability Information
-
-    VULN_BLOCK = [
-        CATEGORIES[0],
-        'ID',
-        'QID',
-        'Url',
-        'Param',
-        'Function',
-        'Form Entry Point',
-        'Access Path',
-        'Authentication',
-        'Ajax Request',
-        'Ajax Request ID',
-        'Ignored',
-        'Ignore Reason',
-        'Ignore Date',
-        'Ignore User',
-        'Ignore Comments',
-        'First Time Detected',
-        'Last Time Detected',
-        'Last Time Tested',
-        'Times Detected',
-        'Payload #1',
-        'Request Method #1',
-        'Request URL #1',
-        'Request Headers #1',
-        'Response #1',
-        'Evidence #1',
-    ]
-
-    INFO_HEADER = [
-        'Vulnerability Category',
-        'ID',
-        'QID',
-        'Response #1',
-        'Last Time Detected',
-    ]
-    INFO_BLOCK = [
-        CATEGORIES[2],
-        'ID',
-        'QID',
-        'Results',
-        'Detection Date',
-    ]
-
-    QID_HEADER = [
-        'QID',
-        'Id',
-        'Title',
-        'Category',
-        'Severity Level',
-        'Groups',
-        'OWASP',
-        'WASC',
-        'CWE',
-        'CVSS Base',
-        'CVSS Temporal',
-        'Description',
-        'Impact',
-        'Solution',
-    ]
-    GROUP_HEADER = ['GROUP', 'Name', 'Category']
-    OWASP_HEADER = ['OWASP', 'Code', 'Name']
-    WASC_HEADER = ['WASC', 'Code', 'Name']
-    SCAN_META = ['Web Application Name', 'URL', 'Owner', 'Scope', 'Operating System']
-    CATEGORY_HEADER = ['Category', 'Severity', 'Level', 'Description']
-
-
 class qualysUtils:
     def __init__(self):
         self.logger = logging.getLogger('qualysUtils')
 
-    def grab_section(
-            self,
-            report,
-            section,
-            end=[],
-            pop_last=False,
-    ):
+    def grab_section(self, report, section, end=[], pop_last=False):
         temp_list = []
         max_col_count = 0
-        with open(report, 'rb') as csvfile:
+        with open(report, 'rt') as csvfile:
             q_report = csv.reader(csvfile, delimiter=',', quotechar='"')
             for line in q_report:
                 if set(line) == set(section):
@@ -289,44 +184,53 @@ class qualysUtils:
         return _data
 
 class qualysScanReport:
-    # URL Vulnerability Information
-    WEB_SCAN_VULN_BLOCK = list(qualysReportFields.VULN_BLOCK)
-    WEB_SCAN_VULN_BLOCK.insert(WEB_SCAN_VULN_BLOCK.index('QID'), 'Detection ID')
+    CATEGORIES = ['VULNERABILITY', 'SENSITIVECONTENT', 'INFORMATION_GATHERED']
 
-    WEB_SCAN_VULN_HEADER = list(WEB_SCAN_VULN_BLOCK)
-    WEB_SCAN_VULN_HEADER[WEB_SCAN_VULN_BLOCK.index(qualysReportFields.CATEGORIES[0])] = \
-        'Vulnerability Category'
+    WEB_SCAN_BLOCK = [
+        "ID", "Detection ID", "QID", "Url", "Param/Cookie", "Function",
+        "Form Entry Point", "Access Path", "Authentication", "Ajax Request",
+        "Ajax Request ID", "Ignored", "Ignore Reason", "Ignore Date", "Ignore User",
+        "Ignore Comments", "Detection Date", "Payload #1", "Request Method #1",
+        "Request URL #1", "Request Headers #1", "Response #1", "Evidence #1",
+        "Unique ID", "Flags", "Protocol", "Virtual Host", "IP", "Port", "Result",
+        "Info#1", "CVSS V3 Base", "CVSS V3 Temporal", "CVSS V3 Attack Vector",
+        "Request Body #1"
+    ]
+    WEB_SCAN_VULN_BLOCK = [CATEGORIES[0]] + WEB_SCAN_BLOCK
+    WEB_SCAN_SENSITIVE_BLOCK = [CATEGORIES[1]] + WEB_SCAN_BLOCK
 
-    WEB_SCAN_SENSITIVE_HEADER = list(WEB_SCAN_VULN_HEADER)
-    WEB_SCAN_SENSITIVE_HEADER.insert(WEB_SCAN_SENSITIVE_HEADER.index('Url'
-                                                                     ), 'Content')
+    WEB_SCAN_HEADER = ["Vulnerability Category"] + WEB_SCAN_BLOCK
+    WEB_SCAN_HEADER[WEB_SCAN_HEADER.index("Detection Date")] = "Last Time Detected"
 
-    WEB_SCAN_SENSITIVE_BLOCK = list(WEB_SCAN_SENSITIVE_HEADER)
-    WEB_SCAN_SENSITIVE_BLOCK.insert(WEB_SCAN_SENSITIVE_BLOCK.index('QID'), 'Detection ID')
-    WEB_SCAN_SENSITIVE_BLOCK[WEB_SCAN_SENSITIVE_BLOCK.index('Vulnerability Category'
-                                                            )] = qualysReportFields.CATEGORIES[1]
 
-    WEB_SCAN_INFO_HEADER = list(qualysReportFields.INFO_HEADER)
-    WEB_SCAN_INFO_HEADER.insert(WEB_SCAN_INFO_HEADER.index('QID'), 'Detection ID')
+    WEB_SCAN_INFO_BLOCK = [
+        "INFORMATION_GATHERED", "ID", "Detection ID", "QID", "Results", "Detection Date",
+        "Unique ID", "Flags", "Protocol", "Virtual Host", "IP", "Port", "Result",
+        "Info#1"
+    ]
 
-    WEB_SCAN_INFO_BLOCK = list(qualysReportFields.INFO_BLOCK)
-    WEB_SCAN_INFO_BLOCK.insert(WEB_SCAN_INFO_BLOCK.index('QID'), 'Detection ID')
+    WEB_SCAN_INFO_HEADER = [
+        "Vulnerability Category", "ID", "Detection ID", "QID", "Results", "Last Time Detected",
+        "Unique ID", "Flags", "Protocol", "Virtual Host", "IP", "Port", "Result",
+        "Info#1"
+    ]
 
-    QID_HEADER = list(qualysReportFields.QID_HEADER)
-    GROUP_HEADER = list(qualysReportFields.GROUP_HEADER)
-    OWASP_HEADER = list(qualysReportFields.OWASP_HEADER)
-    WASC_HEADER = list(qualysReportFields.WASC_HEADER)
-    SCAN_META = list(qualysReportFields.SCAN_META)
-    CATEGORY_HEADER = list(qualysReportFields.CATEGORY_HEADER)
+    QID_HEADER = [
+        "QID", "Id", "Title", "Category", "Severity Level", "Groups", "OWASP", "WASC",
+        "CWE", "CVSS Base", "CVSS Temporal", "Description", "Impact", "Solution",
+        "CVSS V3 Base", "CVSS V3 Temporal", "CVSS V3 Attack Vector"
+    ]
+    GROUP_HEADER = ['GROUP', 'Name', 'Category']
+    OWASP_HEADER = ['OWASP', 'Code', 'Name']
+    WASC_HEADER = ['WASC', 'Code', 'Name']
+    SCAN_META = [
+        "Web Application Name", "URL", "Owner", "Scope", "ID", "Tags",
+        "Custom Attributes"
+    ]
+    CATEGORY_HEADER = ['Category', 'Severity', 'Level', 'Description']
 
-    def __init__(
-            self,
-            config=None,
-            file_in=None,
-            file_stream=False,
-            delimiter=',',
-            quotechar='"',
-    ):
+    def __init__(self, config=None, file_in=None, 
+            file_stream=False, delimiter=',', quotechar='"'):
         self.logger = logging.getLogger('qualysScanReport')
         self.file_in = file_in
         self.file_stream = file_stream
@@ -337,71 +241,79 @@ class qualysScanReport:
             try:
                 self.qw = qualysWhisperAPI(config=config)
             except Exception as e:
-                self.logger.error('Could not load config! Please check settings. Error: {}'.format(str(e)))
+                self.logger.error(
+                        'Could not load config! Please check settings. Error: {}'.format(
+                            str(e)))
 
         if file_stream:
             self.open_file = file_in.splitlines()
         elif file_in:
-
             self.open_file = open(file_in, 'rb')
 
         self.downloaded_file = None
 
     def grab_sections(self, report):
-        all_dataframes = []
-        dict_tracker = {}
-        with open(report, 'rb') as csvfile:
-            dict_tracker['WEB_SCAN_VULN_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
-                                                                                       self.WEB_SCAN_VULN_BLOCK,
-                                                                                       end=[
-                                                                                           self.WEB_SCAN_SENSITIVE_BLOCK,
-                                                                                           self.WEB_SCAN_INFO_BLOCK],
-                                                                                       pop_last=True),
-                                                               columns=self.WEB_SCAN_VULN_HEADER)
-            dict_tracker['WEB_SCAN_SENSITIVE_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
-                                                                                            self.WEB_SCAN_SENSITIVE_BLOCK,
-                                                                                            end=[
-                                                                                                self.WEB_SCAN_INFO_BLOCK,
-                                                                                                self.WEB_SCAN_SENSITIVE_BLOCK],
-                                                                                            pop_last=True),
-                                                                columns=self.WEB_SCAN_SENSITIVE_HEADER)
-            dict_tracker['WEB_SCAN_INFO_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
-                                                                                       self.WEB_SCAN_INFO_BLOCK,
-                                                                                       end=[self.QID_HEADER],
-                                                                                       pop_last=True),
-                                                                columns=self.WEB_SCAN_INFO_HEADER)
-            dict_tracker['QID_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
-                                                                              self.QID_HEADER,
-                                                                              end=[self.GROUP_HEADER],
-                                                                              pop_last=True),
-                                                                columns=self.QID_HEADER)
-            dict_tracker['GROUP_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
-                                                                                self.GROUP_HEADER,
-                                                                                end=[self.OWASP_HEADER],
-                                                                                pop_last=True),
-                                                                columns=self.GROUP_HEADER)
-            dict_tracker['OWASP_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
-                                                                                self.OWASP_HEADER,
-                                                                                end=[self.WASC_HEADER],
-                                                                                pop_last=True),
-                                                                columns=self.OWASP_HEADER)
-            dict_tracker['WASC_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
-                                                                               self.WASC_HEADER, end=[['APPENDIX']],
-                                                                               pop_last=True),
-                                                                columns=self.WASC_HEADER)
+        return {
+            'WEB_SCAN_VULN_BLOCK': pd.DataFrame(
+                self.utils.grab_section(
+                    report,
+                    self.WEB_SCAN_VULN_BLOCK,
+                    end=[self.WEB_SCAN_SENSITIVE_BLOCK, self.WEB_SCAN_INFO_BLOCK],
+                    pop_last=True),
+                columns=self.WEB_SCAN_HEADER),
+            'WEB_SCAN_SENSITIVE_BLOCK': pd.DataFrame(
+                self.utils.grab_section(report,
+                    self.WEB_SCAN_SENSITIVE_BLOCK,
+                    end=[self.WEB_SCAN_INFO_BLOCK, self.WEB_SCAN_SENSITIVE_BLOCK],
+                    pop_last=True),
+                columns=self.WEB_SCAN_HEADER),
+            'WEB_SCAN_INFO_BLOCK': pd.DataFrame(
+                self.utils.grab_section(
+                    report,
+                    self.WEB_SCAN_INFO_BLOCK,
+                    end=[self.QID_HEADER],
+                    pop_last=True),
+                columns=self.WEB_SCAN_INFO_HEADER),
 
-            dict_tracker['SCAN_META'] = pd.DataFrame(self.utils.grab_section(report,
-                                                                             self.SCAN_META,
-                                                                             end=[self.CATEGORY_HEADER],
-                                                                             pop_last=True),
-                                                                columns=self.SCAN_META)
-
-            dict_tracker['CATEGORY_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
-                                                                                   self.CATEGORY_HEADER),
-                                                                columns=self.CATEGORY_HEADER)
-            all_dataframes.append(dict_tracker)
-
-        return all_dataframes
+            'QID_HEADER': pd.DataFrame(
+                self.utils.grab_section(
+                    report,
+                    self.QID_HEADER,
+                    end=[self.GROUP_HEADER],
+                    pop_last=True),
+                columns=self.QID_HEADER),
+            'GROUP_HEADER': pd.DataFrame(
+                self.utils.grab_section(
+                    report,
+                    self.GROUP_HEADER,
+                    end=[self.OWASP_HEADER],
+                    pop_last=True),
+                columns=self.GROUP_HEADER),
+            'OWASP_HEADER': pd.DataFrame(
+                self.utils.grab_section(
+                    report,
+                    self.OWASP_HEADER,
+                    end=[self.WASC_HEADER],
+                    pop_last=True),
+                columns=self.OWASP_HEADER),
+            'WASC_HEADER': pd.DataFrame(
+                self.utils.grab_section(
+                    report,
+                    self.WASC_HEADER,
+                    end=[['APPENDIX']],
+                    pop_last=True),
+                columns=self.WASC_HEADER),
+            'SCAN_META': pd.DataFrame(
+                self.utils.grab_section(report,
+                    self.SCAN_META,
+                    end=[self.CATEGORY_HEADER],
+                    pop_last=True),
+                columns=self.SCAN_META),
+            'CATEGORY_HEADER': pd.DataFrame(
+                self.utils.grab_section(report,
+                    self.CATEGORY_HEADER),
+                columns=self.CATEGORY_HEADER)
+        }
 
     def data_normalizer(self, dataframes):
         """
@@ -409,12 +321,21 @@ class qualysScanReport:
         :param dataframes:
         :return:
         """
-        df_dict = dataframes[0]
-        merged_df = pd.concat([df_dict['WEB_SCAN_VULN_BLOCK'], df_dict['WEB_SCAN_SENSITIVE_BLOCK'],
-                               df_dict['WEB_SCAN_INFO_BLOCK']], axis=0,
-                              ignore_index=False)
-        merged_df = pd.merge(merged_df, df_dict['QID_HEADER'], left_on='QID',
-                             right_on='Id')
+        df_dict = dataframes
+        merged_df = pd.concat([
+            df_dict['WEB_SCAN_VULN_BLOCK'],
+            df_dict['WEB_SCAN_SENSITIVE_BLOCK'],
+            df_dict['WEB_SCAN_INFO_BLOCK']
+        ], axis=0, ignore_index=False)
+
+        merged_df = pd.merge(
+                merged_df,
+                df_dict['QID_HEADER'].drop(
+                    #these columns always seem to be the same as what we're merging into
+                    ['CVSS V3 Attack Vector', 'CVSS V3 Base', 'CVSS V3 Temporal'],
+                    axis=1),
+                left_on='QID', right_on='Id'
+        )
 
         if 'Content' not in merged_df:
             merged_df['Content'] = ''
@@ -431,8 +352,11 @@ class qualysScanReport:
 
         merged_df = merged_df.assign(**df_dict['SCAN_META'].to_dict(orient='records')[0])
 
-        merged_df = pd.merge(merged_df, df_dict['CATEGORY_HEADER'], how='left', left_on=['Category', 'Severity Level'],
-                             right_on=['Category', 'Severity'], suffixes=('Severity', 'CatSev'))
+        merged_df = pd.merge(
+                merged_df, df_dict['CATEGORY_HEADER'],
+                how='left', left_on=['Category', 'Severity Level'],
+                right_on=['Category', 'Severity'], suffixes=('Severity', 'CatSev')
+        )
 
         merged_df = merged_df.replace('N/A', '').fillna('')
 

vulnwhisp/reporting/jira_api.py

@@ -1,15 +1,18 @@
+from __future__ import absolute_import
 import json
 import os
-from datetime import datetime, date, timedelta
+from datetime import datetime, date
 
 from jira import JIRA
-import requests
 import logging
 from bottle import template
 import re
+from six.moves import range
+
 
 class JiraAPI(object):
-    def __init__(self, hostname=None, username=None, password=None, path="", debug=False, clean_obsolete=True, max_time_window=12, decommission_time_window=3):
+    def __init__(self, hostname=None, username=None, password=None, path="", debug=False, clean_obsolete=True,
+                 max_time_window=12, decommission_time_window=3):
         self.logger = logging.getLogger('JiraAPI')
         if debug:
             self.logger.setLevel(logging.DEBUG)
@@ -29,26 +32,31 @@ class JiraAPI(object):
         self.template_path = 'vulnwhisp/reporting/resources/ticket.tpl'
         self.max_ips_ticket = 30
         self.attachment_filename = "vulnerable_assets.txt"
-        self.max_time_tracking = max_time_window #in months
+        self.max_time_tracking = max_time_window  # in months
         if path:
             self.download_tickets(path)
         else:
             self.logger.warn("No local path specified, skipping Jira ticket download.")
-        self.max_decommission_time = decommission_time_window #in months
+        self.max_decommission_time = decommission_time_window  # in months
         # [HIGIENE] close tickets older than 12 months as obsolete (max_time_window defined)
         if clean_obsolete:
             self.close_obsolete_tickets()
         # deletes the tag "server_decommission" from those tickets closed <=3 months ago
         self.decommission_cleanup()
-        
-        self.jira_still_vulnerable_comment = '''This ticket has been reopened due to the vulnerability not having been fixed (if multiple assets are affected, all need to be fixed; if the server is down, lastest known vulnerability might be the one reported).
-        - In the case of the team accepting the risk and wanting to close the ticket, please add the label "*risk_accepted*" to the ticket before closing it.
-        - If server has been decommissioned, please add the label "*server_decommission*" to the ticket before closing it.
-        - If when checking the vulnerability it looks like a false positive, _+please elaborate in a comment+_ and add the label "*false_positive*" before closing it; we will review it and report it to the vendor.
+
+        self.jira_still_vulnerable_comment = '''This ticket has been reopened due to the vulnerability not having been \
+        fixed (if multiple assets are affected, all need to be fixed; if the server is down, lastest known \
+        vulnerability might be the one reported).
+        - In the case of the team accepting the risk and wanting to close the ticket, please add the label \
+        "*risk_accepted*" to the ticket before closing it.
+        - If server has been decommissioned, please add the label "*server_decommission*" to the ticket before closing \
+        it.
+        - If when checking the vulnerability it looks like a false positive, _+please elaborate in a comment+_ and add \
+         the label "*false_positive*" before closing it; we will review it and report it to the vendor.
         
         If you have further doubts, please contact the Security Team.'''
 
-    def create_ticket(self, title, desc, project="IS", components=[], tags=[], attachment_contents = []):
+    def create_ticket(self, title, desc, project="IS", components=[], tags=[], attachment_contents=[]):
         labels = ['vulnerability_management']
         for tag in tags:
             labels.append(str(tag))
@@ -62,53 +70,56 @@ class JiraAPI(object):
             for c in project_obj.components:
                 if component == c.name:
                     self.logger.debug("resolved component name {} to id {}".format(c.name, c.id))
-                    components_ticket.append({ "id": c.id })
-                    exists=True
+                    components_ticket.append({"id": c.id})
+                    exists = True
             if not exists:
                 self.logger.error("Error creating Ticket: component {} not found".format(component))
                 return 0
-                    
+
         new_issue = self.jira.create_issue(project=project,
                                            summary=title,
                                            description=desc,
                                            issuetype={'name': 'Bug'},
                                            labels=labels,
                                            components=components_ticket)
-        
+
         self.logger.info("Ticket {} created successfully".format(new_issue))
-        
+
         if attachment_contents:
             self.add_content_as_attachment(new_issue, attachment_contents)
-        
+
         return new_issue
-    
-    #Basic JIRA Metrics
+
+    # Basic JIRA Metrics
     def metrics_open_tickets(self, project=None):
-        jql = "labels= vulnerability_management and resolution = Unresolved" 
+        jql = "labels= vulnerability_management and resolution = Unresolved"
         if project:
             jql += " and (project='{}')".format(project)
-        self.logger.debug('Executing: {}'.format(jql)) 
+        self.logger.debug('Executing: {}'.format(jql))
         return len(self.jira.search_issues(jql, maxResults=0))
 
     def metrics_closed_tickets(self, project=None):
-        jql = "labels= vulnerability_management and NOT resolution = Unresolved AND created >=startOfMonth(-{})".format(self.max_time_tracking) 
+        jql = "labels= vulnerability_management and NOT resolution = Unresolved AND created >=startOfMonth(-{})".format(
+            self.max_time_tracking)
         if project:
             jql += " and (project='{}')".format(project)
         return len(self.jira.search_issues(jql, maxResults=0))
 
     def sync(self, vulnerabilities, project, components=[]):
-        #JIRA structure of each vulnerability: [source, scan_name, title, diagnosis, consequence, solution, ips, risk, references]
+        # JIRA structure of each vulnerability: [source, scan_name, title, diagnosis, consequence, solution,
+        # ips, risk, references]
         self.logger.info("JIRA Sync started")
 
         for vuln in vulnerabilities:
             # JIRA doesn't allow labels with spaces, so making sure that the scan_name doesn't have spaces
             # if it has, they will be replaced by "_"
-            if " " in  vuln['scan_name']:
+            if " " in vuln['scan_name']:
                 vuln['scan_name'] = "_".join(vuln['scan_name'].split(" "))
-            
-            # we exclude from the vulnerabilities to report those assets that already exist with *risk_accepted*/*server_decommission*
+
+            # we exclude from the vulnerabilities to report those assets that already exist
+            # with *risk_accepted*/*server_decommission*
             vuln = self.exclude_accepted_assets(vuln)
-            
+
             # make sure after exclusion of risk_accepted assets there are still assets
             if vuln['ips']:
                 exists = False
@@ -131,56 +142,65 @@ class JiraAPI(object):
                 # create local text file with assets, attach it to ticket
                 if len(vuln['ips']) > self.max_ips_ticket:
                     attachment_contents = vuln['ips']
-                    vuln['ips'] = ["Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(assets = len(attachment_contents))]
+                    vuln['ips'] = [
+                        "Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(
+                            assets=len(attachment_contents))]
                 try:
                     tpl = template(self.template_path, vuln)
                 except Exception as e:
                     self.logger.error('Exception templating: {}'.format(str(e)))
                     return 0
-                self.create_ticket(title=vuln['title'], desc=tpl, project=project, components=components, tags=[vuln['source'], vuln['scan_name'], 'vulnerability', vuln['risk']], attachment_contents = attachment_contents)
+                self.create_ticket(title=vuln['title'], desc=tpl, project=project, components=components,
+                                   tags=[vuln['source'], vuln['scan_name'], 'vulnerability', vuln['risk']],
+                                   attachment_contents=attachment_contents)
             else:
                 self.logger.info("Ignoring vulnerability as all assets are already reported in a risk_accepted ticket")
-        
+
         self.close_fixed_tickets(vulnerabilities)
         # we reinitialize so the next sync redoes the query with their specific variables
         self.all_tickets = []
         self.excluded_tickets = []
         return True
-    
+
     def exclude_accepted_assets(self, vuln):
         # we want to check JIRA tickets with risk_accepted/server_decommission or false_positive labels sharing the same source
         # will exclude tickets older than 12 months, old tickets will get closed for higiene and recreated if still vulnerable
-        labels = [vuln['source'], vuln['scan_name'], 'vulnerability_management', 'vulnerability'] 
-        
+        labels = [vuln['source'], vuln['scan_name'], 'vulnerability_management', 'vulnerability']
+
         if not self.excluded_tickets:
-            jql = "{} AND labels in (risk_accepted,server_decommission, false_positive) AND NOT labels=advisory AND created >=startOfMonth(-{})".format(" AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
+            jql = "{} AND labels in (risk_accepted,server_decommission, false_positive) AND NOT labels=advisory AND created >=startOfMonth(-{})".format(
+                " AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
             self.excluded_tickets = self.jira.search_issues(jql, maxResults=0)
 
         title = vuln['title']
-        #WARNING: function IGNORES DUPLICATES, after finding a "duplicate" will just return it exists
-        #it wont iterate over the rest of tickets looking for other possible duplicates/similar issues
+        # WARNING: function IGNORES DUPLICATES, after finding a "duplicate" will just return it exists
+        # it wont iterate over the rest of tickets looking for other possible duplicates/similar issues
         self.logger.info("Comparing vulnerability to risk_accepted tickets")
         assets_to_exclude = []
         tickets_excluded_assets = []
         for index in range(len(self.excluded_tickets)):
-            checking_ticketid, checking_title, checking_assets = self.ticket_get_unique_fields(self.excluded_tickets[index])
+            checking_ticketid, checking_title, checking_assets = self.ticket_get_unique_fields(
+                self.excluded_tickets[index])
             if title.encode('ascii') == checking_title.encode('ascii'):
                 if checking_assets:
-                    #checking_assets is a list, we add to our full list for later delete all assets
-                    assets_to_exclude+=checking_assets
+                    # checking_assets is a list, we add to our full list for later delete all assets
+                    assets_to_exclude += checking_assets
                     tickets_excluded_assets.append(checking_ticketid)
-       
+
         if assets_to_exclude:
             assets_to_remove = []
-            self.logger.warn("Vulnerable Assets seen on an already existing risk_accepted Jira ticket: {}".format(', '.join(tickets_excluded_assets)))
+            self.logger.warn("Vulnerable Assets seen on an already existing risk_accepted Jira ticket: {}".format(
+                ', '.join(tickets_excluded_assets)))
             self.logger.debug("Original assets: {}".format(vuln['ips']))
-            #assets in vulnerability have the structure "ip - hostname - port", so we need to match by partial 
+            # assets in vulnerability have the structure "ip - hostname - port", so we need to match by partial
             for exclusion in assets_to_exclude:
                 # for efficiency, we walk the backwards the array of ips from the scanners, as we will be popping out the matches 
                 # and we don't want it to affect the rest of the processing (otherwise, it would miss the asset right after the removed one)
                 for index in range(len(vuln['ips']))[::-1]:
                     if exclusion == vuln['ips'][index].split(" - ")[0]:
-                        self.logger.debug("Deleting asset {} from vulnerability {}, seen in risk_accepted.".format(vuln['ips'][index], title))
+                        self.logger.debug(
+                            "Deleting asset {} from vulnerability {}, seen in risk_accepted.".format(vuln['ips'][index],
+                                                                                                     title))
                         vuln['ips'].pop(index)
             self.logger.debug("Modified assets: {}".format(vuln['ips']))
 
@@ -192,35 +212,37 @@ class JiraAPI(object):
         Returns [exists (bool), is equal (bool), ticketid (str), assets (array)]
         '''
         # we need to return if the vulnerability has already been reported and the ID of the ticket for further processing
-        #function returns array [duplicated(bool), update(bool), ticketid, ticket_assets]
+        # function returns array [duplicated(bool), update(bool), ticketid, ticket_assets]
         title = vuln['title']
-        labels = [vuln['source'], vuln['scan_name'], 'vulnerability_management', 'vulnerability'] 
-        #list(set()) to remove duplicates
+        labels = [vuln['source'], vuln['scan_name'], 'vulnerability_management', 'vulnerability']
+        # list(set()) to remove duplicates
         assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", ",".join(vuln['ips']))))
-        
+
         if not self.all_tickets:
             self.logger.info("Retrieving all JIRA tickets with the following tags {}".format(labels))
             # we want to check all JIRA tickets, to include tickets moved to other queues
             # will exclude tickets older than 12 months, old tickets will get closed for higiene and recreated if still vulnerable
-            jql = "{} AND NOT labels=advisory AND created >=startOfMonth(-{})".format(" AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
-            
+            jql = "{} AND NOT labels=advisory AND created >=startOfMonth(-{})".format(
+                " AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
+
             self.all_tickets = self.jira.search_issues(jql, maxResults=0)
-        
-        #WARNING: function IGNORES DUPLICATES, after finding a "duplicate" will just return it exists
-        #it wont iterate over the rest of tickets looking for other possible duplicates/similar issues
+
+        # WARNING: function IGNORES DUPLICATES, after finding a "duplicate" will just return it exists
+        # it wont iterate over the rest of tickets looking for other possible duplicates/similar issues
         self.logger.info("Comparing Vulnerabilities to created tickets")
         for index in range(len(self.all_tickets)):
             checking_ticketid, checking_title, checking_assets = self.ticket_get_unique_fields(self.all_tickets[index])
             # added "not risk_accepted", as if it is risk_accepted, we will create a new ticket excluding the accepted assets
-            if title.encode('ascii') == checking_title.encode('ascii') and not self.is_risk_accepted(self.jira.issue(checking_ticketid)): 
+            if title.encode('ascii') == checking_title.encode('ascii') and not self.is_risk_accepted(
+                    self.jira.issue(checking_ticketid)):
                 difference = list(set(assets).symmetric_difference(checking_assets))
-                #to check intersection - set(assets) & set(checking_assets)
-                if difference: 
+                # to check intersection - set(assets) & set(checking_assets)
+                if difference:
                     self.logger.info("Asset mismatch, ticket to update. Ticket ID: {}".format(checking_ticketid))
-                    return False, True, checking_ticketid, checking_assets #this will automatically validate
+                    return False, True, checking_ticketid, checking_assets  # this will automatically validate
                 else:
                     self.logger.info("Confirmed duplicated. TickedID: {}".format(checking_ticketid))
-                    return True, False, checking_ticketid, [] #this will automatically validate
+                    return True, False, checking_ticketid, []  # this will automatically validate
         return False, False, "", []
 
     def ticket_get_unique_fields(self, ticket):
@@ -229,19 +251,22 @@ class JiraAPI(object):
 
         assets = self.get_assets_from_description(ticket)
         if not assets:
-            #check if attachment, if so, get assets from attachment
+            # check if attachment, if so, get assets from attachment
             assets = self.get_assets_from_attachment(ticket)
-                
+
         return ticketid, title, assets
 
-    def get_assets_from_description(self, ticket, _raw = False):
+    def get_assets_from_description(self, ticket, _raw=False):
         # Get the assets as a string "host - protocol/port - hostname" separated by "\n"
         # structure the text to have the same structure as the assets from the attachment
         affected_assets = ""
         try:
-            affected_assets = ticket.raw.get('fields', {}).get('description').encode("ascii").split("{panel:title=Affected Assets}")[1].split("{panel}")[0].replace('\n','').replace(' * ','\n').replace('\n', '', 1)
+            affected_assets = \
+            ticket.raw.get('fields', {}).get('description').encode("ascii").split("{panel:title=Affected Assets}")[
+                1].split("{panel}")[0].replace('\n', '').replace(' * ', '\n').replace('\n', '', 1)
         except Exception as e:
-            self.logger.error("Unable to process the Ticket's 'Affected Assets'. Ticket ID: {}. Reason: {}".format(ticket, e))
+            self.logger.error(
+                "Unable to process the Ticket's 'Affected Assets'. Ticket ID: {}. Reason: {}".format(ticket, e))
 
         if affected_assets:
             if _raw:
@@ -257,14 +282,14 @@ class JiraAPI(object):
                 self.logger.error("Ticket IPs regex failed. Ticket ID: {}. Reason: {}".format(ticket, e))
         return False
 
-    def get_assets_from_attachment(self, ticket, _raw = False):
+    def get_assets_from_attachment(self, ticket, _raw=False):
         # Get the assets as a string "host - protocol/port - hostname" separated by "\n"
         affected_assets = []
         try:
             fields = self.jira.issue(ticket.key).raw.get('fields', {})
             attachments = fields.get('attachment', {})
             affected_assets = ""
-            #we will make sure we get the latest version of the file
+            # we will make sure we get the latest version of the file
             latest = ''
             attachment_id = ''
             if attachments:
@@ -272,15 +297,16 @@ class JiraAPI(object):
                     if item.get('filename') == self.attachment_filename:
                         if not latest:
                             latest = item.get('created')
-                            attachment_id = item.get('id') 
+                            attachment_id = item.get('id')
                         else:
                             if latest < item.get('created'):
-                                latest = item.get('created')         
-                                attachment_id = item.get('id') 
+                                latest = item.get('created')
+                                attachment_id = item.get('id')
             affected_assets = self.jira.attachment(attachment_id).get()
 
         except Exception as e:
-            self.logger.error("Failed to get assets from ticket attachment. Ticket ID: {}. Reason: {}".format(ticket, e))
+            self.logger.error(
+                "Failed to get assets from ticket attachment. Ticket ID: {}. Reason: {}".format(ticket, e))
 
         if affected_assets:
             if _raw:
@@ -326,15 +352,15 @@ class JiraAPI(object):
 
     def add_content_as_attachment(self, issue, contents):
         try:
-            #Create the file locally with the data
+            # Create the file locally with the data
             attachment_file = open(self.attachment_filename, "w")
             attachment_file.write("\n".join(contents))
             attachment_file.close()
-            #Push the created file to the ticket
+            # Push the created file to the ticket
             attachment_file = open(self.attachment_filename, "rb")
             self.jira.add_attachment(issue, attachment_file, self.attachment_filename)
             attachment_file.close()
-            #remove the temp file
+            # remove the temp file
             os.remove(self.attachment_filename)
             self.logger.info("Added attachment successfully.")
         except:
@@ -344,21 +370,23 @@ class JiraAPI(object):
         return True
 
     def get_ticket_reported_assets(self, ticket):
-        #[METRICS] return a list with all the affected assets for that vulnerability (including already resolved ones) 
-        return list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b",str(self.jira.issue(ticket).raw))))
+        # [METRICS] return a list with all the affected assets for that vulnerability (including already resolved ones)
+        return list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", str(self.jira.issue(ticket).raw))))
 
     def get_resolution_time(self, ticket):
-        #get time a ticket took to be resolved
+        # get time a ticket took to be resolved
         ticket_obj = self.jira.issue(ticket)
         if self.is_ticket_resolved(ticket_obj):
             ticket_data = ticket_obj.raw.get('fields')
-            #dates follow format '2018-11-06T10:36:13.849+0100'
-            created = [int(x) for x in ticket_data['created'].split('.')[0].replace('T', '-').replace(':','-').split('-')]
-            resolved =[int(x) for x in ticket_data['resolutiondate'].split('.')[0].replace('T', '-').replace(':','-').split('-')]
-            
-            start = datetime(created[0],created[1],created[2],created[3],created[4],created[5])
-            end = datetime(resolved[0],resolved[1],resolved[2],resolved[3],resolved[4],resolved[5])
-            return (end-start).days
+            # dates follow format '2018-11-06T10:36:13.849+0100'
+            created = [int(x) for x in
+                       ticket_data['created'].split('.')[0].replace('T', '-').replace(':', '-').split('-')]
+            resolved = [int(x) for x in
+                        ticket_data['resolutiondate'].split('.')[0].replace('T', '-').replace(':', '-').split('-')]
+
+            start = datetime(created[0], created[1], created[2], created[3], created[4], created[5])
+            end = datetime(resolved[0], resolved[1], resolved[2], resolved[3], resolved[4], resolved[5])
+            return (end - start).days
         else:
             self.logger.error("Ticket {ticket} is not resolved, can't calculate resolution time".format(ticket=ticket))
 
@@ -367,28 +395,28 @@ class JiraAPI(object):
     def ticket_update_assets(self, vuln, ticketid, ticket_assets):
         # correct description will always be in the vulnerability to report, only needed to update description to new one
         self.logger.info("Ticket {} exists, UPDATE requested".format(ticketid))
-        
-        #for now, if a vulnerability has been accepted ('accepted_risk'), ticket is completely ignored and not updated (no new assets)
 
-        #TODO when vulnerability accepted, create a new ticket with only the non-accepted vulnerable assets
-        #this would require go through the downloaded tickets, check duplicates/accepted ones, and if so,
-        #check on their assets to exclude them from the new ticket
+        # for now, if a vulnerability has been accepted ('accepted_risk'), ticket is completely ignored and not updated (no new assets)
+
+        # TODO when vulnerability accepted, create a new ticket with only the non-accepted vulnerable assets
+        # this would require go through the downloaded tickets, check duplicates/accepted ones, and if so,
+        # check on their assets to exclude them from the new ticket
         risk_accepted = False
         ticket_obj = self.jira.issue(ticketid)
         if self.is_ticket_resolved(ticket_obj):
             if self.is_risk_accepted(ticket_obj):
                 return 0
             self.reopen_ticket(ticketid=ticketid, comment=self.jira_still_vulnerable_comment)
-        
-        #First will do the comparison of assets
+
+        # First will do the comparison of assets
         ticket_obj.update()
         assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", ",".join(vuln['ips']))))
         difference = list(set(assets).symmetric_difference(ticket_assets))
-        
+
         comment = ''
         added = ''
         removed = ''
-        #put a comment with the assets that have been added/removed
+        # put a comment with the assets that have been added/removed
         for asset in difference:
             if asset in assets:
                 if not added:
@@ -396,66 +424,71 @@ class JiraAPI(object):
                 added += '* {}\n'.format(asset)
             elif asset in ticket_assets:
                 if not removed:
-                    removed= '\nThe following assets *have been resolved*:\n'
+                    removed = '\nThe following assets *have been resolved*:\n'
                 removed += '* {}\n'.format(asset)
 
         comment = added + removed
-        
-        #then will check if assets are too many that need to be added as an attachment
+
+        # then will check if assets are too many that need to be added as an attachment
         attachment_contents = []
         if len(vuln['ips']) > self.max_ips_ticket:
             attachment_contents = vuln['ips']
-            vuln['ips'] = ["Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(assets = len(attachment_contents))]
-        
-        #fill the ticket description template
+            vuln['ips'] = [
+                "Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(
+                    assets=len(attachment_contents))]
+
+        # fill the ticket description template
         try:
             tpl = template(self.template_path, vuln)
         except Exception as e:
             self.logger.error('Exception updating assets: {}'.format(str(e)))
             return 0
 
-        #proceed checking if it requires adding as an attachment
+        # proceed checking if it requires adding as an attachment
         try:
-            #update attachment with hosts and delete the old versions
+            # update attachment with hosts and delete the old versions
             if attachment_contents:
                 self.clean_old_attachments(ticket_obj)
                 self.add_content_as_attachment(ticket_obj, attachment_contents)
-                
-            ticket_obj.update(description=tpl, comment=comment, fields={"labels":ticket_obj.fields.labels})
+
+            ticket_obj.update(description=tpl, comment=comment, fields={"labels": ticket_obj.fields.labels})
             self.logger.info("Ticket {} updated successfully".format(ticketid))
             self.add_label(ticketid, 'updated')
         except Exception as e:
-            self.logger.error("Error while trying up update ticket {ticketid}.\nReason: {e}".format(ticketid = ticketid, e=e))
+            self.logger.error(
+                "Error while trying up update ticket {ticketid}.\nReason: {e}".format(ticketid=ticketid, e=e))
         return 0
 
     def add_label(self, ticketid, label):
         ticket_obj = self.jira.issue(ticketid)
-        
+
         if label not in [x.encode('utf8') for x in ticket_obj.fields.labels]:
             ticket_obj.fields.labels.append(label)
-        
+
             try:
-                ticket_obj.update(fields={"labels":ticket_obj.fields.labels})
+                ticket_obj.update(fields={"labels": ticket_obj.fields.labels})
                 self.logger.info("Added label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
-            except:
-                self.logger.error("Error while trying to add label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
-        
+            except Exception as e:
+                self.logger.error(
+                    "Error while trying to add label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
+
         return 0
 
     def remove_label(self, ticketid, label):
         ticket_obj = self.jira.issue(ticketid)
-        
+
         if label in [x.encode('utf8') for x in ticket_obj.fields.labels]:
             ticket_obj.fields.labels.remove(label)
-        
+
             try:
-                ticket_obj.update(fields={"labels":ticket_obj.fields.labels})
+                ticket_obj.update(fields={"labels": ticket_obj.fields.labels})
                 self.logger.info("Removed label {label} from ticket {ticket}".format(label=label, ticket=ticketid))
-            except:
-                self.logger.error("Error while trying to remove label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
+            except Exception as e:
+                self.logger.error("Error while trying to remove label {label} to ticket {ticket}".format(label=label,
+                                                                                                         ticket=ticketid))
         else:
             self.logger.error("Error: label {label} not in ticket {ticket}".format(label=label, ticket=ticketid))
-        
+
         return 0
 
     def close_fixed_tickets(self, vulnerabilities):
@@ -475,10 +508,9 @@ class JiraAPI(object):
                 self.logger.info("Ticket {} is still vulnerable".format(ticket))
                 continue
             self.logger.info("Ticket {} is no longer vulnerable".format(ticket))
-            self.close_ticket(ticket, self.JIRA_RESOLUTION_FIXED, comment) 
+            self.close_ticket(ticket, self.JIRA_RESOLUTION_FIXED, comment)
         return 0
 
-
     def is_ticket_reopenable(self, ticket_obj):
         transitions = self.jira.transitions(ticket_obj)
         for transition in transitions:
@@ -497,7 +529,7 @@ class JiraAPI(object):
         return False
 
     def is_ticket_resolved(self, ticket_obj):
-        #Checks if a ticket is resolved or not
+        # Checks if a ticket is resolved or not
         if ticket_obj is not None:
             if ticket_obj.raw['fields'].get('resolution') is not None:
                 if ticket_obj.raw['fields'].get('resolution').get('name') != 'Unresolved':
@@ -507,7 +539,6 @@ class JiraAPI(object):
         self.logger.debug("Checked ticket {} is already open".format(ticket_obj))
         return False
 
-
     def is_risk_accepted(self, ticket_obj):
         if ticket_obj is not None:
             if ticket_obj.raw['fields'].get('labels') is not None:
@@ -528,12 +559,13 @@ class JiraAPI(object):
         self.logger.debug("Ticket {} exists, REOPEN requested".format(ticketid))
         # this will reopen a ticket by ticketid
         ticket_obj = self.jira.issue(ticketid)
-        
+
         if self.is_ticket_resolved(ticket_obj):
             if (not self.is_risk_accepted(ticket_obj) or ignore_labels):
                 try:
                     if self.is_ticket_reopenable(ticket_obj):
-                        error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_REOPEN_ISSUE, comment = comment)
+                        error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_REOPEN_ISSUE,
+                                                           comment=comment)
                         self.logger.info("Ticket {} reopened successfully".format(ticketid))
                         if not ignore_labels:
                             self.add_label(ticketid, 'reopened')
@@ -551,30 +583,32 @@ class JiraAPI(object):
         if not self.is_ticket_resolved(ticket_obj):
             try:
                 if self.is_ticket_closeable(ticket_obj):
-                    #need to add the label before closing the ticket
+                    # need to add the label before closing the ticket
                     self.add_label(ticketid, 'closed')
-                    error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_CLOSE_ISSUE, comment = comment, resolution = {"name": resolution })
+                    error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_CLOSE_ISSUE,
+                                                       comment=comment, resolution={"name": resolution})
                     self.logger.info("Ticket {} closed successfully".format(ticketid))
                     return 1
             except Exception as e:
                 # continue with ticket data so that a new ticket is created in place of the "lost" one
                 self.logger.error("error closing ticket {}: {}".format(ticketid, e))
                 return 0
-                
+
         return 0
 
     def close_obsolete_tickets(self):
         # Close tickets older than 12 months, vulnerabilities not solved will get created a new ticket 
         self.logger.info("Closing obsolete tickets older than {} months".format(self.max_time_tracking))
-        jql = "labels=vulnerability_management AND NOT labels=advisory AND created <startOfMonth(-{}) and resolution=Unresolved".format(self.max_time_tracking)
+        jql = "labels=vulnerability_management AND NOT labels=advisory AND created <startOfMonth(-{}) and resolution=Unresolved".format(
+            self.max_time_tracking)
         tickets_to_close = self.jira.search_issues(jql, maxResults=0)
-        
+
         comment = '''This ticket is being closed for hygiene, as it is more than {} months old.
         If the vulnerability still exists, a new ticket will be opened.'''.format(self.max_time_tracking)
-        
+
         for ticket in tickets_to_close:
-                self.close_ticket(ticket, self.JIRA_RESOLUTION_OBSOLETE, comment)
-        
+            self.close_ticket(ticket, self.JIRA_RESOLUTION_OBSOLETE, comment)
+
         return 0
 
     def project_exists(self, project):
@@ -589,18 +623,19 @@ class JiraAPI(object):
         '''
         saves all tickets locally, local snapshot of vulnerability_management ticktes
         '''
-        #check if file already exists
+        # check if file already exists
         check_date = str(date.today())
-        fname = '{}jira_{}.json'.format(path, check_date) 
+        fname = '{}jira_{}.json'.format(path, check_date)
         if os.path.isfile(fname):
             self.logger.info("File {} already exists, skipping ticket download".format(fname))
             return True
         try:
             self.logger.info("Saving locally tickets from the last {} months".format(self.max_time_tracking))
-            jql = "labels=vulnerability_management AND NOT labels=advisory AND created >=startOfMonth(-{})".format(self.max_time_tracking)
+            jql = "labels=vulnerability_management AND NOT labels=advisory AND created >=startOfMonth(-{})".format(
+                self.max_time_tracking)
             tickets_data = self.jira.search_issues(jql, maxResults=0)
 
-            #TODO process tickets, creating a new field called "_metadata" with all the affected assets well structured
+            # TODO process tickets, creating a new field called "_metadata" with all the affected assets well structured
             # for future processing in ELK/Splunk; this includes downloading attachments with assets and processing them
 
             processed_tickets = []
@@ -621,24 +656,23 @@ class JiraAPI(object):
                         assets_json = self.parse_asset_to_json(assets)
                         _metadata["affected_hosts"].append(assets_json)
 
-
                 temp_ticket = ticket.raw.get('fields')
                 temp_ticket['_metadata'] = _metadata
 
                 processed_tickets.append(temp_ticket)
-            
-            #end of line needed, as writelines() doesn't add it automatically, otherwise one big line
-            to_save = [json.dumps(ticket.raw.get('fields'))+"\n" for ticket in tickets_data]
+
+            # end of line needed, as writelines() doesn't add it automatically, otherwise one big line
+            to_save = [json.dumps(ticket.raw.get('fields')) + "\n" for ticket in tickets_data]
             with open(fname, 'w') as outfile:
                 outfile.writelines(to_save)
                 self.logger.info("Tickets saved succesfully.")
-            
+
             return True
 
         except Exception as e:
             self.logger.error("Tickets could not be saved locally: {}.".format(e))
 
-        return False 
+        return False
 
     def decommission_cleanup(self):
         '''
@@ -646,19 +680,22 @@ class JiraAPI(object):
         closed already for more than x months (default is 3 months) in order to clean solved issues
         for statistics purposes
         '''
-        self.logger.info("Deleting 'server_decommission' tag from tickets closed more than {} months ago".format(self.max_decommission_time))
+        self.logger.info("Deleting 'server_decommission' tag from tickets closed more than {} months ago".format(
+            self.max_decommission_time))
 
-        jql = "labels=vulnerability_management AND labels=server_decommission and resolutiondate <=startOfMonth(-{})".format(self.max_decommission_time)
+        jql = "labels=vulnerability_management AND labels=server_decommission and resolutiondate <=startOfMonth(-{})".format(
+            self.max_decommission_time)
         decommissioned_tickets = self.jira.search_issues(jql, maxResults=0)
-        
+
         comment = '''This ticket is having deleted the *server_decommission* tag, as it is more than {} months old and is expected to already have been decommissioned.
-        If that is not the case and the vulnerability still exists, the vulnerability will be opened again.'''.format(self.max_decommission_time)
-        
+        If that is not the case and the vulnerability still exists, the vulnerability will be opened again.'''.format(
+            self.max_decommission_time)
+
         for ticket in decommissioned_tickets:
-            #we open first the ticket, as we want to make sure the process is not blocked due to 
-            #an unexisting jira workflow or unallowed edit from closed tickets
+            # we open first the ticket, as we want to make sure the process is not blocked due to
+            # an unexisting jira workflow or unallowed edit from closed tickets
             self.reopen_ticket(ticketid=ticket, ignore_labels=True)
             self.remove_label(ticket, 'server_decommission')
             self.close_ticket(ticket, self.JIRA_RESOLUTION_FIXED, comment)
-        
+
         return 0

vulnwhisp/test/mock.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import os
 import logging
 import httpretty
@@ -20,10 +21,12 @@ class mockAPI(object):
 
     def get_directories(self, path):
         dir, subdirs, files = next(os.walk(path))
+        self.logger.debug('Subdirectories found: {}'.format(subdirs))
         return subdirs
 
     def get_files(self, path):
         dir, subdirs, files = next(os.walk(path))
+        self.logger.debug('Files found: {}'.format(files))
         return files
 
     def qualys_vuln_callback(self, request, uri, response_headers):