Merge pull request #230 from nfalke-/2to3

2to3

bin/vuln_whisperer

@@ -93,7 +93,7 @@ def main():
                                        scanname=args.scanname)
                     exit_code += vw.whisper_vulnerabilities()
                 except Exception as e:
-                    logger.error("VulnWhisperer was unable to perform the processing on '{}'".format(args.source))
+                    logger.error("VulnWhisperer was unable to perform the processing on '{}'".format(section))
         else:
             logger.info('Running vulnwhisperer for section {}'.format(args.section))
             vw = vulnWhisperer(config=args.config,

configs/test.ini

@@ -6,8 +6,8 @@ access_key=
 secret_key=
 username=nessus_username
 password=nessus_password
-write_path=/opt/VulnWhisperer/data/nessus/
+write_path=/tmp/VulnWhisperer/data/nessus/
-db_path=/opt/VulnWhisperer/data/database
+db_path=/tmp/VulnWhisperer/data/database
 trash=false
 verbose=true
 
@@ -19,8 +19,8 @@ access_key=
 secret_key=
 username=tenable.io_username
 password=tenable.io_password
-write_path=/opt/VulnWhisperer/data/tenable/
+write_path=/tmp/VulnWhisperer/data/tenable/
-db_path=/opt/VulnWhisperer/data/database
+db_path=/tmp/VulnWhisperer/data/database
 trash=false
 verbose=true
 
@@ -30,8 +30,8 @@ enabled = false
 hostname = qualys_web
 username = exampleuser
 password = examplepass
-write_path=/opt/VulnWhisperer/data/qualys_web/
+write_path=/tmp/VulnWhisperer/data/qualys_web/
-db_path=/opt/VulnWhisperer/data/database
+db_path=/tmp/VulnWhisperer/data/database
 verbose=true
 
 # Set the maximum number of retries each connection should attempt.
@@ -46,8 +46,8 @@ enabled = true
 hostname = qualys_vuln
 username = exampleuser
 password = examplepass
-write_path=/opt/VulnWhisperer/data/qualys_vuln/
+write_path=/tmp/VulnWhisperer/data/qualys_vuln/
-db_path=/opt/VulnWhisperer/data/database
+db_path=/tmp/VulnWhisperer/data/database
 verbose=true
 
 [detectify]
@@ -58,8 +58,8 @@ hostname = detectify
 username = exampleuser
 #password variable used as secretKey
 password = examplepass
-write_path =/opt/VulnWhisperer/data/detectify/
+write_path =/tmp/VulnWhisperer/data/detectify/
-db_path = /opt/VulnWhisperer/data/database
+db_path = /tmp/VulnWhisperer/data/database
 verbose = true
 
 [openvas]
@@ -68,8 +68,8 @@ hostname = openvas
 port = 4000
 username = exampleuser
 password = examplepass
-write_path=/opt/VulnWhisperer/data/openvas/
+write_path=/tmp/VulnWhisperer/data/openvas/
-db_path=/opt/VulnWhisperer/data/database
+db_path=/tmp/VulnWhisperer/data/database
 verbose=true
 
 [jira]
@@ -77,8 +77,8 @@ enabled = false
 hostname = jira-host
 username = username
 password = password
-write_path = /opt/VulnWhisperer/data/jira/
+write_path = /tmp/VulnWhisperer/data/jira/
-db_path = /opt/VulnWhisperer/data/database
+db_path = /tmp/VulnWhisperer/data/database
 verbose = true
 dns_resolv = False
 

setup.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 
+from __future__ import absolute_import
 from setuptools import setup, find_packages
 
 setup(

vulnwhisp/base/config.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import sys
 import logging
 
@@ -5,7 +6,7 @@ import logging
 if sys.version_info > (3, 0):
     import configparser as cp
 else:
-    import ConfigParser as cp
+    import six.moves.configparser as cp
 
 
 class vwConfig(object):

vulnwhisp/frameworks/nessus.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import json
 import logging
 import sys

vulnwhisp/frameworks/openvas.py

@@ -1,5 +1,6 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import
 __author__ = 'Austin Taylor'
 
 import datetime as dt

vulnwhisp/frameworks/qualys_vuln.py

@@ -1,5 +1,6 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import
 __author__ = 'Nathan Young'
 
 import logging
@@ -18,9 +19,9 @@ class qualysWhisperAPI(object):
         self.logger = logging.getLogger('qualysWhisperAPI')
         self.config = config
         try:
-            self.qgc = qualysapi.connect(config, 'qualys_vuln')
+            self.qgc = qualysapi.connect(config_file=config, section='qualys_vuln')
             # Fail early if we can't make a request or auth is incorrect
-            self.qgc.request('about.php')
+            # self.qgc.request('about.php')
             self.logger.info('Connected to Qualys at {}'.format(self.qgc.server))
         except Exception as e:
             self.logger.error('Could not connect to Qualys: {}'.format(str(e)))

vulnwhisp/frameworks/qualys_web.py

@@ -1,5 +1,8 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import
+from six.moves import range
+from functools import reduce
 __author__ = 'Austin Taylor'
 
 from lxml import objectify
@@ -14,24 +17,16 @@ import os
 import csv
 import logging
 import dateutil.parser as dp
+csv.field_size_limit(sys.maxsize)
 
 
 class qualysWhisperAPI(object):
-    COUNT_WEBAPP = '/count/was/webapp'
     COUNT_WASSCAN = '/count/was/wasscan'
     DELETE_REPORT = '/delete/was/report/{report_id}'
-    GET_WEBAPP_DETAILS = '/get/was/webapp/{was_id}'
-    QPS_REST_3 = '/qps/rest/3.0'
-    REPORT_DETAILS = '/get/was/report/{report_id}'
     REPORT_STATUS = '/status/was/report/{report_id}'
     REPORT_CREATE = '/create/was/report'
     REPORT_DOWNLOAD = '/download/was/report/{report_id}'
-    SCAN_DETAILS = '/get/was/wasscan/{scan_id}'
-    SCAN_DOWNLOAD = '/download/was/wasscan/{scan_id}'
-    SEARCH_REPORTS = '/search/was/report'
-    SEARCH_WEB_APPS = '/search/was/webapp'
     SEARCH_WAS_SCAN = '/search/was/wasscan'
-    VERSION = '/qps/rest/portal/version'
 
     def __init__(self, config=None):
         self.logger = logging.getLogger('qualysWhisperAPI')
@@ -41,10 +36,6 @@ class qualysWhisperAPI(object):
             self.logger.info('Connected to Qualys at {}'.format(self.qgc.server))
         except Exception as e:
             self.logger.error('Could not connect to Qualys: {}'.format(str(e)))
-        self.headers = {
-            #"content-type": "text/xml"}
-            "Accept" : "application/json",
-            "Content-Type": "application/json"}
         self.config_parse = qcconf.QualysConnectConfig(config, 'qualys_web')
         try:
             self.template_id = self.config_parse.get_template_id()
@@ -69,14 +60,8 @@ class qualysWhisperAPI(object):
 
     def generate_scan_result_XML(self, limit=1000, offset=1, status='FINISHED'):
         report_xml = E.ServiceRequest(
-            E.filters(
+            E.filters(E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status)),
-                E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status
+            E.preferences(E.startFromOffset(str(offset)), E.limitResults(str(limit))),
-                           ),
-            ),
-            E.preferences(
-                E.startFromOffset(str(offset)),
-                E.limitResults(str(limit))
-            ),
         )
         return report_xml
 
@@ -115,8 +100,10 @@ class qualysWhisperAPI(object):
                 if i % limit == 0:
                     if (total - i) < limit:
                         qualys_api_limit = total - i
-                    self.logger.info('Making a request with a limit of {} at offset {}'.format((str(qualys_api_limit)), str(i + 1)))
+                    self.logger.info('Making a request with a limit of {} at offset {}'
-                    scan_info = self.get_scan_info(limit=qualys_api_limit, offset=i + 1, status=status)
+                            .format((str(qualys_api_limit)), str(i + 1)))
+                    scan_info = self.get_scan_info(
+                            limit=qualys_api_limit, offset=i + 1, status=status)
                     _records.append(scan_info)
             self.logger.debug('Converting XML to DataFrame')
             dataframes = [self.xml_parser(xml) for xml in _records]
@@ -133,7 +120,8 @@ class qualysWhisperAPI(object):
         return self.qgc.request(self.REPORT_STATUS.format(report_id=report_id))
 
     def download_report(self, report_id):
-        return self.qgc.request(self.REPORT_DOWNLOAD.format(report_id=report_id))
+        return self.qgc.request(
+                self.REPORT_DOWNLOAD.format(report_id=report_id), http_method='get')
 
     def generate_scan_report_XML(self, scan_id):
         """Generates a CSV report for an asset based on template defined in .ini file"""
@@ -145,20 +133,8 @@ class qualysWhisperAPI(object):
                     E.format('CSV'),
                     #type is not needed, as the template already has it
                     E.type('WAS_SCAN_REPORT'),
-                    E.template(
+                    E.template(E.id(self.template_id)),
-                        E.id(self.template_id)
+                    E.config(E.scanReport(E.target(E.scans(E.WasScan(E.id(scan_id))))))
-                    ),
-                    E.config(
-                        E.scanReport(
-                            E.target(
-                                E.scans(
-                                    E.WasScan(
-                                        E.id(scan_id)
-                                    )
-                                ),
-                            ),
-                        ),
-                    )
                 )
             )
         )
@@ -175,95 +151,14 @@ class qualysWhisperAPI(object):
     def delete_report(self, report_id):
         return self.qgc.request(self.DELETE_REPORT.format(report_id=report_id))
 
-
-class qualysReportFields:
-    CATEGORIES = ['VULNERABILITY',
-                  'SENSITIVECONTENT',
-                  'INFORMATION_GATHERED']
-
-    # URL Vulnerability Information
-
-    VULN_BLOCK = [
-        CATEGORIES[0],
-        'ID',
-        'QID',
-        'Url',
-        'Param',
-        'Function',
-        'Form Entry Point',
-        'Access Path',
-        'Authentication',
-        'Ajax Request',
-        'Ajax Request ID',
-        'Ignored',
-        'Ignore Reason',
-        'Ignore Date',
-        'Ignore User',
-        'Ignore Comments',
-        'First Time Detected',
-        'Last Time Detected',
-        'Last Time Tested',
-        'Times Detected',
-        'Payload #1',
-        'Request Method #1',
-        'Request URL #1',
-        'Request Headers #1',
-        'Response #1',
-        'Evidence #1',
-    ]
-
-    INFO_HEADER = [
-        'Vulnerability Category',
-        'ID',
-        'QID',
-        'Response #1',
-        'Last Time Detected',
-    ]
-    INFO_BLOCK = [
-        CATEGORIES[2],
-        'ID',
-        'QID',
-        'Results',
-        'Detection Date',
-    ]
-
-    QID_HEADER = [
-        'QID',
-        'Id',
-        'Title',
-        'Category',
-        'Severity Level',
-        'Groups',
-        'OWASP',
-        'WASC',
-        'CWE',
-        'CVSS Base',
-        'CVSS Temporal',
-        'Description',
-        'Impact',
-        'Solution',
-    ]
-    GROUP_HEADER = ['GROUP', 'Name', 'Category']
-    OWASP_HEADER = ['OWASP', 'Code', 'Name']
-    WASC_HEADER = ['WASC', 'Code', 'Name']
-    SCAN_META = ['Web Application Name', 'URL', 'Owner', 'Scope', 'Operating System']
-    CATEGORY_HEADER = ['Category', 'Severity', 'Level', 'Description']
-
-
 class qualysUtils:
     def __init__(self):
         self.logger = logging.getLogger('qualysUtils')
 
-    def grab_section(
+    def grab_section(self, report, section, end=[], pop_last=False):
-            self,
-            report,
-            section,
-            end=[],
-            pop_last=False,
-    ):
         temp_list = []
         max_col_count = 0
-        with open(report, 'rb') as csvfile:
+        with open(report, 'rt') as csvfile:
             q_report = csv.reader(csvfile, delimiter=',', quotechar='"')
             for line in q_report:
                 if set(line) == set(section):
@@ -289,44 +184,53 @@ class qualysUtils:
         return _data
 
 class qualysScanReport:
-    # URL Vulnerability Information
+    CATEGORIES = ['VULNERABILITY', 'SENSITIVECONTENT', 'INFORMATION_GATHERED']
-    WEB_SCAN_VULN_BLOCK = list(qualysReportFields.VULN_BLOCK)
-    WEB_SCAN_VULN_BLOCK.insert(WEB_SCAN_VULN_BLOCK.index('QID'), 'Detection ID')
 
-    WEB_SCAN_VULN_HEADER = list(WEB_SCAN_VULN_BLOCK)
+    WEB_SCAN_BLOCK = [
-    WEB_SCAN_VULN_HEADER[WEB_SCAN_VULN_BLOCK.index(qualysReportFields.CATEGORIES[0])] = \
+        "ID", "Detection ID", "QID", "Url", "Param/Cookie", "Function",
-        'Vulnerability Category'
+        "Form Entry Point", "Access Path", "Authentication", "Ajax Request",
+        "Ajax Request ID", "Ignored", "Ignore Reason", "Ignore Date", "Ignore User",
+        "Ignore Comments", "Detection Date", "Payload #1", "Request Method #1",
+        "Request URL #1", "Request Headers #1", "Response #1", "Evidence #1",
+        "Unique ID", "Flags", "Protocol", "Virtual Host", "IP", "Port", "Result",
+        "Info#1", "CVSS V3 Base", "CVSS V3 Temporal", "CVSS V3 Attack Vector",
+        "Request Body #1"
+    ]
+    WEB_SCAN_VULN_BLOCK = [CATEGORIES[0]] + WEB_SCAN_BLOCK
+    WEB_SCAN_SENSITIVE_BLOCK = [CATEGORIES[1]] + WEB_SCAN_BLOCK
 
-    WEB_SCAN_SENSITIVE_HEADER = list(WEB_SCAN_VULN_HEADER)
+    WEB_SCAN_HEADER = ["Vulnerability Category"] + WEB_SCAN_BLOCK
-    WEB_SCAN_SENSITIVE_HEADER.insert(WEB_SCAN_SENSITIVE_HEADER.index('Url'
+    WEB_SCAN_HEADER[WEB_SCAN_HEADER.index("Detection Date")] = "Last Time Detected"
-                                                                     ), 'Content')
 
-    WEB_SCAN_SENSITIVE_BLOCK = list(WEB_SCAN_SENSITIVE_HEADER)
-    WEB_SCAN_SENSITIVE_BLOCK.insert(WEB_SCAN_SENSITIVE_BLOCK.index('QID'), 'Detection ID')
-    WEB_SCAN_SENSITIVE_BLOCK[WEB_SCAN_SENSITIVE_BLOCK.index('Vulnerability Category'
-                                                            )] = qualysReportFields.CATEGORIES[1]
 
-    WEB_SCAN_INFO_HEADER = list(qualysReportFields.INFO_HEADER)
+    WEB_SCAN_INFO_BLOCK = [
-    WEB_SCAN_INFO_HEADER.insert(WEB_SCAN_INFO_HEADER.index('QID'), 'Detection ID')
+        "INFORMATION_GATHERED", "ID", "Detection ID", "QID", "Results", "Detection Date",
+        "Unique ID", "Flags", "Protocol", "Virtual Host", "IP", "Port", "Result",
+        "Info#1"
+    ]
 
-    WEB_SCAN_INFO_BLOCK = list(qualysReportFields.INFO_BLOCK)
+    WEB_SCAN_INFO_HEADER = [
-    WEB_SCAN_INFO_BLOCK.insert(WEB_SCAN_INFO_BLOCK.index('QID'), 'Detection ID')
+        "Vulnerability Category", "ID", "Detection ID", "QID", "Results", "Last Time Detected",
+        "Unique ID", "Flags", "Protocol", "Virtual Host", "IP", "Port", "Result",
+        "Info#1"
+    ]
 
-    QID_HEADER = list(qualysReportFields.QID_HEADER)
+    QID_HEADER = [
-    GROUP_HEADER = list(qualysReportFields.GROUP_HEADER)
+        "QID", "Id", "Title", "Category", "Severity Level", "Groups", "OWASP", "WASC",
-    OWASP_HEADER = list(qualysReportFields.OWASP_HEADER)
+        "CWE", "CVSS Base", "CVSS Temporal", "Description", "Impact", "Solution",
-    WASC_HEADER = list(qualysReportFields.WASC_HEADER)
+        "CVSS V3 Base", "CVSS V3 Temporal", "CVSS V3 Attack Vector"
-    SCAN_META = list(qualysReportFields.SCAN_META)
+    ]
-    CATEGORY_HEADER = list(qualysReportFields.CATEGORY_HEADER)
+    GROUP_HEADER = ['GROUP', 'Name', 'Category']
+    OWASP_HEADER = ['OWASP', 'Code', 'Name']
+    WASC_HEADER = ['WASC', 'Code', 'Name']
+    SCAN_META = [
+        "Web Application Name", "URL", "Owner", "Scope", "ID", "Tags",
+        "Custom Attributes"
+    ]
+    CATEGORY_HEADER = ['Category', 'Severity', 'Level', 'Description']
 
-    def __init__(
+    def __init__(self, config=None, file_in=None, 
-            self,
+            file_stream=False, delimiter=',', quotechar='"'):
-            config=None,
-            file_in=None,
-            file_stream=False,
-            delimiter=',',
-            quotechar='"',
-    ):
         self.logger = logging.getLogger('qualysScanReport')
         self.file_in = file_in
         self.file_stream = file_stream
@@ -337,71 +241,79 @@ class qualysScanReport:
             try:
                 self.qw = qualysWhisperAPI(config=config)
             except Exception as e:
-                self.logger.error('Could not load config! Please check settings. Error: {}'.format(str(e)))
+                self.logger.error(
+                        'Could not load config! Please check settings. Error: {}'.format(
+                            str(e)))
 
         if file_stream:
             self.open_file = file_in.splitlines()
         elif file_in:
-
             self.open_file = open(file_in, 'rb')
 
         self.downloaded_file = None
 
     def grab_sections(self, report):
-        all_dataframes = []
+        return {
-        dict_tracker = {}
+            'WEB_SCAN_VULN_BLOCK': pd.DataFrame(
-        with open(report, 'rb') as csvfile:
+                self.utils.grab_section(
-            dict_tracker['WEB_SCAN_VULN_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
+                    report,
                     self.WEB_SCAN_VULN_BLOCK,
-                                                                                       end=[
+                    end=[self.WEB_SCAN_SENSITIVE_BLOCK, self.WEB_SCAN_INFO_BLOCK],
-                                                                                           self.WEB_SCAN_SENSITIVE_BLOCK,
-                                                                                           self.WEB_SCAN_INFO_BLOCK],
                     pop_last=True),
-                                                               columns=self.WEB_SCAN_VULN_HEADER)
+                columns=self.WEB_SCAN_HEADER),
-            dict_tracker['WEB_SCAN_SENSITIVE_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
+            'WEB_SCAN_SENSITIVE_BLOCK': pd.DataFrame(
+                self.utils.grab_section(report,
                     self.WEB_SCAN_SENSITIVE_BLOCK,
-                                                                                            end=[
+                    end=[self.WEB_SCAN_INFO_BLOCK, self.WEB_SCAN_SENSITIVE_BLOCK],
-                                                                                                self.WEB_SCAN_INFO_BLOCK,
-                                                                                                self.WEB_SCAN_SENSITIVE_BLOCK],
                     pop_last=True),
-                                                                columns=self.WEB_SCAN_SENSITIVE_HEADER)
+                columns=self.WEB_SCAN_HEADER),
-            dict_tracker['WEB_SCAN_INFO_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
+            'WEB_SCAN_INFO_BLOCK': pd.DataFrame(
+                self.utils.grab_section(
+                    report,
                     self.WEB_SCAN_INFO_BLOCK,
                     end=[self.QID_HEADER],
                     pop_last=True),
-                                                                columns=self.WEB_SCAN_INFO_HEADER)
+                columns=self.WEB_SCAN_INFO_HEADER),
-            dict_tracker['QID_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
+
+            'QID_HEADER': pd.DataFrame(
+                self.utils.grab_section(
+                    report,
                     self.QID_HEADER,
                     end=[self.GROUP_HEADER],
                     pop_last=True),
-                                                                columns=self.QID_HEADER)
+                columns=self.QID_HEADER),
-            dict_tracker['GROUP_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
+            'GROUP_HEADER': pd.DataFrame(
+                self.utils.grab_section(
+                    report,
                     self.GROUP_HEADER,
                     end=[self.OWASP_HEADER],
                     pop_last=True),
-                                                                columns=self.GROUP_HEADER)
+                columns=self.GROUP_HEADER),
-            dict_tracker['OWASP_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
+            'OWASP_HEADER': pd.DataFrame(
+                self.utils.grab_section(
+                    report,
                     self.OWASP_HEADER,
                     end=[self.WASC_HEADER],
                     pop_last=True),
-                                                                columns=self.OWASP_HEADER)
+                columns=self.OWASP_HEADER),
-            dict_tracker['WASC_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
+            'WASC_HEADER': pd.DataFrame(
-                                                                               self.WASC_HEADER, end=[['APPENDIX']],
+                self.utils.grab_section(
+                    report,
+                    self.WASC_HEADER,
+                    end=[['APPENDIX']],
                     pop_last=True),
-                                                                columns=self.WASC_HEADER)
+                columns=self.WASC_HEADER),
-
+            'SCAN_META': pd.DataFrame(
-            dict_tracker['SCAN_META'] = pd.DataFrame(self.utils.grab_section(report,
+                self.utils.grab_section(report,
                     self.SCAN_META,
                     end=[self.CATEGORY_HEADER],
                     pop_last=True),
-                                                                columns=self.SCAN_META)
+                columns=self.SCAN_META),
-
+            'CATEGORY_HEADER': pd.DataFrame(
-            dict_tracker['CATEGORY_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
+                self.utils.grab_section(report,
                     self.CATEGORY_HEADER),
                 columns=self.CATEGORY_HEADER)
-            all_dataframes.append(dict_tracker)
+        }
-
-        return all_dataframes
 
     def data_normalizer(self, dataframes):
         """
@@ -409,12 +321,21 @@ class qualysScanReport:
         :param dataframes:
         :return:
         """
-        df_dict = dataframes[0]
+        df_dict = dataframes
-        merged_df = pd.concat([df_dict['WEB_SCAN_VULN_BLOCK'], df_dict['WEB_SCAN_SENSITIVE_BLOCK'],
+        merged_df = pd.concat([
-                               df_dict['WEB_SCAN_INFO_BLOCK']], axis=0,
+            df_dict['WEB_SCAN_VULN_BLOCK'],
-                              ignore_index=False)
+            df_dict['WEB_SCAN_SENSITIVE_BLOCK'],
-        merged_df = pd.merge(merged_df, df_dict['QID_HEADER'], left_on='QID',
+            df_dict['WEB_SCAN_INFO_BLOCK']
-                             right_on='Id')
+        ], axis=0, ignore_index=False)
+
+        merged_df = pd.merge(
+                merged_df,
+                df_dict['QID_HEADER'].drop(
+                    #these columns always seem to be the same as what we're merging into
+                    ['CVSS V3 Attack Vector', 'CVSS V3 Base', 'CVSS V3 Temporal'],
+                    axis=1),
+                left_on='QID', right_on='Id'
+        )
 
         if 'Content' not in merged_df:
             merged_df['Content'] = ''
@@ -431,8 +352,11 @@ class qualysScanReport:
 
         merged_df = merged_df.assign(**df_dict['SCAN_META'].to_dict(orient='records')[0])
 
-        merged_df = pd.merge(merged_df, df_dict['CATEGORY_HEADER'], how='left', left_on=['Category', 'Severity Level'],
+        merged_df = pd.merge(
-                             right_on=['Category', 'Severity'], suffixes=('Severity', 'CatSev'))
+                merged_df, df_dict['CATEGORY_HEADER'],
+                how='left', left_on=['Category', 'Severity Level'],
+                right_on=['Category', 'Severity'], suffixes=('Severity', 'CatSev')
+        )
 
         merged_df = merged_df.replace('N/A', '').fillna('')
 

vulnwhisp/reporting/jira_api.py

@@ -1,15 +1,18 @@
+from __future__ import absolute_import
 import json
 import os
-from datetime import datetime, date, timedelta
+from datetime import datetime, date
 
 from jira import JIRA
-import requests
 import logging
 from bottle import template
 import re
+from six.moves import range
+
 
 class JiraAPI(object):
-    def __init__(self, hostname=None, username=None, password=None, path="", debug=False, clean_obsolete=True, max_time_window=12, decommission_time_window=3):
+    def __init__(self, hostname=None, username=None, password=None, path="", debug=False, clean_obsolete=True,
+                 max_time_window=12, decommission_time_window=3):
         self.logger = logging.getLogger('JiraAPI')
         if debug:
             self.logger.setLevel(logging.DEBUG)
@@ -41,10 +44,15 @@ class JiraAPI(object):
         # deletes the tag "server_decommission" from those tickets closed <=3 months ago
         self.decommission_cleanup()
 
-        self.jira_still_vulnerable_comment = '''This ticket has been reopened due to the vulnerability not having been fixed (if multiple assets are affected, all need to be fixed; if the server is down, lastest known vulnerability might be the one reported).
+        self.jira_still_vulnerable_comment = '''This ticket has been reopened due to the vulnerability not having been \
-        - In the case of the team accepting the risk and wanting to close the ticket, please add the label "*risk_accepted*" to the ticket before closing it.
+        fixed (if multiple assets are affected, all need to be fixed; if the server is down, lastest known \
-        - If server has been decommissioned, please add the label "*server_decommission*" to the ticket before closing it.
+        vulnerability might be the one reported).
-        - If when checking the vulnerability it looks like a false positive, _+please elaborate in a comment+_ and add the label "*false_positive*" before closing it; we will review it and report it to the vendor.
+        - In the case of the team accepting the risk and wanting to close the ticket, please add the label \
+        "*risk_accepted*" to the ticket before closing it.
+        - If server has been decommissioned, please add the label "*server_decommission*" to the ticket before closing \
+        it.
+        - If when checking the vulnerability it looks like a false positive, _+please elaborate in a comment+_ and add \
+         the label "*false_positive*" before closing it; we will review it and report it to the vendor.
         
         If you have further doubts, please contact the Security Team.'''
 
@@ -91,13 +99,15 @@ class JiraAPI(object):
         return len(self.jira.search_issues(jql, maxResults=0))
 
     def metrics_closed_tickets(self, project=None):
-        jql = "labels= vulnerability_management and NOT resolution = Unresolved AND created >=startOfMonth(-{})".format(self.max_time_tracking) 
+        jql = "labels= vulnerability_management and NOT resolution = Unresolved AND created >=startOfMonth(-{})".format(
+            self.max_time_tracking)
         if project:
             jql += " and (project='{}')".format(project)
         return len(self.jira.search_issues(jql, maxResults=0))
 
     def sync(self, vulnerabilities, project, components=[]):
-        #JIRA structure of each vulnerability: [source, scan_name, title, diagnosis, consequence, solution, ips, risk, references]
+        # JIRA structure of each vulnerability: [source, scan_name, title, diagnosis, consequence, solution,
+        # ips, risk, references]
         self.logger.info("JIRA Sync started")
 
         for vuln in vulnerabilities:
@@ -106,7 +116,8 @@ class JiraAPI(object):
             if " " in vuln['scan_name']:
                 vuln['scan_name'] = "_".join(vuln['scan_name'].split(" "))
 
-            # we exclude from the vulnerabilities to report those assets that already exist with *risk_accepted*/*server_decommission*
+            # we exclude from the vulnerabilities to report those assets that already exist
+            # with *risk_accepted*/*server_decommission*
             vuln = self.exclude_accepted_assets(vuln)
 
             # make sure after exclusion of risk_accepted assets there are still assets
@@ -131,13 +142,17 @@ class JiraAPI(object):
                 # create local text file with assets, attach it to ticket
                 if len(vuln['ips']) > self.max_ips_ticket:
                     attachment_contents = vuln['ips']
-                    vuln['ips'] = ["Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(assets = len(attachment_contents))]
+                    vuln['ips'] = [
+                        "Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(
+                            assets=len(attachment_contents))]
                 try:
                     tpl = template(self.template_path, vuln)
                 except Exception as e:
                     self.logger.error('Exception templating: {}'.format(str(e)))
                     return 0
-                self.create_ticket(title=vuln['title'], desc=tpl, project=project, components=components, tags=[vuln['source'], vuln['scan_name'], 'vulnerability', vuln['risk']], attachment_contents = attachment_contents)
+                self.create_ticket(title=vuln['title'], desc=tpl, project=project, components=components,
+                                   tags=[vuln['source'], vuln['scan_name'], 'vulnerability', vuln['risk']],
+                                   attachment_contents=attachment_contents)
             else:
                 self.logger.info("Ignoring vulnerability as all assets are already reported in a risk_accepted ticket")
 
@@ -153,7 +168,8 @@ class JiraAPI(object):
         labels = [vuln['source'], vuln['scan_name'], 'vulnerability_management', 'vulnerability']
 
         if not self.excluded_tickets:
-            jql = "{} AND labels in (risk_accepted,server_decommission, false_positive) AND NOT labels=advisory AND created >=startOfMonth(-{})".format(" AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
+            jql = "{} AND labels in (risk_accepted,server_decommission, false_positive) AND NOT labels=advisory AND created >=startOfMonth(-{})".format(
+                " AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
             self.excluded_tickets = self.jira.search_issues(jql, maxResults=0)
 
         title = vuln['title']
@@ -163,7 +179,8 @@ class JiraAPI(object):
         assets_to_exclude = []
         tickets_excluded_assets = []
         for index in range(len(self.excluded_tickets)):
-            checking_ticketid, checking_title, checking_assets = self.ticket_get_unique_fields(self.excluded_tickets[index])
+            checking_ticketid, checking_title, checking_assets = self.ticket_get_unique_fields(
+                self.excluded_tickets[index])
             if title.encode('ascii') == checking_title.encode('ascii'):
                 if checking_assets:
                     # checking_assets is a list, we add to our full list for later delete all assets
@@ -172,7 +189,8 @@ class JiraAPI(object):
 
         if assets_to_exclude:
             assets_to_remove = []
-            self.logger.warn("Vulnerable Assets seen on an already existing risk_accepted Jira ticket: {}".format(', '.join(tickets_excluded_assets)))
+            self.logger.warn("Vulnerable Assets seen on an already existing risk_accepted Jira ticket: {}".format(
+                ', '.join(tickets_excluded_assets)))
             self.logger.debug("Original assets: {}".format(vuln['ips']))
             # assets in vulnerability have the structure "ip - hostname - port", so we need to match by partial
             for exclusion in assets_to_exclude:
@@ -180,7 +198,9 @@ class JiraAPI(object):
                 # and we don't want it to affect the rest of the processing (otherwise, it would miss the asset right after the removed one)
                 for index in range(len(vuln['ips']))[::-1]:
                     if exclusion == vuln['ips'][index].split(" - ")[0]:
-                        self.logger.debug("Deleting asset {} from vulnerability {}, seen in risk_accepted.".format(vuln['ips'][index], title))
+                        self.logger.debug(
+                            "Deleting asset {} from vulnerability {}, seen in risk_accepted.".format(vuln['ips'][index],
+                                                                                                     title))
                         vuln['ips'].pop(index)
             self.logger.debug("Modified assets: {}".format(vuln['ips']))
 
@@ -202,7 +222,8 @@ class JiraAPI(object):
             self.logger.info("Retrieving all JIRA tickets with the following tags {}".format(labels))
             # we want to check all JIRA tickets, to include tickets moved to other queues
             # will exclude tickets older than 12 months, old tickets will get closed for higiene and recreated if still vulnerable
-            jql = "{} AND NOT labels=advisory AND created >=startOfMonth(-{})".format(" AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
+            jql = "{} AND NOT labels=advisory AND created >=startOfMonth(-{})".format(
+                " AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
 
             self.all_tickets = self.jira.search_issues(jql, maxResults=0)
 
@@ -212,7 +233,8 @@ class JiraAPI(object):
         for index in range(len(self.all_tickets)):
             checking_ticketid, checking_title, checking_assets = self.ticket_get_unique_fields(self.all_tickets[index])
             # added "not risk_accepted", as if it is risk_accepted, we will create a new ticket excluding the accepted assets
-            if title.encode('ascii') == checking_title.encode('ascii') and not self.is_risk_accepted(self.jira.issue(checking_ticketid)): 
+            if title.encode('ascii') == checking_title.encode('ascii') and not self.is_risk_accepted(
+                    self.jira.issue(checking_ticketid)):
                 difference = list(set(assets).symmetric_difference(checking_assets))
                 # to check intersection - set(assets) & set(checking_assets)
                 if difference:
@@ -239,9 +261,12 @@ class JiraAPI(object):
         # structure the text to have the same structure as the assets from the attachment
         affected_assets = ""
         try:
-            affected_assets = ticket.raw.get('fields', {}).get('description').encode("ascii").split("{panel:title=Affected Assets}")[1].split("{panel}")[0].replace('\n','').replace(' * ','\n').replace('\n', '', 1)
+            affected_assets = \
+            ticket.raw.get('fields', {}).get('description').encode("ascii").split("{panel:title=Affected Assets}")[
+                1].split("{panel}")[0].replace('\n', '').replace(' * ', '\n').replace('\n', '', 1)
         except Exception as e:
-            self.logger.error("Unable to process the Ticket's 'Affected Assets'. Ticket ID: {}. Reason: {}".format(ticket, e))
+            self.logger.error(
+                "Unable to process the Ticket's 'Affected Assets'. Ticket ID: {}. Reason: {}".format(ticket, e))
 
         if affected_assets:
             if _raw:
@@ -280,7 +305,8 @@ class JiraAPI(object):
             affected_assets = self.jira.attachment(attachment_id).get()
 
         except Exception as e:
-            self.logger.error("Failed to get assets from ticket attachment. Ticket ID: {}. Reason: {}".format(ticket, e))
+            self.logger.error(
+                "Failed to get assets from ticket attachment. Ticket ID: {}. Reason: {}".format(ticket, e))
 
         if affected_assets:
             if _raw:
@@ -353,8 +379,10 @@ class JiraAPI(object):
         if self.is_ticket_resolved(ticket_obj):
             ticket_data = ticket_obj.raw.get('fields')
             # dates follow format '2018-11-06T10:36:13.849+0100'
-            created = [int(x) for x in ticket_data['created'].split('.')[0].replace('T', '-').replace(':','-').split('-')]
+            created = [int(x) for x in
-            resolved =[int(x) for x in ticket_data['resolutiondate'].split('.')[0].replace('T', '-').replace(':','-').split('-')]
+                       ticket_data['created'].split('.')[0].replace('T', '-').replace(':', '-').split('-')]
+            resolved = [int(x) for x in
+                        ticket_data['resolutiondate'].split('.')[0].replace('T', '-').replace(':', '-').split('-')]
 
             start = datetime(created[0], created[1], created[2], created[3], created[4], created[5])
             end = datetime(resolved[0], resolved[1], resolved[2], resolved[3], resolved[4], resolved[5])
@@ -405,7 +433,9 @@ class JiraAPI(object):
         attachment_contents = []
         if len(vuln['ips']) > self.max_ips_ticket:
             attachment_contents = vuln['ips']
-            vuln['ips'] = ["Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(assets = len(attachment_contents))]
+            vuln['ips'] = [
+                "Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(
+                    assets=len(attachment_contents))]
 
         # fill the ticket description template
         try:
@@ -425,7 +455,8 @@ class JiraAPI(object):
             self.logger.info("Ticket {} updated successfully".format(ticketid))
             self.add_label(ticketid, 'updated')
         except Exception as e:
-            self.logger.error("Error while trying up update ticket {ticketid}.\nReason: {e}".format(ticketid = ticketid, e=e))
+            self.logger.error(
+                "Error while trying up update ticket {ticketid}.\nReason: {e}".format(ticketid=ticketid, e=e))
         return 0
 
     def add_label(self, ticketid, label):
@@ -437,8 +468,9 @@ class JiraAPI(object):
             try:
                 ticket_obj.update(fields={"labels": ticket_obj.fields.labels})
                 self.logger.info("Added label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
-            except:
+            except Exception as e:
-                self.logger.error("Error while trying to add label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
+                self.logger.error(
+                    "Error while trying to add label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
 
         return 0
 
@@ -451,8 +483,9 @@ class JiraAPI(object):
             try:
                 ticket_obj.update(fields={"labels": ticket_obj.fields.labels})
                 self.logger.info("Removed label {label} from ticket {ticket}".format(label=label, ticket=ticketid))
-            except:
+            except Exception as e:
-                self.logger.error("Error while trying to remove label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
+                self.logger.error("Error while trying to remove label {label} to ticket {ticket}".format(label=label,
+                                                                                                         ticket=ticketid))
         else:
             self.logger.error("Error: label {label} not in ticket {ticket}".format(label=label, ticket=ticketid))
 
@@ -478,7 +511,6 @@ class JiraAPI(object):
             self.close_ticket(ticket, self.JIRA_RESOLUTION_FIXED, comment)
         return 0
 
-
     def is_ticket_reopenable(self, ticket_obj):
         transitions = self.jira.transitions(ticket_obj)
         for transition in transitions:
@@ -507,7 +539,6 @@ class JiraAPI(object):
         self.logger.debug("Checked ticket {} is already open".format(ticket_obj))
         return False
 
-
     def is_risk_accepted(self, ticket_obj):
         if ticket_obj is not None:
             if ticket_obj.raw['fields'].get('labels') is not None:
@@ -533,7 +564,8 @@ class JiraAPI(object):
             if (not self.is_risk_accepted(ticket_obj) or ignore_labels):
                 try:
                     if self.is_ticket_reopenable(ticket_obj):
-                        error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_REOPEN_ISSUE, comment = comment)
+                        error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_REOPEN_ISSUE,
+                                                           comment=comment)
                         self.logger.info("Ticket {} reopened successfully".format(ticketid))
                         if not ignore_labels:
                             self.add_label(ticketid, 'reopened')
@@ -553,7 +585,8 @@ class JiraAPI(object):
                 if self.is_ticket_closeable(ticket_obj):
                     # need to add the label before closing the ticket
                     self.add_label(ticketid, 'closed')
-                    error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_CLOSE_ISSUE, comment = comment, resolution = {"name": resolution })
+                    error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_CLOSE_ISSUE,
+                                                       comment=comment, resolution={"name": resolution})
                     self.logger.info("Ticket {} closed successfully".format(ticketid))
                     return 1
             except Exception as e:
@@ -566,7 +599,8 @@ class JiraAPI(object):
     def close_obsolete_tickets(self):
         # Close tickets older than 12 months, vulnerabilities not solved will get created a new ticket 
         self.logger.info("Closing obsolete tickets older than {} months".format(self.max_time_tracking))
-        jql = "labels=vulnerability_management AND NOT labels=advisory AND created <startOfMonth(-{}) and resolution=Unresolved".format(self.max_time_tracking)
+        jql = "labels=vulnerability_management AND NOT labels=advisory AND created <startOfMonth(-{}) and resolution=Unresolved".format(
+            self.max_time_tracking)
         tickets_to_close = self.jira.search_issues(jql, maxResults=0)
 
         comment = '''This ticket is being closed for hygiene, as it is more than {} months old.
@@ -597,7 +631,8 @@ class JiraAPI(object):
             return True
         try:
             self.logger.info("Saving locally tickets from the last {} months".format(self.max_time_tracking))
-            jql = "labels=vulnerability_management AND NOT labels=advisory AND created >=startOfMonth(-{})".format(self.max_time_tracking)
+            jql = "labels=vulnerability_management AND NOT labels=advisory AND created >=startOfMonth(-{})".format(
+                self.max_time_tracking)
             tickets_data = self.jira.search_issues(jql, maxResults=0)
 
             # TODO process tickets, creating a new field called "_metadata" with all the affected assets well structured
@@ -621,7 +656,6 @@ class JiraAPI(object):
                         assets_json = self.parse_asset_to_json(assets)
                         _metadata["affected_hosts"].append(assets_json)
 
-
                 temp_ticket = ticket.raw.get('fields')
                 temp_ticket['_metadata'] = _metadata
 
@@ -646,13 +680,16 @@ class JiraAPI(object):
         closed already for more than x months (default is 3 months) in order to clean solved issues
         for statistics purposes
         '''
-        self.logger.info("Deleting 'server_decommission' tag from tickets closed more than {} months ago".format(self.max_decommission_time))
+        self.logger.info("Deleting 'server_decommission' tag from tickets closed more than {} months ago".format(
+            self.max_decommission_time))
 
-        jql = "labels=vulnerability_management AND labels=server_decommission and resolutiondate <=startOfMonth(-{})".format(self.max_decommission_time)
+        jql = "labels=vulnerability_management AND labels=server_decommission and resolutiondate <=startOfMonth(-{})".format(
+            self.max_decommission_time)
         decommissioned_tickets = self.jira.search_issues(jql, maxResults=0)
 
         comment = '''This ticket is having deleted the *server_decommission* tag, as it is more than {} months old and is expected to already have been decommissioned.
-        If that is not the case and the vulnerability still exists, the vulnerability will be opened again.'''.format(self.max_decommission_time)
+        If that is not the case and the vulnerability still exists, the vulnerability will be opened again.'''.format(
+            self.max_decommission_time)
 
         for ticket in decommissioned_tickets:
             # we open first the ticket, as we want to make sure the process is not blocked due to

vulnwhisp/test/mock.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import os
 import logging
 import httpretty
@@ -20,10 +21,12 @@ class mockAPI(object):
 
     def get_directories(self, path):
         dir, subdirs, files = next(os.walk(path))
+        self.logger.debug('Subdirectories found: {}'.format(subdirs))
         return subdirs
 
     def get_files(self, path):
         dir, subdirs, files = next(os.walk(path))
+        self.logger.debug('Files found: {}'.format(files))
         return files
 
     def qualys_vuln_callback(self, request, uri, response_headers):

vulnwhisp/vulnwhisp.py

@@ -1,13 +1,17 @@
 #!/usr/bin/python
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import
+from six.moves import range
+from functools import reduce
+
 __author__ = 'Austin Taylor'
 
-from base.config import vwConfig
+from .base.config import vwConfig
-from frameworks.nessus import NessusAPI
+from .frameworks.nessus import NessusAPI
-from frameworks.qualys_web import qualysScanReport
+from .frameworks.qualys_web import qualysScanReport
-from frameworks.qualys_vuln import qualysVulnScan
+from .frameworks.qualys_vuln import qualysVulnScan
-from frameworks.openvas import OpenVAS_API
+from .frameworks.openvas import OpenVAS_API
-from reporting.jira_api import JiraAPI
+from .reporting.jira_api import JiraAPI
 import pandas as pd
 from lxml import objectify
 import sys
@@ -21,7 +25,6 @@ import socket
 
 
 class vulnWhispererBase(object):
-
     CONFIG_SECTION = None
 
     def __init__(
@@ -65,8 +68,6 @@ class vulnWhispererBase(object):
             self.db_path = self.config.get(self.CONFIG_SECTION, 'db_path')
             self.verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
 
-
-
         if self.db_name is not None:
             if self.db_path:
                 self.database = os.path.join(self.db_path,
@@ -88,7 +89,8 @@ class vulnWhispererBase(object):
                 self.cur = self.conn.cursor()
                 self.logger.info('Connected to database at {loc}'.format(loc=self.database))
             except Exception as e:
-                self.logger.error('Could not connect to database at {loc}\nReason: {e} - Please ensure the path exist'.format(
+                self.logger.error(
+                    'Could not connect to database at {loc}\nReason: {e} - Please ensure the path exist'.format(
                         e=e,
                         loc=self.database))
         else:
@@ -185,7 +187,8 @@ class vulnWhispererBase(object):
         """
         try:
             self.conn.text_factory = str
-            self.cur.execute('SELECT uuid FROM scan_history where source = "{config_section}"'.format(config_section=self.CONFIG_SECTION))
+            self.cur.execute('SELECT uuid FROM scan_history where source = "{config_section}"'.format(
+                config_section=self.CONFIG_SECTION))
             results = frozenset([r[0] for r in self.cur.fetchall()])
         except:
             results = []
@@ -208,7 +211,9 @@ class vulnWhispererBase(object):
 
         try:
             self.conn.text_factory = str
-            self.cur.execute('SELECT filename FROM scan_history WHERE source="{}" AND scan_name="{}" ORDER BY last_modified DESC LIMIT 1;'.format(source, scan_name))
+            self.cur.execute(
+                'SELECT filename FROM scan_history WHERE source="{}" AND scan_name="{}" ORDER BY last_modified DESC LIMIT 1;'.format(
+                    source, scan_name))
             # should always return just one filename
             results = [r[0] for r in self.cur.fetchall()][0]
 
@@ -216,10 +221,13 @@ class vulnWhispererBase(object):
             # TODO delete backward compatibility check after some versions
             last_column_table = self.cur.execute('PRAGMA table_info(scan_history)').fetchall()[-1][1]
             if results and last_column_table == self.table_columns[-1]:
-                reported = self.cur.execute('SELECT reported FROM scan_history WHERE filename="{}"'.format(results)).fetchall()
+                reported = self.cur.execute(
+                    'SELECT reported FROM scan_history WHERE filename="{}"'.format(results)).fetchall()
                 reported = reported[0][0]
                 if reported:
-                    self.logger.debug("Last downloaded scan from source {source} scan_name {scan_name} has already been reported".format(source=source, scan_name=scan_name))
+                    self.logger.debug(
+                        "Last downloaded scan from source {source} scan_name {scan_name} has already been reported".format(
+                            source=source, scan_name=scan_name))
 
         except Exception as e:
             self.logger.error("Error when getting latest results from {}.{} : {}".format(source, scan_name, e))
@@ -254,8 +262,8 @@ class vulnWhispererBase(object):
 
         return results
 
-class vulnWhispererNessus(vulnWhispererBase):
 
+class vulnWhispererNessus(vulnWhispererBase):
     CONFIG_SECTION = None
 
     def __init__(
@@ -322,8 +330,6 @@ class vulnWhispererNessus(vulnWhispererBase):
                 return False
                 # sys.exit(1)
 
-
-
     def scan_count(self, scans, completed=False):
         """
 
@@ -365,7 +371,6 @@ class vulnWhispererNessus(vulnWhispererBase):
             scan_records = [s for s in scan_records if s['status'] == 'completed']
         return scan_records
 
-
     def whisper_nessus(self):
         if self.nessus_connect:
             scan_data = self.nessus.scans
@@ -420,7 +425,8 @@ class vulnWhispererNessus(vulnWhispererBase):
                     s['uuid'],
                 )
 
-                # TODO Create directory sync function which scans the directory for files that exist already and populates the database
+                # TODO Create directory sync function which scans the directory for files that exist already and
+                #  populates the database
 
                 folder_id = s['folder_id']
                 if self.CONFIG_SECTION == 'tenable':
@@ -450,22 +456,26 @@ class vulnWhispererNessus(vulnWhispererBase):
                                 0,
                             )
                             self.record_insert(record_meta)
-                            self.logger.info('File {filename} already exist! Updating database'.format(filename=relative_path_name))
+                            self.logger.info(
+                                'File {filename} already exist! Updating database'.format(filename=relative_path_name))
                     else:
                         try:
                             file_req = \
                                 self.nessus.download_scan(scan_id=scan_id, history=history_id,
                                                           export_format='csv')
                         except Exception as e:
-                            self.logger.error('Could not download {} scan {}: {}'.format(self.CONFIG_SECTION, scan_id, str(e)))
+                            self.logger.error(
+                                'Could not download {} scan {}: {}'.format(self.CONFIG_SECTION, scan_id, str(e)))
                             self.exit_code += 1
                             continue
 
                         clean_csv = \
                             pd.read_csv(io.StringIO(file_req.decode('utf-8')))
                         if len(clean_csv) > 2:
-                            self.logger.info('Processing {}/{} for scan: {}'.format(scan_count, len(scan_list), scan_name.encode('utf8')))
+                            self.logger.info('Processing {}/{} for scan: {}'.format(scan_count, len(scan_list),
-                            columns_to_cleanse = ['CVSS','CVE','Description','Synopsis','Solution','See Also','Plugin Output', 'MAC Address']
+                                                                                    scan_name.encode('utf8')))
+                            columns_to_cleanse = ['CVSS', 'CVE', 'Description', 'Synopsis', 'Solution', 'See Also',
+                                                  'Plugin Output', 'MAC Address']
 
                             for col in columns_to_cleanse:
                                 if col in clean_csv:
@@ -486,7 +496,8 @@ class vulnWhispererNessus(vulnWhispererBase):
                             )
                             self.record_insert(record_meta)
                             self.logger.info('{filename} records written to {path} '.format(filename=clean_csv.shape[0],
-                                                                                            path=file_name.encode('utf8')))
+                                                                                            path=file_name.encode(
+                                                                                                'utf8')))
                         else:
                             record_meta = (
                                 scan_name,
@@ -501,27 +512,32 @@ class vulnWhispererNessus(vulnWhispererBase):
                                 0,
                             )
                             self.record_insert(record_meta)
-                            self.logger.warn('{} has no host available... Updating database and skipping!'.format(file_name))
+                            self.logger.warn(
+                                '{} has no host available... Updating database and skipping!'.format(file_name))
             self.conn.close()
             self.logger.info('Scan aggregation complete! Connection to database closed.')
         else:
-            self.logger.error('Failed to use scanner at {host}:{port}'.format(host=self.hostname, port=self.nessus_port))
+            self.logger.error(
+                'Failed to use scanner at {host}:{port}'.format(host=self.hostname, port=self.nessus_port))
             self.exit_code += 1
         return self.exit_code
 
 
 class vulnWhispererQualys(vulnWhispererBase):
-
     CONFIG_SECTION = 'qualys_web'
     COLUMN_MAPPING = {'Access Path': 'access_path',
                       'Ajax Request': 'ajax_request',
                       'Ajax Request ID': 'ajax_request_id',
                       'Authentication': 'authentication',
                       'CVSS Base': 'cvss',
+                      'CVSS V3 Attack Vector': 'cvss_v3_attack_vector',
+                      'CVSS V3 Base': 'cvss_v3_base',
+                      'CVSS V3 Temporal': 'cvss_v3_temporal',
                       'CVSS Temporal': 'cvss_temporal',
                       'CWE': 'cwe',
                       'Category': 'category',
                       'Content': 'content',
+                      'Custom Attributes': 'custom_attributes',
                       'DescriptionSeverity': 'severity_description',
                       'DescriptionCatSev': 'category_description',
                       'Detection ID': 'detection_id',
@@ -537,15 +553,19 @@ class vulnWhispererQualys(vulnWhispererBase):
                       'Ignore User': 'ignore_user',
                       'Ignored': 'ignored',
                       'Impact': 'impact',
+                      'Info#1': 'info_1',
                       'Last Time Detected': 'last_time_detected',
                       'Last Time Tested': 'last_time_tested',
                       'Level': 'level',
                       'OWASP': 'owasp',
                       'Operating System': 'operating_system',
                       'Owner': 'owner',
-                     'Param': 'param',
+                      'Param/Cookie': 'param',
                       'Payload #1': 'payload_1',
+                      'Port': 'port',
+                      'Protocol': 'protocol',
                       'QID': 'plugin_id',
+                      'Request Body #1': 'request_body_1',
                       'Request Headers #1': 'request_headers_1',
                       'Request Method #1': 'request_method_1',
                       'Request URL #1': 'request_url_1',
@@ -554,13 +574,17 @@ class vulnWhispererQualys(vulnWhispererBase):
                       'Severity': 'risk',
                       'Severity Level': 'security_level',
                       'Solution': 'solution',
+                      'Tags': 'tags',
                       'Times Detected': 'times_detected',
                       'Title': 'plugin_name',
                       'URL': 'url',
+                      'Unique ID': 'unique_id',
                       'Url': 'uri',
                       'Vulnerability Category': 'vulnerability_category',
+                      'Virtual Host': 'virutal_host',
                       'WASC': 'wasc',
                       'Web Application Name': 'web_application_name'}
+
     def __init__(
             self,
             config=None,
@@ -668,7 +692,8 @@ class vulnWhispererQualys(vulnWhispererBase):
                         self.logger.info('Removing report {} from Qualys Database'.format(generated_report_id))
                         cleaning_up = self.qualys_scan.qw.delete_report(generated_report_id)
                         os.remove(self.path_check(str(generated_report_id) + '.csv'))
-                        self.logger.info('Deleted report from local disk: {}'.format(self.path_check(str(generated_report_id))))
+                        self.logger.info(
+                            'Deleted report from local disk: {}'.format(self.path_check(str(generated_report_id))))
                 else:
                     self.logger.error('Could not process report ID: {}'.format(status))
 
@@ -676,7 +701,6 @@ class vulnWhispererQualys(vulnWhispererBase):
             self.logger.error('Could not process {}: {}'.format(report_id, str(e)))
         return vuln_ready
 
-
     def identify_scans_to_process(self):
         if self.uuids:
             self.scans_to_process = self.latest_scans[~self.latest_scans['id'].isin(self.uuids)]
@@ -684,7 +708,6 @@ class vulnWhispererQualys(vulnWhispererBase):
             self.scans_to_process = self.latest_scans
         self.logger.info('Identified {new} scans to be processed'.format(new=len(self.scans_to_process)))
 
-
     def process_web_assets(self):
         counter = 0
         self.identify_scans_to_process()
@@ -765,7 +788,6 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
         if report_id:
             self.logger.info('Processing report ID: {}'.format(report_id))
 
-
             scan_name = report_id.replace('-', '')
             report_name = 'openvas_scan_{scan_name}_{last_updated}.{extension}'.format(scan_name=scan_name,
                                                                                        last_updated=launched_date,
@@ -833,7 +855,8 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
             for scan in self.scans_to_process.iterrows():
                 counter += 1
                 info = scan[1]
-                self.logger.info('Processing {}/{} - Report ID: {}'.format(counter, len(self.scans_to_process), info['report_ids']))
+                self.logger.info(
+                    'Processing {}/{} - Report ID: {}'.format(counter, len(self.scans_to_process), info['report_ids']))
                 self.whisper_reports(report_id=info['report_ids'],
                                      launched_date=info['epoch'])
             self.logger.info('Processing complete')
@@ -844,7 +867,6 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
 
 
 class vulnWhispererQualysVuln(vulnWhispererBase):
-
     CONFIG_SECTION = 'qualys_vuln'
     COLUMN_MAPPING = {'cvss_base': 'cvss',
                       'cvss3_base': 'cvss3',
@@ -945,7 +967,6 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
             self.logger.info('Report written to {}'.format(report_name))
         return self.exit_code
 
-
     def identify_scans_to_process(self):
         self.latest_scans = self.qualys_scan.qw.get_all_scans()
         if self.uuids:
@@ -956,7 +977,6 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
             self.scans_to_process = self.latest_scans
         self.logger.info('Identified {new} scans to be processed'.format(new=len(self.scans_to_process)))
 
-
     def process_vuln_scans(self):
         counter = 0
         self.identify_scans_to_process()
@@ -976,7 +996,6 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
 
 
 class vulnWhispererJIRA(vulnWhispererBase):
-
     CONFIG_SECTION = 'jira'
 
     def __init__(
@@ -1028,10 +1047,11 @@ class vulnWhispererJIRA(vulnWhispererBase):
 
         if not self.config.exists_jira_profiles(profiles):
             self.config.update_jira_profiles(profiles)
-            self.logger.info("Jira profiles have been created in {config}, please fill the variables before rerunning the module.".format(config=self.config_path))
+            self.logger.info(
+                "Jira profiles have been created in {config}, please fill the variables before rerunning the module.".format(
+                    config=self.config_path))
             sys.exit(0)
 
-
     def get_env_variables(self, source, scan_name):
         # function returns an array with [jira_project, jira_components, datafile_path]
 
@@ -1069,11 +1089,15 @@ class vulnWhispererJIRA(vulnWhispererBase):
                 fullpath = "{}/{}".format(root, filename)
 
         if reported:
-            self.logger.warn('Last Scan of "{scan_name}" for source "{source}" has already been reported; will be skipped.'.format(scan_name=scan_name, source=source))
+            self.logger.warn(
+                'Last Scan of "{scan_name}" for source "{source}" has already been reported; will be skipped.'.format(
+                    scan_name=scan_name, source=source))
             return [False] * 5
 
         if not fullpath:
-            self.logger.error('Scan of "{scan_name}" for source "{source}" has not been found. Please check that the scanner data files are in place.'.format(scan_name=scan_name, source=source))
+            self.logger.error(
+                'Scan of "{scan_name}" for source "{source}" has not been found. Please check that the scanner data files are in place.'.format(
+                    scan_name=scan_name, source=source))
             sys.exit(1)
 
         dns_resolv = self.config.get('jira', 'dns_resolv')
@@ -1087,7 +1111,6 @@ class vulnWhispererJIRA(vulnWhispererBase):
 
         return project, components, fullpath, min_critical, dns_resolv
 
-
     def parse_nessus_vulnerabilities(self, fullpath, source, scan_name, min_critical):
 
         vulnerabilities = []
@@ -1116,7 +1139,8 @@ class vulnWhispererJIRA(vulnWhispererBase):
                 vuln['consequence'] = df.loc[index]['Description'].replace('\\n', ' ')
                 vuln['solution'] = df.loc[index]['Solution'].replace('\\n', ' ')
                 vuln['ips'] = []
-                vuln['ips'].append("{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], df.loc[index]['Port']))
+                vuln['ips'].append(
+                    "{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], df.loc[index]['Port']))
                 vuln['risk'] = df.loc[index]['Risk'].lower()
 
                 # Nessus "nan" value gets automatically casted to float by python
@@ -1130,7 +1154,8 @@ class vulnWhispererJIRA(vulnWhispererBase):
                 # grouping assets by vulnerability to open on single ticket, as each asset has its own nessus entry
                 for vuln in vulnerabilities:
                     if vuln['title'] == df.loc[index]['Name']:
-                        vuln['ips'].append("{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], df.loc[index]['Port']))
+                        vuln['ips'].append("{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'],
+                                                               df.loc[index]['Port']))
 
         return vulnerabilities
 
@@ -1155,7 +1180,8 @@ class vulnWhispererJIRA(vulnWhispererBase):
                 continue
 
             elif data[index]['type'] == 'Practice' or data[index]['type'] == 'Ig':
-                self.logger.debug("Vulnerability '{vuln}' ignored, as it is 'Practice/Potential', not verified.".format(vuln=data[index]['plugin_name']))
+                self.logger.debug("Vulnerability '{vuln}' ignored, as it is 'Practice/Potential', not verified.".format(
+                    vuln=data[index]['plugin_name']))
                 continue
 
             if not vulnerabilities or data[index]['plugin_name'] not in [entry['title'] for entry in vulnerabilities]:
@@ -1171,7 +1197,8 @@ class vulnWhispererJIRA(vulnWhispererBase):
                 vuln['ips'] = []
                 # TODO ADDED DNS RESOLUTION FROM QUALYS! \n SEPARATORS INSTEAD OF \\n!
 
-                vuln['ips'].append("{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv)))
+                vuln['ips'].append(
+                    "{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv)))
 
                 # different risk system than Nessus!
                 vuln['risk'] = risks[int(data[index]['risk']) - 1]
@@ -1186,7 +1213,8 @@ class vulnWhispererJIRA(vulnWhispererBase):
                 # grouping assets by vulnerability to open on single ticket, as each asset has its own nessus entry
                 for vuln in vulnerabilities:
                     if vuln['title'] == data[index]['plugin_name']:
-                        vuln['ips'].append("{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv)))
+                        vuln['ips'].append(
+                            "{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv)))
 
         return vulnerabilities
 
@@ -1200,7 +1228,7 @@ class vulnWhispererJIRA(vulnWhispererBase):
             if vuln['dns']:
                 values['dns'] = vuln['dns']
             else:
-                if values['ip'] in self.host_resolv_cache.keys():
+                if values['ip'] in list(self.host_resolv_cache.keys()):
                     self.logger.debug("Hostname from {ip} cached, retrieving from cache.".format(ip=values['ip']))
                     values['dns'] = self.host_resolv_cache[values['ip']]
                 else:
@@ -1226,14 +1254,16 @@ class vulnWhispererJIRA(vulnWhispererBase):
 
         return 0
 
-
     def jira_sync(self, source, scan_name):
-        self.logger.info("Jira Sync triggered for source '{source}' and scan '{scan_name}'".format(source=source, scan_name=scan_name))
+        self.logger.info("Jira Sync triggered for source '{source}' and scan '{scan_name}'".format(source=source,
+                                                                                                   scan_name=scan_name))
 
         project, components, fullpath, min_critical, dns_resolv = self.get_env_variables(source, scan_name)
 
         if not project:
-            self.logger.debug("Skipping scan for source '{source}' and scan '{scan_name}': vulnerabilities have already been reported.".format(source=source, scan_name=scan_name))
+            self.logger.debug(
+                "Skipping scan for source '{source}' and scan '{scan_name}': vulnerabilities have already been reported.".format(
+                    source=source, scan_name=scan_name))
             return False
 
         vulnerabilities = []
@@ -1244,7 +1274,8 @@ class vulnWhispererJIRA(vulnWhispererBase):
 
         # ***Qualys VM parsing***
         if source == "qualys_vuln":
-            vulnerabilities = self.parse_qualys_vuln_vulnerabilities(fullpath, source, scan_name, min_critical, dns_resolv)
+            vulnerabilities = self.parse_qualys_vuln_vulnerabilities(fullpath, source, scan_name, min_critical,
+                                                                     dns_resolv)
 
         # ***JIRA sync***
         if vulnerabilities:
@@ -1253,7 +1284,9 @@ class vulnWhispererJIRA(vulnWhispererBase):
 
             self.jira.sync(vulnerabilities, project, components)
         else:
-            self.logger.info("[{source}.{scan_name}] No vulnerabilities or vulnerabilities not parsed.".format(source=source, scan_name=scan_name))
+            self.logger.info(
+                "[{source}.{scan_name}] No vulnerabilities or vulnerabilities not parsed.".format(source=source,
+                                                                                                  scan_name=scan_name))
             self.set_latest_scan_reported(fullpath.split("/")[-1])
             return False
 
@@ -1276,10 +1309,13 @@ class vulnWhispererJIRA(vulnWhispererBase):
                 try:
                     self.jira_sync(self.config.get(scan, 'source'), self.config.get(scan, 'scan_name'))
                 except Exception as e:
-                    self.logger.error("VulnWhisperer wasn't able to report the vulnerabilities from the '{}'s source".format(self.config.get(scan, 'source')))
+                    self.logger.error(
+                        "VulnWhisperer wasn't able to report the vulnerabilities from the '{}'s source".format(
+                            self.config.get(scan, 'source')))
             return True
         return False
 
+
 class vulnWhisperer(object):
 
     def __init__(self,
@@ -1303,7 +1339,6 @@ class vulnWhisperer(object):
         self.scanname = scanname
         self.exit_code = 0
 
-
     def whisper_vulnerabilities(self):
 
         if self.profile == 'nessus':