#!/usr/bin/python
# -*- coding: utf-8 -*-
__author__ = 'Austin Taylor'

import io
import json
import logging
import os
import re
import socket
import sqlite3
import sys
import time
from datetime import datetime, timedelta

import numpy as np
import pandas as pd
from lxml import objectify

from base.config import vwConfig
from frameworks.nessus import NessusAPI
from frameworks.openvas import OpenVAS_API
from frameworks.qualys_vm import qualysVulnScan
from frameworks.qualys_was import qualysScanReport
from reporting.jira_api import JiraAPI


class vulnWhispererBase(object):

    CONFIG_SECTION = None
    SEVERITY_NAME_MAPPING = {'none': 0, 'low': 1, 'medium': 2, 'high': 3, 'critical': 4}
    SEVERITY_NUMBER_MAPPING = {0: 'none', 1: 'low', 2: 'medium', 3: 'high', 4: 'critical'}

    def __init__(
            self,
            config=None,
            db_name='report_tracker.db',
            purge=False,
            verbose=False,
            debug=False,
            section=None,
            scan_filter=None,
            days=None,
            develop=False,
        ):

        if self.CONFIG_SECTION is None:
                raise Exception('Implementing class must define CONFIG_SECTION')

        self.exit_code = 0
        self.db_name = db_name
        self.purge = purge
        self.develop = develop
        self.days = days

        if config is not None:
            self.config = vwConfig(config_in=config)
            try:
                self.enabled = self.config.get(self.CONFIG_SECTION, 'enabled')
            except:
                self.enabled = False
            self.hostname = self.config.get(self.CONFIG_SECTION, 'hostname')
            try:
                self.username = self.config.get(self.CONFIG_SECTION, 'username')
                self.password = self.config.get(self.CONFIG_SECTION, 'password')
            except:
                self.username = None
                self.password = None
            try:
                self.scan_filter = self.config.get(self.CONFIG_SECTION, 'scan_filter')
            except:
                self.scan_filter = scan_filter
            self.write_path = self.config.get(self.CONFIG_SECTION, 'write_path')
            self.db_path = self.config.get(self.CONFIG_SECTION, 'db_path')

        self.logger = logging.getLogger('vulnWhispererBase')
        self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)

        # Preference command line argument over config file
        if scan_filter:
            self.scan_filter = scan_filter

        if self.scan_filter:
            self.logger.info('Filtering for scan names matching "{}"'.format(self.scan_filter))
            # self.scan_filter = re.compile(scan_filter)

        if self.days != None:
            self.logger.info('Searching for scans within {} days to {}'.format(self.days, (datetime.now() - timedelta(days=days)).isoformat()))
            # self.days = dp.parse(days)
            # self.logger.info('Searching for scans after {}'.format(self.days))

        if self.db_name is not None:
            if self.db_path:
                self.database = os.path.join(self.db_path,
                                             db_name)
            else:
                self.database = \
                    os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                 'database', db_name))
            if not os.path.exists(self.db_path):
                os.makedirs(self.db_path)
                self.logger.info('Creating directory {dir}'.format(dir=self.db_path))

            if not os.path.exists(self.database):
                with open(self.database, 'w'):
                    self.logger.info('Creating file {dir}'.format(dir=self.database))

            try:
                self.conn = sqlite3.connect(self.database)
                self.cur = self.conn.cursor()
                self.logger.info('Connected to database at {loc}'.format(loc=self.database))
            except Exception as e:
                self.logger.error('Could not connect to database at {loc}\nReason: {e} - Please ensure the path exists'.format(
                        e=e,
                        loc=self.database))
        else:

            self.logger.error('Please specify a database to connect to!')
            exit(1)

        self.table_columns = [
            'scan_name',
            'scan_id',
            'last_modified',
            'filename',
            'download_time',
            'record_count',
            'source',
            'uuid',
            'processed',
            'reported',
        ]

        self.init()
        self.uuids = self.retrieve_uuids()
        self.processed = 0
        self.skipped = 0
        self.scan_list = []

    def create_table(self):
        self.cur.execute(
            'CREATE TABLE IF NOT EXISTS scan_history (id INTEGER PRIMARY KEY,'
            ' scan_name TEXT, scan_id INTEGER, last_modified DATE, filename TEXT,'
            ' download_time DATE, record_count INTEGER, source TEXT,'
            ' uuid TEXT, processed INTEGER, reported INTEGER)'
            )
        self.conn.commit()

    def delete_table(self):
        self.cur.execute('DROP TABLE IF EXISTS scan_history')
        self.conn.commit()

    def init(self):
        if self.purge:
            self.delete_table()
        self.create_table()

    def path_check(self, _data):
        if self.write_path:
            if '/' or '\\' in _data[-1]:
                data = self.write_path + _data
            else:
                data = self.write_path + '/' + _data
        return data

    def record_insert(self, record):
        #for backwards compatibility with older versions without "reported" field

        try:
            #-1 to get the latest column, 1 to get the column name (old version would be "processed", new "reported")
            #TODO delete backward compatibility check after some versions
            last_column_table = self.cur.execute('PRAGMA table_info(scan_history)').fetchall()[-1][1]
            if last_column_table == self.table_columns[-1]:
                self.cur.execute('insert into scan_history({table_columns}) values (?,?,?,?,?,?,?,?,?,?)'.format(
                    table_columns=', '.join(self.table_columns)), record)

            else:
                self.cur.execute('insert into scan_history({table_columns}) values (?,?,?,?,?,?,?,?,?)'.format(
                    table_columns=', '.join(self.table_columns[:-1])), record[:-1])
            self.conn.commit()
        except Exception as e:
            self.logger.error("Failed to insert record in database. Error: {}".format(e))
            sys.exit(1)

    def set_latest_scan_reported(self, filename):
        #the reason to use the filename instead of the source/scan_name is because the filename already belongs to
        #that latest scan, and we maintain integrity making sure that it is the exact scan we checked
        try:
            self.cur.execute('UPDATE scan_history SET reported = 1 WHERE filename="{}";'.format(filename))
            self.conn.commit()
            self.logger.info('Scan {} marked as successfully processed.'.format(filename))
            return True
        except Exception as e:
            self.logger.error('Failed while setting scan with file {} as processed'.format(filename))

        return False

    def retrieve_uuids(self):
        """
        Retrieves UUIDs from database and checks list to determine which files need to be processed.
        :return:
        """
        try:
            self.conn.text_factory = str
            self.cur.execute('SELECT uuid FROM scan_history where source = "{config_section}"'.format(config_section=self.CONFIG_SECTION))
            results = frozenset([r[0] for r in self.cur.fetchall()])
        except:
            results = []
        return results

    def directory_check(self):
        if not os.path.exists(self.write_path):
            os.makedirs(self.write_path)
            self.logger.info('Directory created at {scan} - Skipping creation'.format(
                scan=self.write_path.encode('utf8')))
        else:
            os.path.exists(self.write_path)
            self.logger.info('Directory already exists for {scan} - Skipping creation'.format(
                scan=self.write_path.encode('utf8')))

    def get_latest_results(self, source, scan_name):
        processed = 0
        results = []

        try:
            self.conn.text_factory = str
            self.cur.execute('SELECT filename FROM scan_history WHERE source="{}" AND scan_name="{}" ORDER BY last_modified DESC LIMIT 1;'.format(source, scan_name))
            #should always return just one filename
            results = [r[0] for r in self.cur.fetchall()][0]

            #-1 to get the latest column, 1 to get the column name (old version would be "processed", new "reported")
            #TODO delete backward compatibility check after some versions
            last_column_table = self.cur.execute('PRAGMA table_info(scan_history)').fetchall()[-1][1]
            if results and last_column_table == self.table_columns[-1]:
                reported = self.cur.execute('SELECT reported FROM scan_history WHERE filename="{}"'.format(results)).fetchall()
                reported = reported[0][0]
                if reported:
                    self.logger.debug("Last downloaded scan from source {source} scan_name {scan_name} has already been reported".format(source=source, scan_name=scan_name))

        except Exception as e:
            self.logger.error("Error when getting latest results from {}.{} : {}".format(source, scan_name, e))
        return results, reported

    def get_scan_profiles(self):
        # Returns a list of source.scan_name elements from the database

        # we get the list of sources
        try:
            self.conn.text_factory = str
            self.cur.execute('SELECT DISTINCT source FROM scan_history;')
            sources = [r[0] for r in self.cur.fetchall()]
        except:
            sources = []
            self.logger.error("Process failed at executing 'SELECT DISTINCT source FROM scan_history;'")

        results = []

        # we get the list of scans within each source
        for source in sources:
            scan_names = []
            try:
                self.conn.text_factory = str
                self.cur.execute("SELECT DISTINCT scan_name FROM scan_history WHERE source='{}';".format(source))
                scan_names = [r[0] for r in self.cur.fetchall()]
                for scan in scan_names:
                    results.append('{}.{}'.format(source,scan))
            except:
                scan_names = []

        return results

    def common_normalise(self, df):
        """Map and transform common data values"""
        self.logger.info('Start common normalisation')

        df.replace({'': np.nan}, inplace=True)

        # Map risk name to a risk value
        if 'risk' in df and not 'risk_number' in df:
            self.logger.debug('Mapping risk name to risk number')
            df['risk_number'] = df['risk'].map(self.SEVERITY_NAME_MAPPING)

        # Map risk value to a risk name
        if 'risk_number' in df and not 'risk' in df:
            self.logger.debug('Mapping risk number to risk name')
            df['risk'] = df['risk_number'].map(self.SEVERITY_NUMBER_MAPPING)

        self.logger.debug('Normalising CVSS')
        for cvss_version in ['cvss', 'cvss2', 'cvss3']:
            # cvssX = cvssX_temporal else cvssX_base
            if cvss_version + '_base' in df:
                self.logger.debug('Normalising {} base'.format(cvss_version))
                df[cvss_version] = df[cvss_version + '_base']
                df[cvss_version] = df[cvss_version + '_temporal'].fillna(df[cvss_version])

            # Combine cvssX temporal and base vectors
            if cvss_version + '_vector' in df and cvss_version + '_temporal_vector' in df:
                self.logger.debug('Normalising {} vector'.format(cvss_version))
                df[cvss_version + '_vector'] = (
                    df[[cvss_version + '_vector', cvss_version + '_temporal_vector']]
                    .apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1)
                    .str.rstrip('/nan')
                )
                df.drop(cvss_version + '_temporal_vector', axis=1, inplace=True)

            # Map cvssX to severity name
            if cvss_version in df:
                self.logger.debug('Normalising {} severity'.format(cvss_version))
                df[cvss_version] = df[cvss_version].astype('float')
                df[cvss_version + '_severity'] = 'informational'
                df.loc[(df[cvss_version] > 0) & (df[cvss_version] < 3), cvss_version + '_severity'] = 'low'
                df.loc[(df[cvss_version] >= 3) & (df[cvss_version] < 6), cvss_version + '_severity'] = 'medium'
                df.loc[(df[cvss_version] >= 6) & (df[cvss_version] < 9), cvss_version + '_severity'] = 'high'
                df.loc[df[cvss_version] > 9, cvss_version + '_severity'] = 'critical'

        # Get a single cvss score derived from cvss3 else cvss2
        if not 'cvss' in df:
            if 'cvss3' in df:
                df['cvss'] = df['cvss3'].fillna(df['cvss2'])
                df['cvss_severity'] = df['cvss3_severity'].fillna(df['cvss2_severity'])
            elif 'cvss2' in df:
                df['cvss'] = df['cvss2']
                df['cvss_severity'] = df['cvss2_severity']

        self.logger.debug('Creating Unique Document ID')
        df['_unique'] = df.index.values
        if 'history_id' in df:
            df['_unique'] = df[['scan_id', 'history_id', '_unique']].apply(lambda x: '_'.join(x.astype(str)), axis=1)
        else:
            df['_unique'] = df[['scan_id', '_unique']].apply(lambda x: '_'.join(x.astype(str)), axis=1)

        return df

    def print_available_scans(self, scan_list):
        """
        Takes a list of dicts with fields 'time', 'scan_name', 'imported' and 'status' and prints a table
        """
        output_string = '| {time} | {scan_name} | {imported} | {status} |'
        print '-' * 110
        print output_string.format(time='Time'.ljust(19), scan_name='Scan Name'.ljust(60), imported='Imported'.ljust(8), status='Status'.ljust(10))
        print '-' * 110
        for scan in sorted(scan_list, key=lambda k: k['time'], reverse=True):
            scan['imported'] = scan['imported'].ljust(8)
            scan['scan_name'] = scan['scan_name'][:60].ljust(60).encode('utf-8')
            scan['time'] = scan['time'][:19].ljust(19)
            scan['status'] = scan['status'][:10].ljust(10)
            print output_string.format(**scan)
        print '-' * 110

        return 0


class vulnWhispererNessus(vulnWhispererBase):

    CONFIG_SECTION = None

    def __init__(
            self,
            config=None,
            db_name='report_tracker.db',
            purge=False,
            verbose=False,
            debug=False,
            profile='nessus',
            scan_filter=None,
            days=None,
            list_scans=None,
    ):
        self.CONFIG_SECTION=profile

        super(vulnWhispererNessus, self).__init__(config=config, verbose=verbose, debug=debug, scan_filter=scan_filter, days=days)

        self.logger = logging.getLogger('vulnWhisperer{}'.format(self.CONFIG_SECTION))
        if not verbose:
            verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
        self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)

        self.develop = True
        self.purge = purge
        self.list_scans = list_scans

        try:
            self.nessus_port = self.config.get(self.CONFIG_SECTION, 'port')
            self.nessus_trash = self.config.getbool(self.CONFIG_SECTION, 'trash')

            try:
                self.access_key = self.config.get(self.CONFIG_SECTION,'access_key')
                self.secret_key = self.config.get(self.CONFIG_SECTION,'secret_key')
            except:
                self.access_key = None
                self.secret_key = None

            try:
                self.logger.info('Attempting to connect to {}...'.format(self.CONFIG_SECTION))
                self.nessus = \
                    NessusAPI(hostname=self.hostname,
                                port=self.nessus_port,
                                username=self.username,
                                password=self.password,
                                profile=self.CONFIG_SECTION,
                                access_key=self.access_key,
                                secret_key=self.secret_key,
                                verbose=verbose,
                                )
                self.nessus_connect = True
                self.logger.info('Connected to {} on {host}:{port}'.format(self.CONFIG_SECTION, host=self.hostname,
                                                                                port=str(self.nessus_port)))
            except Exception as e:
                self.logger.error('Exception: {}'.format(str(e)))
                raise Exception(
                    'Could not connect to {} -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format(
                        self.CONFIG_SECTION,
                        config=self.config.config_in,
                        e=e))
        except Exception as e:
            self.logger.error('Could not properly load your config!\nReason: {e}'.format(e=e))
            sys.exit(1)



    def scan_count(self, scans, completed=False):
        """

        :param scans: Pulls in available scans
        :param completed: Only return completed scans
        :return:
        """

        self.logger.info('Gathering all scan data... this may take a while...')
        scan_records = []
        if self.days:
            earliest_time = int((datetime.now() - timedelta(days=self.days)).strftime("%s"))
        for s in scans:
            if s:
                record = {}
                record['scan_id'] = s['id']
                record['scan_name'] = s.get('name', '')
                record['owner'] = s.get('owner', '')
                record['creation_date'] = s.get('creation_date', '')
                record['starttime'] = s.get('starttime', '')
                record['timezone'] = s.get('timezone', '')
                record['folder_id'] = s.get('folder_id', '')
                try:
                    for h in self.nessus.get_scan_history(s['id']):
                        record['uuid'] = h.get('uuid', '')
                        record['status'] = h.get('status', '')
                        record['history_id'] = h.get('history_id', '')
                        record["last_modification_date"] = h.get(
                            "last_modification_date", ""
                        )
                        record["norm_time"] = self.nessus.get_utc_from_local(
                            int(record["last_modification_date"]),
                            local_tz=self.nessus.tz_conv(record["timezone"]),
                        )
                        if self.days and record["norm_time"] < earliest_time:
                            continue
                        scan_records.append(record.copy())
                except Exception as e:
                    # Generates error each time nonetype is encountered.
                    pass

        if completed:
            scan_records = [s for s in scan_records if s['status'] == 'completed']
        return scan_records


    def whisper_nessus(self):
        if not self.nessus_connect:
            self.logger.error('Failed to use scanner at {host}:{port}'.format(host=self.hostname, port=self.nessus_port))
            self.exit_code += 1
            return self.exit_code

        scan_data = self.nessus.get_scans(self.days)
        folders = scan_data['folders']
        scans = scan_data['scans'] if scan_data['scans'] else []
        all_scans = self.scan_count(scans)

        if self.list_scans:
            scan_list = []
            for scan in all_scans:
                scan['imported'] = 'Yes' if scan['uuid'] in self.uuids else 'No'
                scan['time'] = datetime.utcfromtimestamp(scan['norm_time']).isoformat()
                scan_list.append(scan)
            print 'Available {} scans:'.format(self.CONFIG_SECTION)
            self.print_available_scans(scan_list)
            return 0

        if self.uuids:
            scan_list = [
                scan for scan in all_scans
                if scan["uuid"] not in self.uuids
                and scan["status"] in ["completed", "imported"]
            ]
        else:
            scan_list = all_scans
        if self.scan_filter:
            self.logger.info('Filtering scans that match "{}"'.format(self.scan_filter))
            scan_list = [
                x for x in scan_list
                if re.match(self.scan_filter, x["scan_name"], re.IGNORECASE)
            ]
        self.logger.info(
            "Identified {new} scans to be processed".format(new=len(scan_list))
        )

        if not scan_list:
            self.logger.warn("No new scans to process. Exiting...")
            return self.exit_code

        # Create scan subfolders
        for f in folders:
            if not os.path.exists(self.path_check(f['name'])):
                if f['name'] == 'Trash' and self.nessus_trash:
                    os.makedirs(self.path_check(f['name']))
                elif f['name'] != 'Trash':
                    os.makedirs(self.path_check(f['name']))
            else:
                os.path.exists(self.path_check(f['name']))
                self.logger.info('Directory already exists for {scan} - Skipping creation'.format(
                    scan=self.path_check(f['name']).encode('utf8')))

        scan_count = 0

        # TODO Rewrite this part to go through the scans that have aleady been processed

        for s in scan_list:
            scan_count += 1
            (
                scan_name,
                scan_id,
                history_id,
                norm_time,
                status,
                uuid,
            ) = (
                s['scan_name'],
                s['scan_id'],
                s['history_id'],
                s['norm_time'],
                s['status'],
                s['uuid'],
            )

            # TODO Create directory sync function which scans the directory for files that exist already and populates the database

            folder_id = s['folder_id']
            if self.CONFIG_SECTION == 'tenable':
                folder_name = ''
            else:
                folder_name = next(f['name'] for f in folders if f['id'] == folder_id)
            if status in ['completed', 'imported']:
                file_name = '%s_%s_%s_%s.%s' % (scan_name, scan_id,
                                                history_id, norm_time, 'json')
                repls = (('\\', '_'), ('/', '_'), (' ', '_'))
                file_name = reduce(lambda a, kv: a.replace(*kv), repls, file_name)
                relative_path_name = self.path_check(folder_name + '/' + file_name).encode('utf8')

                if os.path.isfile(relative_path_name):
                    if self.develop:
                        csv_in = pd.read_json(relative_path_name, lines=True)
                        record_meta = (
                            scan_name,
                            scan_id,
                            norm_time,
                            file_name,
                            time.time(),
                            csv_in.shape[0],
                            self.CONFIG_SECTION,
                            uuid,
                            1,
                            0,
                        )
                        self.record_insert(record_meta)
                        self.logger.info('File {filename} already exists! Updating database'.format(filename=relative_path_name))
                else:
                    try:
                        file_req = \
                            self.nessus.download_scan(scan_id=scan_id, history=history_id,
                                                    export_format='csv')
                    except Exception as e:
                        self.logger.error('Could not download {} scan {}: {}'.format(self.CONFIG_SECTION, scan_id, str(e)))
                        self.exit_code += 1
                        continue

                    self.logger.info('Processing {}/{} for scan: {}'.format(scan_count, len(scan_list), scan_name.encode('utf8')))
                    vuln_ready = pd.read_csv(io.StringIO(file_req.decode('utf-8')))

                    # Map and transform fields
                    vuln_ready = self.nessus.normalise(vuln_ready)

                    # Set common fields
                    vuln_ready['history_id'] = history_id
                    vuln_ready['scan_id'] = uuid
                    vuln_ready['scan_name'] = scan_name.encode('utf8')
                    vuln_ready['scan_source'] = self.CONFIG_SECTION
                    vuln_ready['scan_time'] = norm_time
                    vuln_ready['vendor'] = 'Tenable'

                    vuln_ready = self.common_normalise(vuln_ready)

                    vuln_ready.to_json(relative_path_name + '.tmp', orient='records', lines=True)
                    os.rename(relative_path_name + '.tmp', relative_path_name)
                    self.logger.info('{records} records written to {path} '.format(records=vuln_ready.shape[0],
                                                                                   path=relative_path_name))

                    record_meta = (
                        scan_name,
                        scan_id,
                        norm_time,
                        file_name,
                        time.time(),
                        vuln_ready.shape[0],
                        self.CONFIG_SECTION,
                        uuid,
                        1,
                        0,
                    )
                    self.record_insert(record_meta)
                    self.logger.info('Scan {} ({}) written to database'.format(scan_name.encode('utf8'), uuid))

        self.conn.close()
        self.logger.info('Scan aggregation complete! Connection to database closed.')

        return self.exit_code


class vulnWhispererQualysWAS(vulnWhispererBase):

    CONFIG_SECTION = 'qualys_was'
    def __init__(
            self,
            config=None,
            db_name='report_tracker.db',
            purge=False,
            verbose=False,
            debug=False,
            scan_filter=None,
            days=None,
            list_scans=None,
        ):

        super(vulnWhispererQualysWAS, self).__init__(config=config, verbose=verbose, debug=debug, scan_filter=scan_filter, days=days)
        self.logger = logging.getLogger('vulnWhispererQualysWAS')
        if not verbose:
            verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
        self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)

        self.qualys_scan = qualysScanReport(config=config)
        self.latest_scans = self.qualys_scan.qw.get_all_scans(days=self.days)
        self.directory_check()
        self.scans_to_process = None
        self.list_scans = list_scans

    def whisper_reports(self,
                        report_id=None,
                        launched_date=None,
                        scan_name=None,
                        scan_reference=None,
                        output_format='json',
                        cleanup=True):
        """
        report_id: App ID
        updated_date: Last time scan was ran for app_id
        """
        vuln_ready = None

        try:
            if 'Z' in launched_date:
                launched_date = self.qualys_scan.utils.iso_to_epoch(launched_date)
            report_name = 'qualys_was_' + str(report_id) \
                          + '_{last_updated}'.format(last_updated=launched_date) \
                          + '.{extension}'.format(extension=output_format)

            relative_path_name = self.path_check(report_name).encode('utf8')

            if os.path.isfile(relative_path_name):
                #TODO Possibly make this optional to sync directories
                file_length = len(open(relative_path_name).readlines())
                record_meta = (
                    scan_name,
                    scan_reference,
                    launched_date,
                    report_name,
                    time.time(),
                    file_length,
                    self.CONFIG_SECTION,
                    report_id,
                    1,
                    0,
                )
                self.record_insert(record_meta)
                self.logger.info('File {filename} already exists! Updating database'.format(filename=relative_path_name))

            else:
                self.logger.info('Generating report for {}'.format(report_id))
                status = self.qualys_scan.qw.create_report(report_id)
                root = objectify.fromstring(status)
                if root.responseCode == 'SUCCESS':
                    self.logger.info('Successfully generated report! ID: {}'.format(report_id))
                    generated_report_id = root.data.Report.id
                    self.logger.info('New Report ID: {}'.format(generated_report_id))

                    vuln_ready = self.qualys_scan.process_data(path=self.write_path, file_id=str(generated_report_id))
                    # Map and transform fields
                    vuln_ready = self.qualys_scan.normalise(vuln_ready)

                    # Set common fields
                    vuln_ready['app_id'] = report_id
                    vuln_ready['scan_id'] = scan_reference
                    vuln_ready['scan_name'] = scan_name.encode('utf8')
                    vuln_ready['scan_source'] = self.CONFIG_SECTION
                    vuln_ready['scan_time'] = launched_date
                    vuln_ready['vendor'] = 'Qualys'

                    vuln_ready = self.common_normalise(vuln_ready)

                    if output_format == 'json':
                        vuln_ready.to_json(relative_path_name + '.tmp', orient='records', lines=True)
                    elif output_format == 'csv':
                        vuln_ready.to_csv(relative_path_name + '.tmp', index=False, header=True)
                    os.rename(relative_path_name + '.tmp', relative_path_name)

                    self.logger.info('{records} records written to {path} '.format(records=vuln_ready.shape[0],
                                                                                   path=relative_path_name))

                    record_meta = (
                        scan_name,
                        scan_reference,
                        launched_date,
                        report_name,
                        time.time(),
                        vuln_ready.shape[0],
                        self.CONFIG_SECTION,
                        report_id,
                        1,
                        0,
                    )
                    self.record_insert(record_meta)
                    self.logger.info('Scan {} ({}) written to database'.format(scan_name.encode('utf8'), report_id))

                    if cleanup:
                        self.logger.info('Removing report {} from Qualys Database'.format(generated_report_id))
                        cleaning_up = self.qualys_scan.qw.delete_report(generated_report_id)
                        os.remove(self.path_check(str(generated_report_id) + '.csv'))
                        self.logger.info('Deleted report from local disk: {}'.format(self.path_check(str(generated_report_id))))
                else:
                    self.logger.error('Could not process report ID: {}'.format(status))

        except Exception as e:
            self.logger.error('Could not process {}: {}'.format(report_id, str(e)))
        return vuln_ready


    def identify_scans_to_process(self):
        if self.uuids:
            self.scans_to_process = self.latest_scans[~self.latest_scans['id'].isin(self.uuids)]
        else:
            self.scans_to_process = self.latest_scans
        self.logger.info('Identified {new} scans to be processed'.format(new=len(self.scans_to_process)))


    def process_web_assets(self):
        counter = 0

        if self.scan_filter:
            self.logger.info('Filtering scans that match "{}"'.format(self.scan_filter))
            self.latest_scans = self.latest_scans.loc[
                self.latest_scans["name"].str.contains(self.scan_filter, case=False)
            ]

        if self.list_scans:
            if self.uuids:
                self.latest_scans.loc[self.latest_scans['id'].isin(self.uuids), 'imported'] = 'Yes'
            else:
                self.latest_scans['imported'] = 'No'
            self.latest_scans['imported'].fillna('No', inplace=True)
            self.latest_scans.rename(columns={'launchedDate': 'time', 'name': 'scan_name'}, inplace=True)
            print 'Available {} scans:'.format(self.CONFIG_SECTION)
            self.print_available_scans(self.latest_scans[['time', 'scan_name', 'imported', 'status']].to_dict(orient='records'))
            return 0

        self.identify_scans_to_process()
        if self.scans_to_process.shape[0]:
            for app in self.scans_to_process.iterrows():
                counter += 1
                r = app[1]
                self.logger.info('Processing {}/{}'.format(counter, len(self.scans_to_process)))
                self.whisper_reports(report_id=r['id'],
                                     launched_date=r['launchedDate'],
                                     scan_name=r['name'],
                                     scan_reference=r['reference'])
        else:
            self.logger.info('No new scans to process. Exiting...')
        self.conn.close()
        return self.exit_code


class vulnWhispererOpenVAS(vulnWhispererBase):
    CONFIG_SECTION = 'openvas'

    def __init__(
            self,
            config=None,
            db_name='report_tracker.db',
            purge=False,
            verbose=False,
            debug=False,
            scan_filter=None,
            days=None,
            list_scans=None,
    ):
        super(vulnWhispererOpenVAS, self).__init__(config=config, verbose=verbose, debug=debug, scan_filter=scan_filter, days=days)
        self.logger = logging.getLogger('vulnWhispererOpenVAS')
        if not verbose:
            verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
        self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)

        self.directory_check()
        self.port = int(self.config.get(self.CONFIG_SECTION, 'port'))
        self.develop = True
        self.purge = purge
        self.scans_to_process = None
        self.list_scans = list_scans
        self.openvas_api = OpenVAS_API(hostname=self.hostname,
                                       port=self.port,
                                       username=self.username,
                                       password=self.password)

    def whisper_reports(self, output_format='json', launched_date=None, report_id=None, cleanup=True):
        report = None
        if report_id:
            self.logger.info('Processing report ID: {}'.format(report_id))


            scan_name = report_id.replace('-', '')
            report_name = 'openvas_scan_{scan_name}_{last_updated}.{extension}'.format(scan_name=scan_name,
                                                                                       last_updated=launched_date,
                                                                                       extension=output_format)
            relative_path_name = self.path_check(report_name).encode('utf8')
            scan_reference = report_id

            if os.path.isfile(relative_path_name):
                # TODO Possibly make this optional to sync directories
                file_length = len(open(relative_path_name).readlines())
                record_meta = (
                    scan_name,
                    scan_reference,
                    launched_date,
                    report_name,
                    time.time(),
                    file_length,
                    self.CONFIG_SECTION,
                    report_id,
                    1,
                    0,
                )
                self.record_insert(record_meta)
                self.logger.info('File {filename} already exists! Updating database'.format(filename=relative_path_name))


            else:
                vuln_ready = self.openvas_api.process_report(report_id=report_id)
                # Map and transform fields
                vuln_ready = self.openvas_api.normalise(vuln_ready)

                # Set common fields
                vuln_ready['scan_id'] = report_id
                vuln_ready['scan_time'] = launched_date
                vuln_ready['scan_source'] = self.CONFIG_SECTION
                vuln_ready['vendor'] = 'Greenbone'

                vuln_ready = self.common_normalise(vuln_ready)

                vuln_ready.to_json(relative_path_name + '.tmp', orient='records', lines=True)
                os.rename(relative_path_name + '.tmp', relative_path_name)
                self.logger.info('{records} records written to {path} '.format(records=vuln_ready.shape[0],
                                                                               path=relative_path_name))

                record_meta = (
                    scan_name,
                    scan_reference,
                    launched_date,
                    report_name,
                    time.time(),
                    vuln_ready.shape[0],
                    self.CONFIG_SECTION,
                    report_id,
                    1,
                    0,
                )
                self.record_insert(record_meta)
                self.logger.info('Scan {} ({}) written to database'.format(scan_name.encode('utf8'), report_id))

        return report

    def identify_scans_to_process(self):
        if self.uuids:
            self.scans_to_process = self.openvas_api.openvas_reports[
                ~self.openvas_api.openvas_reports.report_ids.isin(self.uuids)]
        else:
            self.scans_to_process = self.openvas_api.openvas_reports
        self.logger.info('Identified {new} scans to be processed'.format(new=len(self.scans_to_process)))

    def process_openvas_scans(self):
        counter = 0
        if self.list_scans:
            self.scans_to_process = self.openvas_api.openvas_reports
            if self.uuids:
                self.scans_to_process.loc[self.scans_to_process['report_ids'].isin(self.uuids), 'imported'] = 'Yes'
            else:
                self.scans_to_process['imported'] = 'No'
            self.scans_to_process['imported'].fillna('No', inplace=True)
            self.scans_to_process['time'] = pd.to_datetime(self.scans_to_process['epoch'], unit='s').astype(str)
            self.scans_to_process.rename(columns={'task': 'scan_name'}, inplace=True)
            print 'Available {} scans:'.format(self.CONFIG_SECTION)

            self.print_available_scans(self.scans_to_process[['time', 'scan_name', 'imported', 'status']].to_dict(orient='records'))
            return 0

        self.identify_scans_to_process()
        if self.scans_to_process.shape[0]:
            for scan in self.scans_to_process.iterrows():
                counter += 1
                info = scan[1]
                self.logger.info('Processing {}/{} - Report ID: {}'.format(counter, len(self.scans_to_process), info['report_ids']))
                self.whisper_reports(report_id=info['report_ids'],
                                     launched_date=info['epoch'])
            self.logger.info('Processing complete')
        else:
            self.logger.info('No new scans to process. Exiting...')
        self.conn.close()
        return self.exit_code


class vulnWhispererQualysVM(vulnWhispererBase):

    CONFIG_SECTION = 'qualys_vm'

    def __init__(
            self,
            config=None,
            db_name='report_tracker.db',
            purge=False,
            verbose=False,
            debug=False,
            scan_filter=None,
            days=None,
            list_scans=None,
        ):

        super(vulnWhispererQualysVM, self).__init__(config=config, verbose=verbose, debug=debug, scan_filter=scan_filter, days=days)
        self.logger = logging.getLogger('vulnWhispererQualysVM')
        if not verbose:
            verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
        self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)

        self.qualys_scan = qualysVulnScan(config=config)
        self.directory_check()
        self.scans_to_process = None
        self.list_scans = list_scans
        self.latest_scans = self.qualys_scan.qw.get_all_scans(days=self.days)

    def whisper_reports(self,
                        report_id=None,
                        launched_date=None,
                        scan_name=None,
                        scan_reference=None,
                        output_format='json',
                        cleanup=True):
            if 'Z' in launched_date:
                launched_date = self.qualys_scan.utils.iso_to_epoch(launched_date)
            report_name = 'qualys_vm_' + report_id.replace('/','_') \
                          + '_{last_updated}'.format(last_updated=launched_date) \
                          + '.{extension}'.format(extension=output_format)

            relative_path_name = self.path_check(report_name).encode('utf8')

            if os.path.isfile(relative_path_name):
                #TODO Possibly make this optional to sync directories
                file_length = len(open(relative_path_name).readlines())
                record_meta = (
                    scan_name,
                    scan_reference,
                    launched_date,
                    report_name,
                    time.time(),
                    file_length,
                    self.CONFIG_SECTION,
                    report_id,
                    1,
                    0,
                )
                self.record_insert(record_meta)
                self.logger.info('File {filename} already exists! Updating database'.format(filename=relative_path_name))

            else:
                try:
                    self.logger.info('Processing {} ({})'.format(scan_name.encode('utf8'), report_id))
                    vuln_ready = self.qualys_scan.process_data(scan_id=report_id)

                    if len(vuln_ready) != 0:
                        # Map and transform fields
                        vuln_ready = self.qualys_scan.normalise(vuln_ready)

                        # Set common fields
                        vuln_ready['scan_name'] = scan_name.encode('utf8')
                        vuln_ready['scan_id'] = report_id
                        vuln_ready['scan_time'] = launched_date
                        vuln_ready['scan_source'] = self.CONFIG_SECTION
                        vuln_ready['vendor'] = 'Qualys'

                        vuln_ready = self.common_normalise(vuln_ready)

                except Exception as e:
                    self.logger.error('Could not process {}: {}'.format(report_id, str(e)))
                    self.exit_code += 1
                    return self.exit_code

                if len(vuln_ready) != 0:
                    if output_format == 'json':
                        vuln_ready.to_json(relative_path_name + '.tmp', orient='records', lines=True)
                    elif output_format == 'csv':
                        vuln_ready.to_csv(relative_path_name + '.tmp', index=False, header=True)
                    os.rename(relative_path_name + '.tmp', relative_path_name)
                    self.logger.info('{records} records written to {path} '.format(records=vuln_ready.shape[0],
                                                                                path=relative_path_name))

                record_meta = (
                    scan_name,
                    scan_reference,
                    launched_date,
                    report_name,
                    time.time(),
                    vuln_ready.shape[0],
                    self.CONFIG_SECTION,
                    report_id,
                    1,
                    0,
                )
                self.record_insert(record_meta)
                self.logger.info('Scan {} ({}) written to database'.format(scan_name.encode('utf8'), report_id))

            return self.exit_code

    def identify_scans_to_process(self):
        if self.uuids:
            self.scans_to_process = self.latest_scans.loc[
                (~self.latest_scans['id'].isin(self.uuids))
                & (self.latest_scans['status'] == 'Finished')]
        else:
            self.scans_to_process = self.latest_scans
        self.logger.info('Identified {new} scans to be processed'.format(new=len(self.scans_to_process)))


    def process_vuln_scans(self):
        counter = 0
        if self.scan_filter:
            self.logger.info('Filtering scans that match "{}"'.format(self.scan_filter))
            self.latest_scans = self.latest_scans.loc[
                self.latest_scans["name"].str.contains(self.scan_filter, case=False)
            ]

        if self.list_scans:
            if self.uuids:
                self.latest_scans.loc[self.latest_scans['id'].isin(self.uuids), 'imported'] = 'Yes'
            else:
                self.latest_scans['imported'] = 'No'
            self.latest_scans['imported'].fillna('No', inplace=True)
            self.latest_scans.rename(columns={'date': 'time', 'name': 'scan_name'}, inplace=True)
            print 'Available {} scans:'.format(self.CONFIG_SECTION)
            self.print_available_scans(self.latest_scans[['time', 'scan_name', 'imported', 'status']].to_dict(orient='records'))
            return 0

        self.identify_scans_to_process()
        if self.scans_to_process.shape[0]:
            for app in self.scans_to_process.iterrows():
                counter += 1
                r = app[1]
                self.logger.info('Processing {}/{}'.format(counter, len(self.scans_to_process)))
                self.exit_code += self.whisper_reports(report_id=r['id'],
                                     launched_date=r['date'],
                                     scan_name=r['name'],
                                     scan_reference=r['type'])
        else:
            self.logger.info('No new scans to process. Exiting...')
        self.conn.close()
        return self.exit_code


class vulnWhispererJIRA(vulnWhispererBase):

    CONFIG_SECTION = 'jira'

    def __init__(
            self,
            config=None,
            db_name='report_tracker.db',
            purge=False,
            verbose=False,
            debug=False,
        ):
        super(vulnWhispererJIRA, self).__init__(config=config, verbose=verbose, debug=debug)

        self.logger = logging.getLogger('vulnWhispererJira')
        if not verbose:
            verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
        self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)

        self.host_resolv_cache = {}
        self.directory_check()

        if config is not None:
            try:
                self.logger.info('Attempting to connect to jira...')
                self.jira = \
                    JiraAPI(hostname=self.hostname,
                              username=self.username,
                              password=self.password,
                              path=self.config.get('jira','write_path'))
                self.jira_connect = True
                self.logger.info('Connected to jira on {host}'.format(host=self.hostname))
            except Exception as e:
                self.logger.error('Exception: {}'.format(str(e)))
                raise Exception(
                    'Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format(
                        config=self.config.config_in, e=e))
                sys.exit(1)

        profiles = []
        profiles = self.get_scan_profiles()

        if not self.config.exists_jira_profiles(profiles):
            self.config.update_jira_profiles(profiles)
            self.logger.info("Jira profiles have been created in {config}, please fill the variables before rerunning the module.".format(config=self.config_path))
            sys.exit(0)


    def get_env_variables(self, source, scan_name):
        # function returns an array with [jira_project, jira_components, datafile_path]

        #Jira variables
        jira_section = self.config.normalize_section("{}.{}".format(source,scan_name))

        project = self.config.get(jira_section,'jira_project')
        if project == "":
            self.logger.error('JIRA project is missing on the configuration file!')
            sys.exit(0)

        # check that project actually exists
        if not self.jira.project_exists(project):
            self.logger.error("JIRA project '{project}' doesn't exist!".format(project=project))
            sys.exit(0)

        components = self.config.get(jira_section,'components').split(',')

        #cleaning empty array from ''
        if not components[0]:
            components = []

        min_critical = self.config.get(jira_section,'min_critical_to_report')
        if not min_critical:
            self.logger.error('"min_critical_to_report" variable on config file is empty.')
            sys.exit(0)

        #datafile path
        filename, reported = self.get_latest_results(source, scan_name)
        fullpath = ""

        # search data files under user specified directory
        for root, dirnames, filenames in os.walk(vwConfig(self.config_path).get(source,'write_path')):
            if filename in filenames:
                fullpath = "{}/{}".format(root,filename)

        if reported:
            self.logger.warn('Last Scan of "{scan_name}" for source "{source}" has already been reported; will be skipped.'.format(scan_name=scan_name, source=source))
            return [False] * 5

        if not fullpath:
            self.logger.error('Scan of "{scan_name}" for source "{source}" has not been found. Please check that the scanner data files are in place.'.format(scan_name=scan_name, source=source))
            sys.exit(1)

        dns_resolv = self.config.get('jira','dns_resolv')
        if dns_resolv in ('False', 'false', ''):
            dns_resolv = False
        elif dns_resolv in ('True', 'true'):
            dns_resolv = True
        else:
            self.logger.error("dns_resolv variable not setup in [jira] section; will not do dns resolution")
            dns_resolv = False

        return project, components, fullpath, min_critical, dns_resolv


    def parse_nessus_vulnerabilities(self, fullpath, source, scan_name, min_critical):

        vulnerabilities = []

        # we need to parse the CSV
        risks = ['none', 'low', 'medium', 'high', 'critical']
        min_risk = int([i for i,x in enumerate(risks) if x == min_critical][0])

        df = pd.read_csv(fullpath, delimiter=',')

        #nessus fields we want - ['Host','Protocol','Port', 'Name', 'Synopsis', 'Description', 'Solution', 'See Also']
        for index in range(len(df)):
            # filtering vulnerabilities by criticality, discarding low risk
            to_report = int([i for i,x in enumerate(risks) if x == df.loc[index]['Risk'].lower()][0])
            if to_report < min_risk:
                continue

            if not vulnerabilities or df.loc[index]['Name'] not in [entry['title'] for entry in vulnerabilities]:
                vuln = {}
                #vulnerabilities should have all the info for creating all JIRA labels
                vuln['source'] = source
                vuln['scan_name'] = scan_name
                #vulnerability variables
                vuln['title'] = df.loc[index]['Name']
                vuln['diagnosis'] =  df.loc[index]['Synopsis'].replace('\\n',' ')
                vuln['consequence'] = df.loc[index]['Description'].replace('\\n',' ')
                vuln['solution'] = df.loc[index]['Solution'].replace('\\n',' ')
                vuln['ips'] = []
                vuln['ips'].append("{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], df.loc[index]['Port']))
                vuln['risk'] = df.loc[index]['Risk'].lower()

                # Nessus "nan" value gets automatically casted to float by python
                if not (type(df.loc[index]['See Also']) is float):
                    vuln['references'] = df.loc[index]['See Also'].split("\\n")
                else:
                    vuln['references'] = []
                vulnerabilities.append(vuln)

            else:
                # grouping assets by vulnerability to open on single ticket, as each asset has its own nessus entry
                for vuln in vulnerabilities:
                    if vuln['title'] == df.loc[index]['Name']:
                        vuln['ips'].append("{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], df.loc[index]['Port']))

        return vulnerabilities

    def parse_qualys_vm_vulnerabilities(self, fullpath, source, scan_name, min_critical, dns_resolv = False):
        #parsing of the qualys vulnerabilities schema
        #parse json
        vulnerabilities = []

        risks = ['info', 'low', 'medium', 'high', 'critical']
        # +1 as array is 0-4, but score is 1-5
        min_risk = int([i for i,x in enumerate(risks) if x == min_critical][0])+1

        try:
            data=[json.loads(line) for line in open(fullpath).readlines()]
        except Exception as e:
            self.logger.warn("Scan has no vulnerabilities, skipping.")
            return vulnerabilities

        #qualys fields we want - []
        for index in range(len(data)):
            if int(data[index]['risk']) < min_risk:
                continue

            elif data[index]['type'] == 'Practice' or data[index]['type'] == 'Ig':
                self.logger.debug("Vulnerability '{vuln}' ignored, as it is 'Practice/Potential', not verified.".format(vuln=data[index]['signature']))
                continue

            if not vulnerabilities or data[index]['signature'] not in [entry['title'] for entry in vulnerabilities]:
                vuln = {}
                #vulnerabilities should have all the info for creating all JIRA labels
                vuln['source'] = source
                vuln['scan_name'] = scan_name
                #vulnerability variables
                vuln['title'] = data[index]['signature']
                vuln['diagnosis'] =  data[index]['threat'].replace('\\n',' ')
                vuln['consequence'] = data[index]['impact'].replace('\\n',' ')
                vuln['solution'] = data[index]['solution'].replace('\\n',' ')
                vuln['ips'] = []
                #TODO ADDED DNS RESOLUTION FROM QUALYS! \n SEPARATORS INSTEAD OF \\n!

                vuln['ips'].append("{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv)))

                #different risk system than Nessus!
                vuln['risk'] = risks[int(data[index]['risk'])-1]

                # Nessus "nan" value gets automatically casted to float by python
                if not (type(data[index]['vendor_reference']) is float or data[index]['vendor_reference'] == None):
                    vuln['references'] = data[index]['vendor_reference'].split("\\n")
                else:
                    vuln['references'] = []
                vulnerabilities.append(vuln)
            else:
                # grouping assets by vulnerability to open on single ticket, as each asset has its own nessus entry
                for vuln in vulnerabilities:
                    if vuln['title'] == data[index]['signature']:
                        vuln['ips'].append("{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv)))

        return vulnerabilities

    def get_asset_fields(self, vuln, dns_resolv):
        values = {}
        values['ip'] = vuln['ip']
        values['protocol'] = vuln['protocol']
        values['port'] = vuln['port']
        values['dns'] = ''
        if dns_resolv:
            if vuln['dns']:
                values['dns'] = vuln['dns']
            else:
                if values['ip'] in self.host_resolv_cache.keys():
                    self.logger.debug("Hostname from {ip} cached, retrieving from cache.".format(ip=values['ip']))
                    values['dns'] = self.host_resolv_cache[values['ip']]
                else:
                    self.logger.debug("No hostname, trying to resolve {ip}'s  hostname.".format(ip=values['ip']))
                    try:
                        values['dns'] = socket.gethostbyaddr(vuln['ip'])[0]
                        self.host_resolv_cache[values['ip']] = values['dns']
                        self.logger.debug("Hostname found: {hostname}.".format(hostname=values['dns']))
                    except:
                        self.host_resolv_cache[values['ip']] = ''
                        self.logger.debug("Hostname not found for: {ip}.".format(ip=values['ip']))

        for key in values.keys():
            if not values[key]:
                values[key] = 'N/A'

        return values

    def parse_vulnerabilities(self, fullpath, source, scan_name, min_critical):
        #TODO: SINGLE LOCAL SAVE FORMAT FOR ALL SCANNERS
        #JIRA standard vuln format - ['source', 'scan_name', 'title', 'diagnosis', 'consequence', 'solution', 'ips', 'references']

        return 0


    def jira_sync(self, source, scan_name):
        self.logger.info("Jira Sync triggered for source '{source}' and scan '{scan_name}'".format(source=source, scan_name=scan_name))

        project, components, fullpath, min_critical, dns_resolv = self.get_env_variables(source, scan_name)

        if not project:
            self.logger.debug("Skipping scan for source '{source}' and scan '{scan_name}': vulnerabilities have already been reported.".format(source=source, scan_name=scan_name))
            return False

        vulnerabilities = []

        #***Nessus parsing***
        if source == "nessus":
            vulnerabilities = self.parse_nessus_vulnerabilities(fullpath, source, scan_name, min_critical)

        #***Qualys VM parsing***
        if source == "qualys_vm":
            vulnerabilities = self.parse_qualys_vm_vulnerabilities(fullpath, source, scan_name, min_critical, dns_resolv)

        #***JIRA sync***
        if vulnerabilities:
            self.logger.info('{source} data has been successfuly parsed'.format(source=source.upper()))
            self.logger.info('Starting JIRA sync')

            self.jira.sync(vulnerabilities, project, components)
        else:
            self.logger.info("[{source}.{scan_name}] No vulnerabilities or vulnerabilities not parsed.".format(source=source, scan_name=scan_name))
            self.set_latest_scan_reported(fullpath.split("/")[-1])
            return False

        self.set_latest_scan_reported(fullpath.split("/")[-1])
        return True

    def sync_all(self):
        autoreport_sections = self.config.get_sections_with_attribute('autoreport')

        if autoreport_sections:
            for scan in autoreport_sections:
                self.jira_sync(self.config.get(scan, 'source'), self.config.get(scan, 'scan_name'))
            return True
        return False

class vulnWhisperer(object):

    def __init__(self,
                 profile=None,
                 verbose=False,
                 debug=False,
                 config=None,
                 source=None,
                 scan_filter=None,
                 days=None,
                 scanname=None,
                 list_scans=None):

        self.logger = logging.getLogger('vulnWhisperer')
        self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)
        self.profile = profile
        self.verbose = verbose
        self.debug = debug
        self.config = config
        self.source = source
        self.scan_filter = scan_filter
        self.list_scans = list_scans
        self.days = days
        self.scanname = scanname
        self.exit_code = 0

    def whisper_vulnerabilities(self):
        self.logger.setLevel(logging.INFO)
        self.logger.info('Running {} framwork'.format(self.profile))
        if self.profile == 'nessus':
            vw = vulnWhispererNessus(config=self.config,
                                     profile=self.profile,
                                     scan_filter=self.scan_filter,
                                     days=self.days,
                                     verbose=self.verbose,
                                     debug=self.debug,
                                     list_scans=self.list_scans)
            self.exit_code += vw.whisper_nessus()

        elif self.profile == 'qualys_was':
            vw = vulnWhispererQualysWAS(config=self.config,
                                        scan_filter=self.scan_filter,
                                        days=self.days,
                                        verbose=self.verbose,
                                        debug=self.debug,
                                        list_scans=self.list_scans)
            self.exit_code += vw.process_web_assets()

        elif self.profile == 'openvas':
            vw_openvas = vulnWhispererOpenVAS(config=self.config,
                                              scan_filter=self.scan_filter,
                                              days=self.days,
                                              verbose=self.verbose,
                                              debug=self.debug,
                                              list_scans=self.list_scans)
            self.exit_code += vw_openvas.process_openvas_scans()

        elif self.profile == 'tenable':
            vw = vulnWhispererNessus(config=self.config,
                                     profile=self.profile,
                                     scan_filter=self.scan_filter,
                                     days=self.days,
                                     verbose=self.verbose,
                                     debug=self.debug,
                                     list_scans=self.list_scans)
            self.exit_code += vw.whisper_nessus()

        elif self.profile == 'qualys_vm':
            vw = vulnWhispererQualysVM(config=self.config,
                                       scan_filter=self.scan_filter,
                                       days=self.days,
                                       verbose=self.verbose,
                                       debug=self.debug,
                                       list_scans=self.list_scans)
            self.exit_code += vw.process_vuln_scans()

        elif self.profile == 'jira':
            #first we check config fields are created, otherwise we create them
            vw = vulnWhispererJIRA(config=self.config,
                                   verbose=self.verbose,
                                   debug=self.debug)
            if not (self.source and self.scanname):
                self.logger.info('No source/scan_name selected, all enabled scans will be synced')
                success = vw.sync_all()
                if not success:
                    self.logger.error('All scans sync failed!')
                    self.logger.error('Source scanner and scan name needed!')
                    return 0
            else:
                vw.jira_sync(self.source, self.scanname)

        return self.exit_code