VulnWhisperer/vulnwhisp/vulnwhisp.py

#!/usr/bin/python
# -*- coding: utf-8 -*-
__author__ = 'Austin Taylor'

from base.config import vwConfig
from frameworks.nessus import NessusAPI
from frameworks.qualys_web import qualysScanReport
from frameworks.qualys_vuln import qualysVulnScan
from frameworks.openvas import OpenVAS_API
from reporting.jira_api import JiraAPI
import pandas as pd
from lxml import objectify
import sys
import os
import io
import time
import sqlite3
import json
import logging
import socket


class vulnWhispererBase(object):

    CONFIG_SECTION = None

    def __init__(
            self,
            config=None,
            db_name='report_tracker.db',
            purge=False,
            verbose=None,
            debug=False,
            username=None,
            password=None,
            section=None,
            develop=False,
        ):
        self.logger = logging.getLogger('vulnWhispererBase')
        if debug:
            self.logger.setLevel(logging.DEBUG)

        if self.CONFIG_SECTION is None:
                raise Exception('Implementing class must define CONFIG_SECTION')

        self.exit_code = 0
        self.db_name = db_name
        self.purge = purge
        self.develop = develop

        if config is not None:
            self.config = vwConfig(config_in=config)
            try:
                self.enabled = self.config.get(self.CONFIG_SECTION, 'enabled')
            except:
                self.enabled = False
            self.hostname = self.config.get(self.CONFIG_SECTION, 'hostname')
            self.username = self.config.get(self.CONFIG_SECTION, 'username')
            self.password = self.config.get(self.CONFIG_SECTION, 'password')
            self.write_path = self.config.get(self.CONFIG_SECTION, 'write_path')
            self.db_path = self.config.get(self.CONFIG_SECTION, 'db_path')
            self.verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')



        if self.db_name is not None:
            if self.db_path:
                self.database = os.path.join(self.db_path,
                                             db_name)
            else:
                self.database = \
                    os.path.abspath(os.path.join(os.path.dirname(__file__),
                                                 'database', db_name))
            if not os.path.exists(self.db_path):
                os.makedirs(self.db_path)
                self.logger.info('Creating directory {dir}'.format(dir=self.db_path))

            if not os.path.exists(self.database):
                with open(self.database, 'w'):
                    self.logger.info('Creating file {dir}'.format(dir=self.database))

            try:
                self.conn = sqlite3.connect(self.database)
                self.cur = self.conn.cursor()
                self.logger.info('Connected to database at {loc}'.format(loc=self.database))
            except Exception as e:
                self.logger.error('Could not connect to database at {loc}\nReason: {e} - Please ensure the path exist'.format(
                        e=e,
                        loc=self.database))
        else:

            self.logger.error('Please specify a database to connect to!')
            exit(1)

        self.table_columns = [
            'scan_name',
            'scan_id',
            'last_modified',
            'filename',
            'download_time',
            'record_count',
            'source',
            'uuid',
            'processed',
            'reported',
        ]

        self.init()
        self.uuids = self.retrieve_uuids()
        self.processed = 0
        self.skipped = 0
        self.scan_list = []

    def create_table(self):
        self.cur.execute(
            'CREATE TABLE IF NOT EXISTS scan_history (id INTEGER PRIMARY KEY,'
            ' scan_name TEXT, scan_id INTEGER, last_modified DATE, filename TEXT,'
            ' download_time DATE, record_count INTEGER, source TEXT,'
            ' uuid TEXT, processed INTEGER, reported INTEGER)'
            )
        self.conn.commit()

    def delete_table(self):
        self.cur.execute('DROP TABLE IF EXISTS scan_history')
        self.conn.commit()

    def init(self):
        if self.purge:
            self.delete_table()
        self.create_table()

    def cleanser(self, _data):
        repls = (('\n', r'\n'), ('\r', r'\r'))
        data = reduce(lambda a, kv: a.replace(*kv), repls, _data)
        return data

    def path_check(self, _data):
        if self.write_path:
            if '/' or '\\' in _data[-1]:
                data = self.write_path + _data
            else:
                data = self.write_path + '/' + _data
        return data

    def record_insert(self, record):
        #for backwards compatibility with older versions without "reported" field
        
        try:
            #-1 to get the latest column, 1 to get the column name (old version would be "processed", new "reported")
            #TODO delete backward compatibility check after some versions
            last_column_table = self.cur.execute('PRAGMA table_info(scan_history)').fetchall()[-1][1]
            if last_column_table == self.table_columns[-1]:
                self.cur.execute('insert into scan_history({table_columns}) values (?,?,?,?,?,?,?,?,?,?)'.format(
                    table_columns=', '.join(self.table_columns)), record)

            else:
                self.cur.execute('insert into scan_history({table_columns}) values (?,?,?,?,?,?,?,?,?)'.format(
                    table_columns=', '.join(self.table_columns[:-1])), record[:-1])
            self.conn.commit()
        except Exception as e:
            self.logger.error("Failed to insert record in database. Error: {}".format(e))
            sys.exit(1)

    def set_latest_scan_reported(self, filename):
        #the reason to use the filename instead of the source/scan_name is because the filename already belongs to
        #that latest scan, and we maintain integrity making sure that it is the exact scan we checked
        try:
            self.cur.execute('UPDATE scan_history SET reported = 1 WHERE filename="{}";'.format(filename))
            self.conn.commit()
            self.logger.info('Scan {} marked as successfully processed.'.format(filename))
            return True
        except Exception as e:
            self.logger.error('Failed while setting scan with file {} as processed'.format(filename))
            
        return False

    def retrieve_uuids(self):
        """
        Retrieves UUIDs from database and checks list to determine which files need to be processed.
        :return:
        """
        try:
            self.conn.text_factory = str
            self.cur.execute('SELECT uuid FROM scan_history where source = "{config_section}"'.format(config_section=self.CONFIG_SECTION))
            results = frozenset([r[0] for r in self.cur.fetchall()])
        except:
            results = []
        return results

    def directory_check(self):
        if not os.path.exists(self.write_path):
            os.makedirs(self.write_path)
            self.logger.info('Directory created at {scan} - Skipping creation'.format(
                scan=self.write_path.encode('utf8')))
        else:
            os.path.exists(self.write_path)
            self.logger.info('Directory already exist for {scan} - Skipping creation'.format(
                scan=self.write_path.encode('utf8')))

    def get_latest_results(self, source, scan_name):
        processed = 0
        results = []
        
        try:
            self.conn.text_factory = str
            self.cur.execute('SELECT filename FROM scan_history WHERE source="{}" AND scan_name="{}" ORDER BY last_modified DESC LIMIT 1;'.format(source, scan_name))
            #should always return just one filename
            results = [r[0] for r in self.cur.fetchall()][0]

            #-1 to get the latest column, 1 to get the column name (old version would be "processed", new "reported")
            #TODO delete backward compatibility check after some versions
            last_column_table = self.cur.execute('PRAGMA table_info(scan_history)').fetchall()[-1][1]
            if results and last_column_table == self.table_columns[-1]:
                reported = self.cur.execute('SELECT reported FROM scan_history WHERE filename="{}"'.format(results)).fetchall()
                reported = reported[0][0]
                if reported:
                    self.logger.debug("Last downloaded scan from source {source} scan_name {scan_name} has already been reported".format(source=source, scan_name=scan_name))

        except Exception as e:
            self.logger.error("Error when getting latest results from {}.{} : {}".format(source, scan_name, e))
        return results, reported
        
    def get_scan_profiles(self):
        # Returns a list of source.scan_name elements from the database
        
        # we get the list of sources
        try:
            self.conn.text_factory = str
            self.cur.execute('SELECT DISTINCT source FROM scan_history;')
            sources = [r[0] for r in self.cur.fetchall()]
        except:
            sources = []
            self.logger.error("Process failed at executing 'SELECT DISTINCT source FROM scan_history;'")
        
        results = []

        # we get the list of scans within each source
        for source in sources:
            scan_names = []
            try:
                self.conn.text_factory = str
                self.cur.execute("SELECT DISTINCT scan_name FROM scan_history WHERE source='{}';".format(source))
                scan_names = [r[0] for r in self.cur.fetchall()]
                for scan in scan_names:
                    results.append('{}.{}'.format(source,scan))
            except:
                scan_names = []

        return results

class vulnWhispererNessus(vulnWhispererBase):

    CONFIG_SECTION = None

    def __init__(
            self,
            config=None,
            db_name='report_tracker.db',
            purge=False,
            verbose=None,
            debug=False,
            username=None,
            password=None,
            profile='nessus'
    ):
        self.CONFIG_SECTION=profile

        super(vulnWhispererNessus, self).__init__(config=config)

        self.logger = logging.getLogger('vulnWhispererNessus')
        if debug:
            self.logger.setLevel(logging.DEBUG)
        self.port = int(self.config.get(self.CONFIG_SECTION, 'port'))

        self.develop = True
        self.purge = purge

        if config is not None:
            try:
                self.nessus_port = self.config.get(self.CONFIG_SECTION, 'port')

                self.nessus_trash = self.config.getbool(self.CONFIG_SECTION,
                                                        'trash')

                try:
                    self.logger.info('Attempting to connect to nessus...')
                    self.nessus = \
                        NessusAPI(hostname=self.hostname,
                                  port=self.nessus_port,
                                  username=self.username,
                                  password=self.password)
                    self.nessus_connect = True
                    self.logger.info('Connected to nessus on {host}:{port}'.format(host=self.hostname,
                                                                                   port=str(self.nessus_port)))
                except Exception as e:
                    self.logger.error('Exception: {}'.format(str(e)))
                    raise Exception(
                        'Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format(
                            config=self.config.config_in,
                            e=e))
            except Exception as e:
                self.logger.error('Could not properly load your config!\nReason: {e}'.format(e=e))
                sys.exit(1)



    def scan_count(self, scans, completed=False):
        """

        :param scans: Pulls in available scans
        :param completed: Only return completed scans
        :return:
        """

        self.logger.info('Gathering all scan data... this may take a while...')
        scan_records = []
        for s in scans:
            if s:
                record = {}
                record['scan_id'] = s['id']
                record['scan_name'] = s.get('name', '')
                record['owner'] = s.get('owner', '')
                record['creation_date'] = s.get('creation_date', '')
                record['starttime'] = s.get('starttime', '')
                record['timezone'] = s.get('timezone', '')
                record['folder_id'] = s.get('folder_id', '')
                try:
                    for h in self.nessus.get_scan_history(s['id']):
                        record['uuid'] = h.get('uuid', '')
                        record['status'] = h.get('status', '')
                        record['history_id'] = h.get('history_id', '')
                        record['last_modification_date'] = \
                            h.get('last_modification_date', '')
                        record['norm_time'] = \
                            self.nessus.get_utc_from_local(int(record['last_modification_date'
                                                               ]),
                                                           local_tz=self.nessus.tz_conv(record['timezone'
                                                                                        ]))
                        scan_records.append(record.copy())
                except Exception as e:
                    # Generates error each time nonetype is encountered.
                    pass

        if completed:
            scan_records = [s for s in scan_records if s['status'] == 'completed']
        return scan_records


    def whisper_nessus(self):
        if self.nessus_connect:
            scan_data = self.nessus.scans
            folders = scan_data['folders']
            scans = scan_data['scans'] if scan_data['scans'] else []
            all_scans = self.scan_count(scans)
            if self.uuids:
                scan_list = [scan for scan in all_scans if scan['uuid']
                             not in self.uuids and scan['status'] in ['completed', 'imported']]
            else:
                scan_list = all_scans
            self.logger.info('Identified {new} scans to be processed'.format(new=len(scan_list)))

            if not scan_list:
                self.logger.warn('No new scans to process. Exiting...')
                return self.exit_code

            # Create scan subfolders

            for f in folders:
                if not os.path.exists(self.path_check(f['name'])):
                    if f['name'] == 'Trash' and self.nessus_trash:
                        os.makedirs(self.path_check(f['name']))
                    elif f['name'] != 'Trash':
                        os.makedirs(self.path_check(f['name']))
                else:
                    os.path.exists(self.path_check(f['name']))
                    self.logger.info('Directory already exist for {scan} - Skipping creation'.format(
                        scan=self.path_check(f['name']).encode('utf8')))

            # try download and save scans into each folder the belong to

            scan_count = 0

            # TODO Rewrite this part to go through the scans that have aleady been processed

            for s in scan_list:
                scan_count += 1
                (
                    scan_name,
                    scan_id,
                    history_id,
                    norm_time,
                    status,
                    uuid,
                ) = (
                    s['scan_name'],
                    s['scan_id'],
                    s['history_id'],
                    s['norm_time'],
                    s['status'],
                    s['uuid'],
                )

                # TODO Create directory sync function which scans the directory for files that exist already and populates the database

                folder_id = s['folder_id']
                if self.CONFIG_SECTION == 'tenable':
                    folder_name = ''
                else:
                    folder_name = next(f['name'] for f in folders if f['id'] == folder_id)
                if status == 'completed':
                    file_name = '%s_%s_%s_%s.%s' % (scan_name, scan_id,
                                                    history_id, norm_time, 'csv')
                    repls = (('\\', '_'), ('/', '_'), ('/', '_'), (' ', '_'))
                    file_name = reduce(lambda a, kv: a.replace(*kv), repls, file_name)
                    relative_path_name = self.path_check(folder_name + '/' + file_name)

                    if os.path.isfile(relative_path_name):
                        if self.develop:
                            csv_in = pd.read_csv(relative_path_name)
                            record_meta = (
                                scan_name,
                                scan_id,
                                norm_time,
                                file_name,
                                time.time(),
                                csv_in.shape[0],
                                self.CONFIG_SECTION,
                                uuid,
                                1,
                                0,
                            )
                            self.record_insert(record_meta)
                            self.logger.info('File {filename} already exist! Updating database'.format(filename=relative_path_name))
                    else:
                        try:
                            file_req = \
                                self.nessus.download_scan(scan_id=scan_id, history=history_id,
                                                        export_format='csv', profile=self.CONFIG_SECTION)
                        except Exception as e:
                            self.logger.error('Could not download {} scan {}: {}'.format(self.CONFIG_SECTION, scan_id, str(e)))
                            self.exit_code += 1
                            continue
                            
                        clean_csv = \
                            pd.read_csv(io.StringIO(file_req.decode('utf-8')))
                        if len(clean_csv) > 2:
                            self.logger.info('Processing {}/{} for scan: {}'.format(scan_count, len(scan_list), scan_name.encode('utf8')))
                            columns_to_cleanse = ['CVSS','CVE','Description','Synopsis','Solution','See Also','Plugin Output']

                            for col in columns_to_cleanse:
                                clean_csv[col] = clean_csv[col].astype(str).apply(self.cleanser)

                            clean_csv.to_csv(relative_path_name, index=False)
                            record_meta = (
                                scan_name,
                                scan_id,
                                norm_time,
                                file_name,
                                time.time(),
                                clean_csv.shape[0],
                                self.CONFIG_SECTION,
                                uuid,
                                1,
                                0,
                            )
                            self.record_insert(record_meta)
                            self.logger.info('{filename} records written to {path} '.format(filename=clean_csv.shape[0],
                                                                                            path=file_name.encode('utf8')))
                        else:
                            record_meta = (
                                scan_name,
                                scan_id,
                                norm_time,
                                file_name,
                                time.time(),
                                clean_csv.shape[0],
                                self.CONFIG_SECTION,
                                uuid,
                                1,
                                0,
                            )
                            self.record_insert(record_meta)
                            self.logger.warn('{} has no host available... Updating database and skipping!'.format(file_name))
            self.conn.close()
            self.logger.info('Scan aggregation complete! Connection to database closed.')
        else:
            self.logger.error('Failed to use scanner at {host}:{port}'.format(host=self.hostname, port=self.nessus_port))
            self.exit_code += 1
        return self.exit_code


class vulnWhispererQualys(vulnWhispererBase):

    CONFIG_SECTION = 'qualys_web'
    COLUMN_MAPPING = {'Access Path': 'access_path',
                     'Ajax Request': 'ajax_request',
                     'Ajax Request ID': 'ajax_request_id',
                     'Authentication': 'authentication',
                     'CVSS Base': 'cvss',
                     'CVSS Temporal': 'cvss_temporal',
                     'CWE': 'cwe',
                     'Category': 'category',
                     'Content': 'content',
                     'DescriptionSeverity': 'severity_description',
                     'DescriptionCatSev': 'category_description',
                     'Detection ID': 'detection_id',
                     'Evidence #1': 'evidence_1',
                     'First Time Detected': 'first_time_detected',
                     'Form Entry Point': 'form_entry_point',
                     'Function': 'function',
                     'Groups': 'groups',
                     'ID': 'id',
                     'Ignore Comments': 'ignore_comments',
                     'Ignore Date': 'ignore_date',
                     'Ignore Reason': 'ignore_reason',
                     'Ignore User': 'ignore_user',
                     'Ignored': 'ignored',
                     'Impact': 'impact',
                     'Last Time Detected': 'last_time_detected',
                     'Last Time Tested': 'last_time_tested',
                     'Level': 'level',
                     'OWASP': 'owasp',
                     'Operating System': 'operating_system',
                     'Owner': 'owner',
                     'Param': 'param',
                     'Payload #1': 'payload_1',
                     'QID': 'plugin_id',
                     'Request Headers #1': 'request_headers_1',
                     'Request Method #1': 'request_method_1',
                     'Request URL #1': 'request_url_1',
                     'Response #1': 'response_1',
                     'Scope': 'scope',
                     'Severity': 'risk',
                     'Severity Level': 'security_level',
                     'Solution': 'solution',
                     'Times Detected': 'times_detected',
                     'Title': 'plugin_name',
                     'URL': 'url',
                     'Url': 'uri',
                     'Vulnerability Category': 'vulnerability_category',
                     'WASC': 'wasc',
                     'Web Application Name': 'web_application_name'}
    def __init__(
            self,
            config=None,
            db_name='report_tracker.db',
            purge=False,
            verbose=None,
            debug=False,
            username=None,
            password=None,
        ):

        super(vulnWhispererQualys, self).__init__(config=config)
        self.logger = logging.getLogger('vulnWhispererQualys')
        if debug:
            self.logger.setLevel(logging.DEBUG)
        
        self.qualys_scan = qualysScanReport(config=config)
        self.latest_scans = self.qualys_scan.qw.get_all_scans()
        self.directory_check()
        self.scans_to_process = None

    def whisper_reports(self,
                        report_id=None,
                        launched_date=None,
                        scan_name=None,
                        scan_reference=None,
                        output_format='json',
                        cleanup=True):
        """
        report_id: App ID
        updated_date: Last time scan was ran for app_id
        """
        vuln_ready = None

        try:
            if 'Z' in launched_date:
                launched_date = self.qualys_scan.utils.iso_to_epoch(launched_date)
            report_name = 'qualys_web_' + str(report_id) \
                          + '_{last_updated}'.format(last_updated=launched_date) \
                          + '.{extension}'.format(extension=output_format)

            relative_path_name = self.path_check(report_name)

            if os.path.isfile(relative_path_name):
                #TODO Possibly make this optional to sync directories
                file_length = len(open(relative_path_name).readlines())
                record_meta = (
                    scan_name,
                    scan_reference,
                    launched_date,
                    report_name,
                    time.time(),
                    file_length,
                    self.CONFIG_SECTION,
                    report_id,
                    1,
                    0,
                )
                self.record_insert(record_meta)
                self.logger.info('File {filename} already exist! Updating database'.format(filename=relative_path_name))

            else:
                self.logger.info('Generating report for {}'.format(report_id))
                status = self.qualys_scan.qw.create_report(report_id)
                root = objectify.fromstring(status)
                if root.responseCode == 'SUCCESS':
                    self.logger.info('Successfully generated report! ID: {}'.format(report_id))
                    generated_report_id = root.data.Report.id
                    self.logger.info('New Report ID: {}'.format(generated_report_id))

                    vuln_ready = self.qualys_scan.process_data(path=self.write_path, file_id=str(generated_report_id))

                    vuln_ready['scan_name'] = scan_name
                    vuln_ready['scan_reference'] = scan_reference
                    vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)

                    record_meta = (
                        scan_name,
                        scan_reference,
                        launched_date,
                        report_name,
                        time.time(),
                        vuln_ready.shape[0],
                        self.CONFIG_SECTION,
                        report_id,
                        1,
                        0,
                    )
                    self.record_insert(record_meta)

                    if output_format == 'json':
                        with open(relative_path_name, 'w') as f:
                            f.write(vuln_ready.to_json(orient='records', lines=True))
                            f.write('\n')

                    elif output_format == 'csv':
                       vuln_ready.to_csv(relative_path_name, index=False, header=True)  # add when timestamp occured

                    self.logger.info('Report written to {}'.format(report_name))

                    if cleanup:
                        self.logger.info('Removing report {} from Qualys Database'.format(generated_report_id))
                        cleaning_up = \
                            self.qualys_scan.qw.delete_report(generated_report_id)
                        os.remove(self.path_check(str(generated_report_id) + '.csv'))
                        self.logger.info('Deleted report from local disk: {}'.format(self.path_check(str(generated_report_id))))
                else:
                    self.logger.error('Could not process report ID: {}'.format(status))

        except Exception as e:
            self.logger.error('Could not process {}: {}'.format(report_id, str(e)))
        return vuln_ready


    def identify_scans_to_process(self):
        if self.uuids:
            self.scans_to_process = self.latest_scans[~self.latest_scans['id'].isin(self.uuids)]
        else:
            self.scans_to_process = self.latest_scans
        self.logger.info('Identified {new} scans to be processed'.format(new=len(self.scans_to_process)))


    def process_web_assets(self):
        counter = 0
        self.identify_scans_to_process()
        if self.scans_to_process.shape[0]:
            for app in self.scans_to_process.iterrows():
                counter += 1
                r = app[1]
                self.logger.info('Processing {}/{}'.format(counter, len(self.scans_to_process)))
                self.whisper_reports(report_id=r['id'],
                                     launched_date=r['launchedDate'],
                                     scan_name=r['name'],
                                     scan_reference=r['reference'])
        else:
            self.logger.info('No new scans to process. Exiting...')
        self.conn.close()
        return self.exit_code


class vulnWhispererOpenVAS(vulnWhispererBase):
    CONFIG_SECTION = 'openvas'
    COLUMN_MAPPING = {'IP': 'asset',
                      'Hostname': 'hostname',
                      'Port': 'port',
                      'Port Protocol': 'protocol',
                      'CVSS': 'cvss',
                      'Severity': 'severity',
                      'Solution Type': 'category',
                      'NVT Name': 'plugin_name',
                      'Summary': 'synopsis',
                      'Specific Result': 'plugin_output',
                      'NVT OID': 'nvt_oid',
                      'Task ID': 'task_id',
                      'Task Name': 'task_name',
                      'Timestamp': 'timestamp',
                      'Result ID': 'result_id',
                      'Impact': 'description',
                      'Solution': 'solution',
                      'Affected Software/OS': 'affected_software',
                      'Vulnerability Insight': 'vulnerability_insight',
                      'Vulnerability Detection Method': 'vulnerability_detection_method',
                      'Product Detection Result': 'product_detection_result',
                      'BIDs': 'bids',
                      'CERTs': 'certs',
                      'Other References': 'see_also'
                      }

    def __init__(
            self,
            config=None,
            db_name='report_tracker.db',
            purge=False,
            verbose=None,
            debug=False,
            username=None,
            password=None,
    ):
        super(vulnWhispererOpenVAS, self).__init__(config=config)
        self.logger = logging.getLogger('vulnWhispererOpenVAS')
        if debug:
            self.logger.setLevel(logging.DEBUG)

        self.directory_check()
        self.port = int(self.config.get(self.CONFIG_SECTION, 'port'))
        self.develop = True
        self.purge = purge
        self.scans_to_process = None
        self.openvas_api = OpenVAS_API(hostname=self.hostname,
                                       port=self.port,
                                       username=self.username,
                                       password=self.password)

    def whisper_reports(self, output_format='json', launched_date=None, report_id=None, cleanup=True):
        report = None
        if report_id:
            self.logger.info('Processing report ID: {}'.format(report_id))


            scan_name = report_id.replace('-', '')
            report_name = 'openvas_scan_{scan_name}_{last_updated}.{extension}'.format(scan_name=scan_name,
                                                                                       last_updated=launched_date,
                                                                                       extension=output_format)
            relative_path_name = self.path_check(report_name)
            scan_reference = report_id

            if os.path.isfile(relative_path_name):
                # TODO Possibly make this optional to sync directories
                file_length = len(open(relative_path_name).readlines())
                record_meta = (
                    scan_name,
                    scan_reference,
                    launched_date,
                    report_name,
                    time.time(),
                    file_length,
                    self.CONFIG_SECTION,
                    report_id,
                    1,
                    0,
                )
                self.record_insert(record_meta)
                self.logger.info('File {filename} already exist! Updating database'.format(filename=relative_path_name))

                record_meta = (
                    scan_name,
                    scan_reference,
                    launched_date,
                    report_name,
                    time.time(),
                    file_length,
                    self.CONFIG_SECTION,
                    report_id,
                    1,
                )

            else:
                vuln_ready = self.openvas_api.process_report(report_id=report_id)
                vuln_ready['scan_name'] = scan_name
                vuln_ready['scan_reference'] = report_id
                vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
                vuln_ready.port = vuln_ready.port.fillna(0).astype(int)
                vuln_ready.fillna('', inplace=True)
                if output_format == 'json':
                    with open(relative_path_name, 'w') as f:
                        f.write(vuln_ready.to_json(orient='records', lines=True))
                        f.write('\n')
                self.logger.info('Report written to {}'.format(report_name))

        return report

    def identify_scans_to_process(self):
        if self.uuids:
            self.scans_to_process = self.openvas_api.openvas_reports[
                ~self.openvas_api.openvas_reports.report_ids.isin(self.uuids)]
        else:
            self.scans_to_process = self.openvas_api.openvas_reports
        self.logger.info('Identified {new} scans to be processed'.format(new=len(self.scans_to_process)))

    def process_openvas_scans(self):
        counter = 0
        self.identify_scans_to_process()
        if self.scans_to_process.shape[0]:
            for scan in self.scans_to_process.iterrows():
                counter += 1
                info = scan[1]
                self.logger.info('Processing {}/{} - Report ID: {}'.format(counter, len(self.scans_to_process), info['report_ids']))
                self.whisper_reports(report_id=info['report_ids'],
                                     launched_date=info['epoch'])
            self.logger.info('Processing complete')
        else:
            self.logger.info('No new scans to process. Exiting...')
        self.conn.close()
        return self.exit_code


class vulnWhispererQualysVuln(vulnWhispererBase):

    CONFIG_SECTION = 'qualys_vuln'
    COLUMN_MAPPING = {'cvss_base': 'cvss',
                     'cvss3_base': 'cvss3',
                     'cve_id': 'cve',
                     'os': 'operating_system',
                     'qid': 'plugin_id',
                     'severity': 'risk',
                     'title': 'plugin_name'}

    def __init__(
            self,
            config=None,
            db_name='report_tracker.db',
            purge=False,
            verbose=None,
            debug=False,
            username=None,
            password=None,
        ):
        
        super(vulnWhispererQualysVuln, self).__init__(config=config)
        self.logger = logging.getLogger('vulnWhispererQualysVuln')
        if debug:
            self.logger.setLevel(logging.DEBUG)

        self.qualys_scan = qualysVulnScan(config=config)
        self.directory_check()
        self.scans_to_process = None

    def whisper_reports(self,
                        report_id=None,
                        launched_date=None,
                        scan_name=None,
                        scan_reference=None,
                        output_format='json',
                        cleanup=True):
            launched_date
            if 'Z' in launched_date:
                launched_date = self.qualys_scan.utils.iso_to_epoch(launched_date)
            report_name = 'qualys_vuln_' + report_id.replace('/','_') \
                          + '_{last_updated}'.format(last_updated=launched_date) \
                          + '.json'

            relative_path_name = self.path_check(report_name)

            if os.path.isfile(relative_path_name):
                #TODO Possibly make this optional to sync directories
                file_length = len(open(relative_path_name).readlines())
                record_meta = (
                    scan_name,
                    scan_reference,
                    launched_date,
                    report_name,
                    time.time(),
                    file_length,
                    self.CONFIG_SECTION,
                    report_id,
                    1,
                    0,
                )
                self.record_insert(record_meta)
                self.logger.info('File {filename} already exist! Updating database'.format(filename=relative_path_name))

            else:
                try:
                    self.logger.info('Processing report ID: {}'.format(report_id))
                    vuln_ready = self.qualys_scan.process_data(scan_id=report_id)
                    vuln_ready['scan_name'] = scan_name
                    vuln_ready['scan_reference'] = report_id
                    vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
                except Exception as e:
                    self.logger.error('Could not process {}: {}'.format(report_id, str(e)))
                    self.exit_code += 1
                    return self.exit_code

                record_meta = (
                    scan_name,
                    scan_reference,
                    launched_date,
                    report_name,
                    time.time(),
                    vuln_ready.shape[0],
                    self.CONFIG_SECTION,
                    report_id,
                    1,
                    0,
                )
                self.record_insert(record_meta)

                if output_format == 'json':
                    with open(relative_path_name, 'w') as f:
                        f.write(vuln_ready.to_json(orient='records', lines=True))
                        f.write('\n')

                self.logger.info('Report written to {}'.format(report_name))
            return self.exit_code


    def identify_scans_to_process(self):
        self.latest_scans = self.qualys_scan.qw.get_all_scans()
        if self.uuids:
            self.scans_to_process = self.latest_scans.loc[
                (~self.latest_scans['id'].isin(self.uuids))
                & (self.latest_scans['status'] == 'Finished')]
        else:
            self.scans_to_process = self.latest_scans
        self.logger.info('Identified {new} scans to be processed'.format(new=len(self.scans_to_process)))


    def process_vuln_scans(self):
        counter = 0
        self.identify_scans_to_process()
        if self.scans_to_process.shape[0]:
            for app in self.scans_to_process.iterrows():
                counter += 1
                r = app[1]
                self.logger.info('Processing {}/{}'.format(counter, len(self.scans_to_process)))
                self.exit_code += self.whisper_reports(report_id=r['id'],
                                     launched_date=r['date'],
                                     scan_name=r['name'],
                                     scan_reference=r['type'])
        else:
            self.logger.info('No new scans to process. Exiting...')
        self.conn.close()
        return self.exit_code


class vulnWhispererJIRA(vulnWhispererBase):

    CONFIG_SECTION = 'jira'

    def __init__(
            self,
            config=None,
            db_name='report_tracker.db',
            purge=False,
            verbose=None,
            debug=False,
            username=None,
            password=None,
        ):
        super(vulnWhispererJIRA, self).__init__(config=config)
        self.logger = logging.getLogger('vulnWhispererJira')
        if debug:
            self.logger.setLevel(logging.DEBUG)
        self.config_path = config
        self.config = vwConfig(config)
        self.host_resolv_cache = {}
        self.directory_check()   
                 
        if config is not None:
            try:
                self.logger.info('Attempting to connect to jira...')
                self.jira = \
                    JiraAPI(hostname=self.hostname,
                              username=self.username,
                              password=self.password,
                              path=self.config.get('jira','write_path'))
                self.jira_connect = True
                self.logger.info('Connected to jira on {host}'.format(host=self.hostname))
            except Exception as e:
                self.logger.error('Exception: {}'.format(str(e)))
                raise Exception(
                    'Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format(
                        config=self.config.config_in, e=e))
                sys.exit(1)
   
        profiles = []
        profiles = self.get_scan_profiles()
        
        if not self.config.exists_jira_profiles(profiles):
            self.config.update_jira_profiles(profiles)
            self.logger.info("Jira profiles have been created in {config}, please fill the variables before rerunning the module.".format(config=self.config_path))
            sys.exit(0)
    
   
    def get_env_variables(self, source, scan_name):
        # function returns an array with [jira_project, jira_components, datafile_path]

        #Jira variables
        jira_section = self.config.normalize_section("{}.{}".format(source,scan_name))

        project = self.config.get(jira_section,'jira_project')
        if project == "":
            self.logger.error('JIRA project is missing on the configuration file!')
            sys.exit(0)
        
        # check that project actually exists
        if not self.jira.project_exists(project):
            self.logger.error("JIRA project '{project}' doesn't exist!".format(project=project))
            sys.exit(0)
         
        components = self.config.get(jira_section,'components').split(',')
        
        #cleaning empty array from ''
        if not components[0]:
            components = []
        
        min_critical = self.config.get(jira_section,'min_critical_to_report')
        if not min_critical:
            self.logger.error('"min_critical_to_report" variable on config file is empty.')
            sys.exit(0)
            
        #datafile path
        filename, reported = self.get_latest_results(source, scan_name)
        fullpath = ""
        
        # search data files under user specified directory
        for root, dirnames, filenames in os.walk(vwConfig(self.config_path).get(source,'write_path')):
            if filename in filenames:
                fullpath = "{}/{}".format(root,filename)
        
        if reported:
            self.logger.warn('Last Scan of "{scan_name}" for source "{source}" has already been reported; will be skipped.'.format(scan_name=scan_name, source=source))
            return [False] * 5

        if not fullpath:
            self.logger.error('Scan of "{scan_name}" for source "{source}" has not been found. Please check that the scanner data files are in place.'.format(scan_name=scan_name, source=source))
            sys.exit(1)
    
        dns_resolv = self.config.get('jira','dns_resolv')
        if dns_resolv in ('False', 'false', ''):
            dns_resolv = False
        elif dns_resolv in ('True', 'true'):
            dns_resolv = True
        else:
            self.logger.error("dns_resolv variable not setup in [jira] section; will not do dns resolution")
            dns_resolv = False

        return project, components, fullpath, min_critical, dns_resolv


    def parse_nessus_vulnerabilities(self, fullpath, source, scan_name, min_critical):
        
        vulnerabilities = []

        # we need to parse the CSV
        risks = ['none', 'low', 'medium', 'high', 'critical'] 
        min_risk = int([i for i,x in enumerate(risks) if x == min_critical][0])

        df = pd.read_csv(fullpath, delimiter=',')
        
        #nessus fields we want - ['Host','Protocol','Port', 'Name', 'Synopsis', 'Description', 'Solution', 'See Also']
        for index in range(len(df)):
            # filtering vulnerabilities by criticality, discarding low risk
            to_report = int([i for i,x in enumerate(risks) if x == df.loc[index]['Risk'].lower()][0])
            if to_report < min_risk:
                continue
            
            if not vulnerabilities or df.loc[index]['Name'] not in [entry['title'] for entry in vulnerabilities]:
                vuln = {}
                #vulnerabilities should have all the info for creating all JIRA labels
                vuln['source'] = source
                vuln['scan_name'] = scan_name
                #vulnerability variables
                vuln['title'] = df.loc[index]['Name']
                vuln['diagnosis'] =  df.loc[index]['Synopsis'].replace('\\n',' ')
                vuln['consequence'] = df.loc[index]['Description'].replace('\\n',' ')
                vuln['solution'] = df.loc[index]['Solution'].replace('\\n',' ')
                vuln['ips'] = []
                vuln['ips'].append("{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], df.loc[index]['Port']))
                vuln['risk'] = df.loc[index]['Risk'].lower()
                
                # Nessus "nan" value gets automatically casted to float by python
                if not (type(df.loc[index]['See Also']) is float):
                    vuln['references'] = df.loc[index]['See Also'].split("\\n")
                else:
                    vuln['references'] = []
                vulnerabilities.append(vuln)

            else:
                # grouping assets by vulnerability to open on single ticket, as each asset has its own nessus entry
                for vuln in vulnerabilities:
                    if vuln['title'] == df.loc[index]['Name']:
                        vuln['ips'].append("{} - {}/{}".format(df.loc[index]['Host'], df.loc[index]['Protocol'], df.loc[index]['Port']))
        
        return vulnerabilities
    
    def parse_qualys_vuln_vulnerabilities(self, fullpath, source, scan_name, min_critical, dns_resolv = False):
        #parsing of the qualys vulnerabilities schema
        #parse json
        vulnerabilities = []

        risks = ['info', 'low', 'medium', 'high', 'critical'] 
        # +1 as array is 0-4, but score is 1-5
        min_risk = int([i for i,x in enumerate(risks) if x == min_critical][0])+1
        
        try:
            data=[json.loads(line) for line in open(fullpath).readlines()] 
        except Exception as e:
            self.logger.warn("Scan has no vulnerabilities, skipping.")
            return vulnerabilities
       
        #qualys fields we want - []
        for index in range(len(data)):
            if int(data[index]['risk']) < min_risk:
                continue

            elif data[index]['type'] == 'Practice' or data[index]['type'] == 'Ig':
                self.logger.debug("Vulnerability '{vuln}' ignored, as it is 'Practice/Potential', not verified.".format(vuln=data[index]['plugin_name']))
                continue
            
            if not vulnerabilities or data[index]['plugin_name'] not in [entry['title'] for entry in vulnerabilities]:
                vuln = {}
                #vulnerabilities should have all the info for creating all JIRA labels
                vuln['source'] = source
                vuln['scan_name'] = scan_name
                #vulnerability variables
                vuln['title'] = data[index]['plugin_name']
                vuln['diagnosis'] =  data[index]['threat'].replace('\\n',' ')
                vuln['consequence'] = data[index]['impact'].replace('\\n',' ')
                vuln['solution'] = data[index]['solution'].replace('\\n',' ')
                vuln['ips'] = []
                #TODO ADDED DNS RESOLUTION FROM QUALYS! \n SEPARATORS INSTEAD OF \\n!
                
                vuln['ips'].append("{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv)))

                #different risk system than Nessus!
                vuln['risk'] = risks[int(data[index]['risk'])-1]
                
                # Nessus "nan" value gets automatically casted to float by python
                if not (type(data[index]['vendor_reference']) is float or data[index]['vendor_reference'] == None):
                    vuln['references'] = data[index]['vendor_reference'].split("\\n")
                else:
                    vuln['references'] = []
                vulnerabilities.append(vuln)
            else:
                # grouping assets by vulnerability to open on single ticket, as each asset has its own nessus entry
                for vuln in vulnerabilities:
                    if vuln['title'] == data[index]['plugin_name']:
                        vuln['ips'].append("{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv)))

        return vulnerabilities

    def get_asset_fields(self, vuln, dns_resolv):
        values = {}
        values['ip'] = vuln['ip']
        values['protocol'] = vuln['protocol'] 
        values['port'] = vuln['port'] 
        values['dns'] = ''
        if dns_resolv:
            if vuln['dns']:
                values['dns'] = vuln['dns']
            else:
                if values['ip'] in self.host_resolv_cache.keys():
                    self.logger.debug("Hostname from {ip} cached, retrieving from cache.".format(ip=values['ip']))
                    values['dns'] = self.host_resolv_cache[values['ip']]
                else:
                    self.logger.debug("No hostname, trying to resolve {ip}'s  hostname.".format(ip=values['ip']))
                    try:
                        values['dns'] = socket.gethostbyaddr(vuln['ip'])[0]
                        self.host_resolv_cache[values['ip']] = values['dns']
                        self.logger.debug("Hostname found: {hostname}.".format(hostname=values['dns']))
                    except:
                        self.host_resolv_cache[values['ip']] = ''
                        self.logger.debug("Hostname not found for: {ip}.".format(ip=values['ip']))

        for key in values.keys():
            if not values[key]:
                values[key] = 'N/A'

        return values

    def parse_vulnerabilities(self, fullpath, source, scan_name, min_critical):
        #TODO: SINGLE LOCAL SAVE FORMAT FOR ALL SCANNERS
        #JIRA standard vuln format - ['source', 'scan_name', 'title', 'diagnosis', 'consequence', 'solution', 'ips', 'references']

        return 0


    def jira_sync(self, source, scan_name):
        self.logger.info("Jira Sync triggered for source '{source}' and scan '{scan_name}'".format(source=source, scan_name=scan_name))

        project, components, fullpath, min_critical, dns_resolv = self.get_env_variables(source, scan_name)

        if not project:
            self.logger.debug("Skipping scan for source '{source}' and scan '{scan_name}': vulnerabilities have already been reported.".format(source=source, scan_name=scan_name))
            return False

        vulnerabilities = []

        #***Nessus parsing***
        if source == "nessus":
            vulnerabilities = self.parse_nessus_vulnerabilities(fullpath, source, scan_name, min_critical)

        #***Qualys VM parsing***
        if source == "qualys_vuln":
            vulnerabilities = self.parse_qualys_vuln_vulnerabilities(fullpath, source, scan_name, min_critical, dns_resolv)
        
        #***JIRA sync***
        if vulnerabilities:
            self.logger.info('{source} data has been successfuly parsed'.format(source=source.upper()))
            self.logger.info('Starting JIRA sync')
            
            self.jira.sync(vulnerabilities, project, components)
        else:
            self.logger.info("[{source}.{scan_name}] No vulnerabilities or vulnerabilities not parsed.".format(source=source, scan_name=scan_name))
            self.set_latest_scan_reported(fullpath.split("/")[-1])
            return False

        self.set_latest_scan_reported(fullpath.split("/")[-1])
        return True

    def sync_all(self):
        autoreport_sections = self.config.get_sections_with_attribute('autoreport')

        if autoreport_sections:
            for scan in autoreport_sections:
                self.jira_sync(self.config.get(scan, 'source'), self.config.get(scan, 'scan_name'))
            return True
        return False

class vulnWhisperer(object):

    def __init__(self,
                 profile=None,
                 verbose=None,
                 username=None,
                 password=None,
                 config=None,
                 source=None,
                 scanname=None):

        self.logger = logging.getLogger('vulnWhisperer')
        if verbose:
            self.logger.setLevel(logging.DEBUG)
        self.profile = profile
        self.config = config
        self.username = username
        self.password = password
        self.verbose = verbose
        self.source = source
        self.scanname = scanname
        self.exit_code = 0


    def whisper_vulnerabilities(self):

        if self.profile == 'nessus':
            vw = vulnWhispererNessus(config=self.config,
                                     username=self.username,
                                     password=self.password,
                                     verbose=self.verbose,
                                     profile=self.profile)
            self.exit_code += vw.whisper_nessus()

        elif self.profile == 'qualys_web':
            vw = vulnWhispererQualys(config=self.config)
            self.exit_code += vw.process_web_assets()

        elif self.profile == 'openvas':
            vw_openvas = vulnWhispererOpenVAS(config=self.config)
            self.exit_code += vw_openvas.process_openvas_scans()

        elif self.profile == 'tenable':
            vw = vulnWhispererNessus(config=self.config,
                                     username=self.username,
                                     password=self.password,
                                     verbose=self.verbose,
                                     profile=self.profile)
            self.exit_code += vw.whisper_nessus()

        elif self.profile == 'qualys_vuln':
            vw = vulnWhispererQualysVuln(config=self.config)
            self.exit_code += vw.process_vuln_scans()
        
        elif self.profile == 'jira':
            #first we check config fields are created, otherwise we create them
            vw = vulnWhispererJIRA(config=self.config)
            if not (self.source and self.scanname):
                self.logger.info('No source/scan_name selected, all enabled scans will be synced')
                success = vw.sync_all()
                if not success:
                    self.logger.error('All scans sync failed!')
                    self.logger.error('Source scanner and scan name needed!')
                    return 0
            else:
                vw.jira_sync(self.source, self.scanname)

        return self.exit_code