diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 131145d..5dbd5b8 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -81,12 +81,13 @@ services: entrypoint: [ "vuln_whisperer", "-F", + "-v", "-c", "/opt/VulnWhisperer/vulnwhisperer.ini", "--mock", "--mock_dir", "/tests/data" - ] + ] volumes: - ./data/vulnwhisperer/:/opt/VulnWhisperer/data # - ./resources/elk6/vulnwhisperer.ini:/opt/VulnWhisperer/vulnwhisperer.ini diff --git a/vulnwhisp/frameworks/nessus.py b/vulnwhisp/frameworks/nessus.py index 0bac446..494baee 100755 --- a/vulnwhisp/frameworks/nessus.py +++ b/vulnwhisp/frameworks/nessus.py @@ -42,7 +42,6 @@ class NessusAPI(object): 'system type': 'category', 'vulnerability state': 'state' } - SEVERITY_MAPPING = {'none': 0, 'low': 1, 'medium': 2, 'high': 3, 'critical': 4} def __init__(self, hostname=None, port=None, username=None, password=None, verbose=True, profile=None, access_key=None, secret_key=None): self.logger = logging.getLogger('NessusAPI') @@ -229,10 +228,6 @@ class NessusAPI(object): df['protocol'] = df['protocol'].str.lower() df['risk'] = df['risk'].str.lower() - # Map risk to a SEVERITY MAPPING value - self.logger.debug('Mapping risk to severity number') - df['risk_number'] = df['risk'].map(self.SEVERITY_MAPPING) - df.fillna('', inplace=True) return df diff --git a/vulnwhisp/frameworks/openvas.py b/vulnwhisp/frameworks/openvas.py index c411f7d..bb95b30 100644 --- a/vulnwhisp/frameworks/openvas.py +++ b/vulnwhisp/frameworks/openvas.py @@ -13,6 +13,20 @@ from bs4 import BeautifulSoup class OpenVAS_API(object): OMP = '/omp' + COLUMN_MAPPING = { + 'affected software/os': 'affected_software', + 'cves': 'cve', + 'impact': 'description', + 'nvt name': 'signature', + 'nvt oid': 'signature_id', + 'other references': 'exploitability', + 'port protocol': 'protocol', + 'severity': 'risk', + 'solution type': 'category', + 'task name': 'scan_name', + 'specific result': 'plugin_output', + 'summary': 'synopsis', + } def __init__(self, hostname=None, @@ -200,9 +214,16 @@ class OpenVAS_API(object): def map_fields(self, df): self.logger.debug('Mapping fields') + # Lowercase and map fields from COLUMN_MAPPING + df.columns = [x.lower() for x in df.columns] + df.rename(columns=self.COLUMN_MAPPING, inplace=True) + df.columns = [x.replace(' ', '_') for x in df.columns] return df def transform_values(self, df): self.logger.debug('Transforming values') + df['port'].fillna(0).astype(int) + df['risk'] = df['risk'].str.lower() + df['asset'] = df['ip'] df.fillna('', inplace=True) return df diff --git a/vulnwhisp/frameworks/qualys_vm.py b/vulnwhisp/frameworks/qualys_vm.py index eb420ba..b510958 100644 --- a/vulnwhisp/frameworks/qualys_vm.py +++ b/vulnwhisp/frameworks/qualys_vm.py @@ -89,8 +89,6 @@ class qualysVulnScan: 'title': 'signature' } - SEVERITY_MAPPING = {0: 'none', 1: 'low', 2: 'medium', 3: 'high',4: 'critical'} - def __init__( self, config=None, @@ -176,7 +174,6 @@ class qualysVulnScan: # Convert Qualys severity to standardised risk number df['risk_number'] = df['severity'].astype(int)-1 - df['risk'] = df['risk_number'].map(self.SEVERITY_MAPPING) df.fillna('', inplace=True) diff --git a/vulnwhisp/frameworks/qualys_was.py b/vulnwhisp/frameworks/qualys_was.py index ab918fd..a0530af 100644 --- a/vulnwhisp/frameworks/qualys_was.py +++ b/vulnwhisp/frameworks/qualys_was.py @@ -300,8 +300,6 @@ class qualysScanReport: 'Vulnerability Category': 'type', } - SEVERITY_MAPPING = {0: 'none', 1: 'low', 2: 'medium', 3: 'high', 4: 'critical'} - # URL Vulnerability Information WEB_SCAN_VULN_BLOCK = list(qualysReportFields.VULN_BLOCK) WEB_SCAN_VULN_BLOCK.insert(WEB_SCAN_VULN_BLOCK.index('QID'), 'Detection ID') @@ -521,7 +519,6 @@ class qualysScanReport: # Convert Qualys severity to standardised risk number df['risk_number'] = df['severity'].astype(int)-1 - df['risk'] = df['risk_number'].map(self.SEVERITY_MAPPING) # Extract dns field from URL df['dns'] = df['url'].str.extract('https?://([^/]+)', expand=False) diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index 28ce12c..3c8484f 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -26,6 +26,8 @@ from reporting.jira_api import JiraAPI class vulnWhispererBase(object): CONFIG_SECTION = None + SEVERITY_NAME_MAPPING = {'none': 0, 'low': 1, 'medium': 2, 'high': 3, 'critical': 4} + SEVERITY_NUMBER_MAPPING = {0: 'none', 1: 'low', 2: 'medium', 3: 'high', 4: 'critical'} def __init__( self, @@ -251,6 +253,16 @@ class vulnWhispererBase(object): df.replace({'': np.nan}, inplace=True) + # Map risk name to a risk value + if 'risk' in df and not 'risk_number' in df: + self.logger.debug('Mapping risk name to risk number') + df['risk_number'] = df['risk'].map(self.SEVERITY_NAME_MAPPING) + + # Map risk value to a risk name + if 'risk_number' in df and not 'risk' in df: + self.logger.debug('Mapping risk number to risk name') + df['risk'] = df['risk_number'].map(self.SEVERITY_NUMBER_MAPPING) + self.logger.debug('Normalising CVSS') for cvss_version in ['cvss2', 'cvss3']: # cvssX = cvssX_temporal else cvssX_base @@ -694,32 +706,6 @@ class vulnWhispererQualysWAS(vulnWhispererBase): class vulnWhispererOpenVAS(vulnWhispererBase): CONFIG_SECTION = 'openvas' - COLUMN_MAPPING = {'IP': 'asset', - 'Hostname': 'hostname', - 'Port': 'port', - 'Port Protocol': 'protocol', - 'CVEs': 'cve', - 'CVSS': 'cvss', - 'Severity': 'severity', - 'Solution Type': 'category', - 'NVT Name': 'signature', - 'Summary': 'synopsis', - 'Specific Result': 'plugin_output', - 'NVT OID': 'nvt_oid', - 'Task ID': 'task_id', - 'Task Name': 'scan_name', - 'Timestamp': 'timestamp', - 'Result ID': 'result_id', - 'Impact': 'description', - 'Solution': 'solution', - 'Affected Software/OS': 'affected_software', - 'Vulnerability Insight': 'vulnerability_insight', - 'Vulnerability Detection Method': 'vulnerability_detection_method', - 'Product Detection Result': 'product_detection_result', - 'BIDs': 'bids', - 'CERTs': 'certs', - 'Other References': 'see_also' - } def __init__( self, @@ -782,12 +768,7 @@ class vulnWhispererOpenVAS(vulnWhispererBase): # Map and transform fields vuln_ready = self.openvas_api.normalise(vuln_ready) - # TODO move the following to the openvas_api.transform_values - vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True) - vuln_ready.port = vuln_ready.port.replace('', 0).astype(int) - # Set common fields - # vuln_ready['scan_name'] = scan_name.encode('utf8') vuln_ready['scan_id'] = report_id vuln_ready['scan_time'] = launched_date vuln_ready['scan_source'] = self.CONFIG_SECTION @@ -841,7 +822,7 @@ class vulnWhispererOpenVAS(vulnWhispererBase): return self.exit_code -class vulnWhispererQualysVuln(vulnWhispererBase): +class vulnWhispererQualysVM(vulnWhispererBase): CONFIG_SECTION = 'qualys_vm' @@ -854,8 +835,8 @@ class vulnWhispererQualysVuln(vulnWhispererBase): debug=False, ): - super(vulnWhispererQualysVuln, self).__init__(config=config, verbose=verbose, debug=debug) - self.logger = logging.getLogger('vulnWhispererQualysVuln') + super(vulnWhispererQualysVM, self).__init__(config=config, verbose=verbose, debug=debug) + self.logger = logging.getLogger('vulnWhispererQualysVM') if not verbose: verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose') self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING) @@ -1306,7 +1287,7 @@ class vulnWhisperer(object): self.exit_code += vw.whisper_nessus() elif self.profile == 'qualys_vm': - vw = vulnWhispererQualysVuln(config=self.config, + vw = vulnWhispererQualysVM(config=self.config, verbose=self.verbose, debug=self.debug) self.exit_code += vw.process_vuln_scans()