map openvas and make risk mapping common

This commit is contained in:
pemontto
2019-05-07 17:14:42 +01:00
parent 44d75c397f
commit c350ec73c2
6 changed files with 39 additions and 47 deletions

View File

@ -81,12 +81,13 @@ services:
entrypoint: [ entrypoint: [
"vuln_whisperer", "vuln_whisperer",
"-F", "-F",
"-v",
"-c", "-c",
"/opt/VulnWhisperer/vulnwhisperer.ini", "/opt/VulnWhisperer/vulnwhisperer.ini",
"--mock", "--mock",
"--mock_dir", "--mock_dir",
"/tests/data" "/tests/data"
] ]
volumes: volumes:
- ./data/vulnwhisperer/:/opt/VulnWhisperer/data - ./data/vulnwhisperer/:/opt/VulnWhisperer/data
# - ./resources/elk6/vulnwhisperer.ini:/opt/VulnWhisperer/vulnwhisperer.ini # - ./resources/elk6/vulnwhisperer.ini:/opt/VulnWhisperer/vulnwhisperer.ini

View File

@ -42,7 +42,6 @@ class NessusAPI(object):
'system type': 'category', 'system type': 'category',
'vulnerability state': 'state' 'vulnerability state': 'state'
} }
SEVERITY_MAPPING = {'none': 0, 'low': 1, 'medium': 2, 'high': 3, 'critical': 4}
def __init__(self, hostname=None, port=None, username=None, password=None, verbose=True, profile=None, access_key=None, secret_key=None): def __init__(self, hostname=None, port=None, username=None, password=None, verbose=True, profile=None, access_key=None, secret_key=None):
self.logger = logging.getLogger('NessusAPI') self.logger = logging.getLogger('NessusAPI')
@ -229,10 +228,6 @@ class NessusAPI(object):
df['protocol'] = df['protocol'].str.lower() df['protocol'] = df['protocol'].str.lower()
df['risk'] = df['risk'].str.lower() df['risk'] = df['risk'].str.lower()
# Map risk to a SEVERITY MAPPING value
self.logger.debug('Mapping risk to severity number')
df['risk_number'] = df['risk'].map(self.SEVERITY_MAPPING)
df.fillna('', inplace=True) df.fillna('', inplace=True)
return df return df

View File

@ -13,6 +13,20 @@ from bs4 import BeautifulSoup
class OpenVAS_API(object): class OpenVAS_API(object):
OMP = '/omp' OMP = '/omp'
COLUMN_MAPPING = {
'affected software/os': 'affected_software',
'cves': 'cve',
'impact': 'description',
'nvt name': 'signature',
'nvt oid': 'signature_id',
'other references': 'exploitability',
'port protocol': 'protocol',
'severity': 'risk',
'solution type': 'category',
'task name': 'scan_name',
'specific result': 'plugin_output',
'summary': 'synopsis',
}
def __init__(self, def __init__(self,
hostname=None, hostname=None,
@ -200,9 +214,16 @@ class OpenVAS_API(object):
def map_fields(self, df): def map_fields(self, df):
self.logger.debug('Mapping fields') self.logger.debug('Mapping fields')
# Lowercase and map fields from COLUMN_MAPPING
df.columns = [x.lower() for x in df.columns]
df.rename(columns=self.COLUMN_MAPPING, inplace=True)
df.columns = [x.replace(' ', '_') for x in df.columns]
return df return df
def transform_values(self, df): def transform_values(self, df):
self.logger.debug('Transforming values') self.logger.debug('Transforming values')
df['port'].fillna(0).astype(int)
df['risk'] = df['risk'].str.lower()
df['asset'] = df['ip']
df.fillna('', inplace=True) df.fillna('', inplace=True)
return df return df

View File

@ -89,8 +89,6 @@ class qualysVulnScan:
'title': 'signature' 'title': 'signature'
} }
SEVERITY_MAPPING = {0: 'none', 1: 'low', 2: 'medium', 3: 'high',4: 'critical'}
def __init__( def __init__(
self, self,
config=None, config=None,
@ -176,7 +174,6 @@ class qualysVulnScan:
# Convert Qualys severity to standardised risk number # Convert Qualys severity to standardised risk number
df['risk_number'] = df['severity'].astype(int)-1 df['risk_number'] = df['severity'].astype(int)-1
df['risk'] = df['risk_number'].map(self.SEVERITY_MAPPING)
df.fillna('', inplace=True) df.fillna('', inplace=True)

View File

@ -300,8 +300,6 @@ class qualysScanReport:
'Vulnerability Category': 'type', 'Vulnerability Category': 'type',
} }
SEVERITY_MAPPING = {0: 'none', 1: 'low', 2: 'medium', 3: 'high', 4: 'critical'}
# URL Vulnerability Information # URL Vulnerability Information
WEB_SCAN_VULN_BLOCK = list(qualysReportFields.VULN_BLOCK) WEB_SCAN_VULN_BLOCK = list(qualysReportFields.VULN_BLOCK)
WEB_SCAN_VULN_BLOCK.insert(WEB_SCAN_VULN_BLOCK.index('QID'), 'Detection ID') WEB_SCAN_VULN_BLOCK.insert(WEB_SCAN_VULN_BLOCK.index('QID'), 'Detection ID')
@ -521,7 +519,6 @@ class qualysScanReport:
# Convert Qualys severity to standardised risk number # Convert Qualys severity to standardised risk number
df['risk_number'] = df['severity'].astype(int)-1 df['risk_number'] = df['severity'].astype(int)-1
df['risk'] = df['risk_number'].map(self.SEVERITY_MAPPING)
# Extract dns field from URL # Extract dns field from URL
df['dns'] = df['url'].str.extract('https?://([^/]+)', expand=False) df['dns'] = df['url'].str.extract('https?://([^/]+)', expand=False)

View File

@ -26,6 +26,8 @@ from reporting.jira_api import JiraAPI
class vulnWhispererBase(object): class vulnWhispererBase(object):
CONFIG_SECTION = None CONFIG_SECTION = None
SEVERITY_NAME_MAPPING = {'none': 0, 'low': 1, 'medium': 2, 'high': 3, 'critical': 4}
SEVERITY_NUMBER_MAPPING = {0: 'none', 1: 'low', 2: 'medium', 3: 'high', 4: 'critical'}
def __init__( def __init__(
self, self,
@ -251,6 +253,16 @@ class vulnWhispererBase(object):
df.replace({'': np.nan}, inplace=True) df.replace({'': np.nan}, inplace=True)
# Map risk name to a risk value
if 'risk' in df and not 'risk_number' in df:
self.logger.debug('Mapping risk name to risk number')
df['risk_number'] = df['risk'].map(self.SEVERITY_NAME_MAPPING)
# Map risk value to a risk name
if 'risk_number' in df and not 'risk' in df:
self.logger.debug('Mapping risk number to risk name')
df['risk'] = df['risk_number'].map(self.SEVERITY_NUMBER_MAPPING)
self.logger.debug('Normalising CVSS') self.logger.debug('Normalising CVSS')
for cvss_version in ['cvss2', 'cvss3']: for cvss_version in ['cvss2', 'cvss3']:
# cvssX = cvssX_temporal else cvssX_base # cvssX = cvssX_temporal else cvssX_base
@ -694,32 +706,6 @@ class vulnWhispererQualysWAS(vulnWhispererBase):
class vulnWhispererOpenVAS(vulnWhispererBase): class vulnWhispererOpenVAS(vulnWhispererBase):
CONFIG_SECTION = 'openvas' CONFIG_SECTION = 'openvas'
COLUMN_MAPPING = {'IP': 'asset',
'Hostname': 'hostname',
'Port': 'port',
'Port Protocol': 'protocol',
'CVEs': 'cve',
'CVSS': 'cvss',
'Severity': 'severity',
'Solution Type': 'category',
'NVT Name': 'signature',
'Summary': 'synopsis',
'Specific Result': 'plugin_output',
'NVT OID': 'nvt_oid',
'Task ID': 'task_id',
'Task Name': 'scan_name',
'Timestamp': 'timestamp',
'Result ID': 'result_id',
'Impact': 'description',
'Solution': 'solution',
'Affected Software/OS': 'affected_software',
'Vulnerability Insight': 'vulnerability_insight',
'Vulnerability Detection Method': 'vulnerability_detection_method',
'Product Detection Result': 'product_detection_result',
'BIDs': 'bids',
'CERTs': 'certs',
'Other References': 'see_also'
}
def __init__( def __init__(
self, self,
@ -782,12 +768,7 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
# Map and transform fields # Map and transform fields
vuln_ready = self.openvas_api.normalise(vuln_ready) vuln_ready = self.openvas_api.normalise(vuln_ready)
# TODO move the following to the openvas_api.transform_values
vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
vuln_ready.port = vuln_ready.port.replace('', 0).astype(int)
# Set common fields # Set common fields
# vuln_ready['scan_name'] = scan_name.encode('utf8')
vuln_ready['scan_id'] = report_id vuln_ready['scan_id'] = report_id
vuln_ready['scan_time'] = launched_date vuln_ready['scan_time'] = launched_date
vuln_ready['scan_source'] = self.CONFIG_SECTION vuln_ready['scan_source'] = self.CONFIG_SECTION
@ -841,7 +822,7 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
return self.exit_code return self.exit_code
class vulnWhispererQualysVuln(vulnWhispererBase): class vulnWhispererQualysVM(vulnWhispererBase):
CONFIG_SECTION = 'qualys_vm' CONFIG_SECTION = 'qualys_vm'
@ -854,8 +835,8 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
debug=False, debug=False,
): ):
super(vulnWhispererQualysVuln, self).__init__(config=config, verbose=verbose, debug=debug) super(vulnWhispererQualysVM, self).__init__(config=config, verbose=verbose, debug=debug)
self.logger = logging.getLogger('vulnWhispererQualysVuln') self.logger = logging.getLogger('vulnWhispererQualysVM')
if not verbose: if not verbose:
verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose') verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING) self.logger.setLevel(logging.DEBUG if debug else logging.INFO if verbose else logging.WARNING)
@ -1306,7 +1287,7 @@ class vulnWhisperer(object):
self.exit_code += vw.whisper_nessus() self.exit_code += vw.whisper_nessus()
elif self.profile == 'qualys_vm': elif self.profile == 'qualys_vm':
vw = vulnWhispererQualysVuln(config=self.config, vw = vulnWhispererQualysVM(config=self.config,
verbose=self.verbose, verbose=self.verbose,
debug=self.debug) debug=self.debug)
self.exit_code += vw.process_vuln_scans() self.exit_code += vw.process_vuln_scans()