diff --git a/resources/elk6/pipeline/1000_nessus_process_file.conf b/resources/elk6/pipeline/1000_nessus_process_file.conf index 2be2e03..335dd20 100644 --- a/resources/elk6/pipeline/1000_nessus_process_file.conf +++ b/resources/elk6/pipeline/1000_nessus_process_file.conf @@ -40,59 +40,42 @@ filter { tag_on_failure => [] } - translate { - field => "[risk]" - destination => "[risk_number]" - dictionary => { - "None" => 0 - "Low" => 1 - "Medium" => 2 - "High" => 3 - "Critical" => 4 - } - } - - mutate { - add_field => { "risk_score" => "%{cvss}" } - } - mutate { + convert => { "cvss" => "float"} convert => { "cvss_base" => "float"} convert => { "cvss_temporal" => "float"} - convert => { "cvss" => "float"} + convert => { "cvss3" => "float"} convert => { "cvss3_base" => "float"} convert => { "cvss3_temporal" => "float"} - convert => { "cvss3" => "float"} convert => { "id" => "integer"} convert => { "plugin_id" => "integer"} convert => { "risk_number" => "integer"} - convert => { "risk_score" => "float"} convert => { "total_times_detected" => "integer"} } - if [risk_score] == 0 { + if [cvss] == 0 { mutate { - add_field => { "risk_score_name" => "info" } + add_field => { "cvss_severity" => "info" } } } - if [risk_score] > 0 and [risk_score] < 3 { + if [cvss] > 0 and [cvss] < 3 { mutate { - add_field => { "risk_score_name" => "low" } + add_field => { "cvss_severity" => "low" } } } - if [risk_score] >= 3 and [risk_score] < 6 { + if [cvss] >= 3 and [cvss] < 6 { mutate { - add_field => { "risk_score_name" => "medium" } + add_field => { "cvss_severity" => "medium" } } } - if [risk_score] >=6 and [risk_score] < 9 { + if [cvss] >=6 and [cvss] < 9 { mutate { - add_field => { "risk_score_name" => "high" } + add_field => { "cvss_severity" => "high" } } } - if [risk_score] >= 9 { + if [cvss] >= 9 { mutate { - add_field => { "risk_score_name" => "critical" } + add_field => { "cvss_severity" => "critical" } } } } diff --git a/resources/elk6/pipeline/2000_qualys_web_scans.conf b/resources/elk6/pipeline/2000_qualys_web_scans.conf index 329257f..6a4e11f 100644 --- a/resources/elk6/pipeline/2000_qualys_web_scans.conf +++ b/resources/elk6/pipeline/2000_qualys_web_scans.conf @@ -30,18 +30,6 @@ filter { tag_on_failure => [] } - translate { - field => "[risk_number]" - destination => "[risk]" - dictionary => { - "0" => "Info" - "1" => "Low" - "2" => "Medium" - "3" => "High" - "4" => "Critical" - } - } - if "qualys_web" in [tags] { mutate { add_field => { "asset" => "%{web_application_name}" } @@ -49,46 +37,41 @@ filter { } mutate { - add_field => { "risk_score" => "%{cvss}" } - } - - mutate { + convert => { "cvss" => "float"} convert => { "cvss_base" => "float"} convert => { "cvss_temporal" => "float"} - convert => { "cvss" => "float"} + convert => { "cvss3" => "float"} convert => { "cvss3_base" => "float"} convert => { "cvss3_temporal" => "float"} - convert => { "cvss3" => "float"} convert => { "id" => "integer"} convert => { "plugin_id" => "integer"} convert => { "risk_number" => "integer"} - convert => { "risk_score" => "float"} convert => { "total_times_detected" => "integer"} } - if [risk_score] == 0 { + if [cvss] == 0 { mutate { - add_field => { "risk_score_name" => "info" } + add_field => { "cvss_severity" => "info" } } } - if [risk_score] > 0 and [risk_score] < 3 { + if [cvss] > 0 and [cvss] < 3 { mutate { - add_field => { "risk_score_name" => "low" } + add_field => { "cvss_severity" => "low" } } } - if [risk_score] >= 3 and [risk_score] < 6 { + if [cvss] >= 3 and [cvss] < 6 { mutate { - add_field => { "risk_score_name" => "medium" } + add_field => { "cvss_severity" => "medium" } } } - if [risk_score] >=6 and [risk_score] < 9 { + if [cvss] >=6 and [cvss] < 9 { mutate { - add_field => { "risk_score_name" => "high" } + add_field => { "cvss_severity" => "high" } } } - if [risk_score] >= 9 { + if [cvss] >= 9 { mutate { - add_field => { "risk_score_name" => "critical" } + add_field => { "cvss_severity" => "critical" } } } diff --git a/resources/elk6/pipeline/3000_openvas.conf b/resources/elk6/pipeline/3000_openvas.conf index 0bf12c1..97b91ea 100644 --- a/resources/elk6/pipeline/3000_openvas.conf +++ b/resources/elk6/pipeline/3000_openvas.conf @@ -94,48 +94,44 @@ filter { } mutate { - add_field => { "risk_score" => "%{cvss}" } - } - - mutate { + convert => { "cvss" => "float"} convert => { "cvss_base" => "float"} convert => { "cvss_temporal" => "float"} - convert => { "cvss" => "float"} + convert => { "cvss3" => "float"} convert => { "cvss3_base" => "float"} convert => { "cvss3_temporal" => "float"} - convert => { "cvss3" => "float"} convert => { "id" => "integer"} convert => { "plugin_id" => "integer"} convert => { "risk_number" => "integer"} - convert => { "risk_score" => "float"} convert => { "total_times_detected" => "integer"} } - if [risk_score] == 0 { + if [cvss] == 0 { mutate { - add_field => { "risk_score_name" => "info" } + add_field => { "cvss_severity" => "info" } } } - if [risk_score] > 0 and [risk_score] < 3 { + if [cvss] > 0 and [cvss] < 3 { mutate { - add_field => { "risk_score_name" => "low" } + add_field => { "cvss_severity" => "low" } } } - if [risk_score] >= 3 and [risk_score] < 6 { + if [cvss] >= 3 and [cvss] < 6 { mutate { - add_field => { "risk_score_name" => "medium" } + add_field => { "cvss_severity" => "medium" } } } - if [risk_score] >=6 and [risk_score] < 9 { + if [cvss] >=6 and [cvss] < 9 { mutate { - add_field => { "risk_score_name" => "high" } + add_field => { "cvss_severity" => "high" } } } - if [risk_score] >= 9 { + if [cvss] >= 9 { mutate { - add_field => { "risk_score_name" => "critical" } + add_field => { "cvss_severity" => "critical" } } } + # Add your critical assets by subnet or by hostname. Comment this field out if you don't want to tag any, but the asset panel will break. if [asset] =~ "^10\.0\.100\." { mutate { diff --git a/vulnwhisp/frameworks/nessus.py b/vulnwhisp/frameworks/nessus.py index 11e4258..1d6de7f 100755 --- a/vulnwhisp/frameworks/nessus.py +++ b/vulnwhisp/frameworks/nessus.py @@ -217,6 +217,7 @@ class NessusAPI(object): self.logger.debug('Changing case of fields') df['cve'] = df['cve'].str.upper() df['protocol'] = df['protocol'].str.lower() + df['risk'] = df['risk'].str.lower() # Copy asset to IP df['ip'] = df['asset'] @@ -225,27 +226,5 @@ class NessusAPI(object): self.logger.debug('Mapping risk to severity number') df['risk_number'] = df['risk'].str.lower().map(self.SEVERITY_MAPPING) - if self.profile == 'tenable': - self.logger.debug('Combinging CVSS vectors for tenable') - # Combine CVSS vectors - df['cvss_vector'] = ( - df[['cvss_vector', 'cvss_temporal_vector']] - .apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1) - .str.rstrip('/nan') - ) - df['cvss3_vector'] = ( - df[['cvss3_vector', 'cvss3_temporal_vector']] - .apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1) - .str.rstrip('/nan') - ) - - df.drop(['cvss_temporal_vector', 'cvss3_temporal_vector'], axis=1, inplace=True) - - # CVSS score = cvss3_temporal or cvss3_base or cvss_temporal or cvss_base - df['cvss'] = df['cvss_base'] - df.loc[df['cvss_temporal'].notnull(), 'cvss'] = df['cvss_temporal'] - df['cvss3'] = df['cvss3_base'] - df.loc[df['cvss3_temporal'].notnull(), 'cvss3'] = df['cvss3_temporal'] - df.fillna('', inplace=True) return df \ No newline at end of file diff --git a/vulnwhisp/frameworks/qualys_vuln.py b/vulnwhisp/frameworks/qualys_vuln.py index 4cf21de..118d8f2 100644 --- a/vulnwhisp/frameworks/qualys_vuln.py +++ b/vulnwhisp/frameworks/qualys_vuln.py @@ -90,6 +90,8 @@ class qualysVulnScan: 'title': 'plugin_name' } + SEVERITY_MAPPING = {0: 'none', 1: 'low', 2: 'medium', 3: 'high',4: 'critical'} + def __init__( self, config=None, @@ -184,23 +186,9 @@ class qualysVulnScan: .apply(lambda x: x[0]) ) - # Combine base and temporal - df['cvss_vector'] = ( - df[['cvss_vector', 'cvss_temporal_vector']] - .apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1) - .str.rstrip('/nan') - ) - - df.drop('cvss_temporal_vector', axis=1, inplace=True) - # Convert Qualys severity to standardised risk number df['risk_number'] = df['severity'].astype(int)-1 - - # CVSS score = cvss3_temporal or cvss3_base or cvss_temporal or cvss_base - df['cvss'] = df['cvss_base'] - df.loc[df['cvss_temporal'].notnull(), 'cvss'] = df['cvss_temporal'] - df['cvss3'] = df['cvss3_base'] - df.loc[df['cvss3_temporal'].notnull(), 'cvss3'] = df['cvss3_temporal'] + df['risk'] = df['risk_number'].map(self.SEVERITY_MAPPING) df.fillna('', inplace=True) diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index cd73320..954e8ba 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -242,6 +242,52 @@ class vulnWhispererBase(object): scan_names = [] return results + def common_normalise(self, df): + """Map and transform common data values""" + self.logger.info('Start common mapping') + if 'cvss_base' in df: + self.logger.info('Normalising CVSS') + # CVSS = cvss_temporal or cvss_base + df['cvss'] = df['cvss_base'] + df.loc[df['cvss_temporal'].notnull(), 'cvss'] = df['cvss_temporal'] + # Map CVSS to severity name + df.loc[df['cvss'] == 0, 'cvss_severity'] = 'info' + df.loc[(df['cvss'] > 0) & (df['cvss'] < 3), 'cvss_severity'] = 'info' + df.loc[(df['cvss'] >= 3) & (df['cvss'] < 6), 'cvss_severity'] = 'medium' + df.loc[(df['cvss'] >= 6) & (df['cvss'] < 9), 'cvss_severity'] = 'high' + df.loc[df['cvss'] > 9, 'cvss_severity'] = 'critical' + + if 'cvss3_base' in df: + self.logger.info('Normalising CVSS3') + # CVSS3 = cvss3_temporal or cvss3_base + df['cvss3'] = df['cvss3_base'] + df.loc[df['cvss3_temporal'].notnull(), 'cvss3'] = df['cvss3_temporal'] + # Map CVSS to severity name + df.loc[df['cvss3'] == 0, 'cvss3_severity'] = 'info' + df.loc[(df['cvss3'] > 0) & (df['cvss3'] < 3), 'cvss3_severity'] = 'info' + df.loc[(df['cvss3'] >= 3) & (df['cvss3'] < 6), 'cvss3_severity'] = 'medium' + df.loc[(df['cvss3'] >= 6) & (df['cvss3'] < 9), 'cvss3_severity'] = 'high' + df.loc[df['cvss3'] > 9, 'cvss3_severity'] = 'critical' + + # Combine CVSS and CVSS3 vectors + if 'cvss_vector' in df and 'cvss_temporal_vector' in df: + self.logger.info('Normalising CVSS Vector') + df['cvss_vector'] = ( + df[['cvss_vector', 'cvss_temporal_vector']] + .apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1) + .str.rstrip('/nan') + ) + df.drop('cvss_temporal_vector', axis=1, inplace=True) + if 'cvss3_vector' in df and 'cvss3_temporal_vector' in df: + self.logger.info('Normalising CVSS Vector') + df['cvss3_vector'] = ( + df[['cvss3_vector', 'cvss3_temporal_vector']] + .apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1) + .str.rstrip('/nan') + ) + df.drop('cvss3_temporal_vector', axis=1, inplace=True) + return df + class vulnWhispererNessus(vulnWhispererBase): @@ -444,22 +490,23 @@ class vulnWhispererNessus(vulnWhispererBase): self.exit_code += 1 continue - clean_csv = pd.read_csv(io.StringIO(file_req.decode('utf-8'))) - if len(clean_csv) > 2: + vuln_ready = pd.read_csv(io.StringIO(file_req.decode('utf-8'))) + if len(vuln_ready) > 2: self.logger.info('Processing {}/{} for scan: {}'.format(scan_count, len(scan_list), scan_name.encode('utf8'))) # Map and transform fields - clean_csv = self.nessus.normalise(clean_csv) + vuln_ready = self.nessus.normalise(vuln_ready) + vuln_ready = self.common_normalise(vuln_ready) # Set common fields - clean_csv['scan_name'] = scan_name.encode('utf8') - clean_csv['scan_id'] = uuid + vuln_ready['scan_name'] = scan_name.encode('utf8') + vuln_ready['scan_id'] = uuid # Add timestamp and convert to milliseconds - clean_csv['_timestamp'] = norm_time - clean_csv['scan_source'] = self.CONFIG_SECTION + vuln_ready['_timestamp'] = norm_time + vuln_ready['scan_source'] = self.CONFIG_SECTION - clean_csv.to_json(relative_path_name, orient='records', lines=True) + vuln_ready.to_json(relative_path_name, orient='records', lines=True) record_meta = ( scan_name, @@ -467,14 +514,14 @@ class vulnWhispererNessus(vulnWhispererBase): norm_time, file_name, time.time(), - clean_csv.shape[0], + vuln_ready.shape[0], self.CONFIG_SECTION, uuid, 1, 0, ) self.record_insert(record_meta) - self.logger.info('{filename} records written to {path} '.format(filename=clean_csv.shape[0], + self.logger.info('{filename} records written to {path} '.format(filename=vuln_ready.shape[0], path=file_name.encode('utf8'))) else: record_meta = ( @@ -483,7 +530,7 @@ class vulnWhispererNessus(vulnWhispererBase): norm_time, file_name, time.time(), - clean_csv.shape[0], + vuln_ready.shape[0], self.CONFIG_SECTION, uuid, 1, @@ -623,6 +670,7 @@ class vulnWhispererQualys(vulnWhispererBase): vuln_ready = self.qualys_scan.process_data(path=self.write_path, file_id=str(generated_report_id)) # Map and transform fields vuln_ready = self.qualys_scan.normalise(vuln_ready) + vuln_ready = self.common_normalise(vuln_ready) # TODO remove the line below once normalising complete vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True) @@ -795,6 +843,7 @@ class vulnWhispererOpenVAS(vulnWhispererBase): vuln_ready = self.openvas_api.process_report(report_id=report_id) # Map and transform fields vuln_ready = self.openvas_api.normalise(vuln_ready) + vuln_ready = self.common_normalise(vuln_ready) # TODO move the following to the openvas_api.transform_values vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True) vuln_ready.port = vuln_ready.port.fillna(0).astype(int) @@ -900,6 +949,7 @@ class vulnWhispererQualysVuln(vulnWhispererBase): vuln_ready = self.qualys_scan.process_data(scan_id=report_id) # Map and transform fields vuln_ready = self.qualys_scan.normalise(vuln_ready) + vuln_ready = self.common_normalise(vuln_ready) # Set common fields vuln_ready['scan_name'] = scan_name.encode('utf8')