Move to a common normalisation function

resources/elk6/pipeline/1000_nessus_process_file.conf

@@ -40,59 +40,42 @@ filter {
       tag_on_failure => []
     }
 
-    translate {
-      field => "[risk]"
-      destination => "[risk_number]"
-      dictionary => {
-        "None" => 0
-        "Low" => 1
-        "Medium" => 2
-        "High" => 3
-        "Critical" => 4
-      }
-    }
-
-    mutate {
-      add_field => { "risk_score" => "%{cvss}" }
-    }
-
     mutate {
+      convert => { "cvss" => "float"}
       convert => { "cvss_base" => "float"}
       convert => { "cvss_temporal" => "float"}
-      convert => { "cvss" => "float"}
+      convert => { "cvss3" => "float"}
       convert => { "cvss3_base" => "float"}
       convert => { "cvss3_temporal" => "float"}
-      convert => { "cvss3" => "float"}
       convert => { "id" => "integer"}
       convert => { "plugin_id" => "integer"}
       convert => { "risk_number" => "integer"}
-      convert => { "risk_score" => "float"}
       convert => { "total_times_detected" => "integer"}
     }
 
-    if [risk_score] == 0 {
+    if [cvss] == 0 {
       mutate {
-        add_field => { "risk_score_name" => "info" }
+        add_field => { "cvss_severity" => "info" }
       }
     }
-    if [risk_score] > 0 and [risk_score] < 3 {
+    if [cvss] > 0 and [cvss] < 3 {
       mutate {
-        add_field => { "risk_score_name" => "low" }
+        add_field => { "cvss_severity" => "low" }
       }
     }
-    if [risk_score] >= 3 and [risk_score] < 6 {
+    if [cvss] >= 3 and [cvss] < 6 {
       mutate {
-        add_field => { "risk_score_name" => "medium" }
+        add_field => { "cvss_severity" => "medium" }
       }
     }
-    if [risk_score] >=6 and [risk_score] < 9 {
+    if [cvss] >=6 and [cvss] < 9 {
       mutate {
-        add_field => { "risk_score_name" => "high" }
+        add_field => { "cvss_severity" => "high" }
       }
     }
-    if [risk_score] >= 9 {
+    if [cvss] >= 9 {
       mutate {
-        add_field => { "risk_score_name" => "critical" }
+        add_field => { "cvss_severity" => "critical" }
       }
     }
   }

resources/elk6/pipeline/2000_qualys_web_scans.conf

@@ -30,18 +30,6 @@ filter {
       tag_on_failure => []
     }
 
-    translate {
-      field => "[risk_number]"
-      destination => "[risk]"
-      dictionary => {
-        "0" => "Info"
-        "1" => "Low"
-        "2" => "Medium"
-        "3" => "High"
-        "4" => "Critical"
-      }
-    }
-
     if "qualys_web" in [tags] {
       mutate {
         add_field => { "asset" => "%{web_application_name}" }
@@ -49,46 +37,41 @@ filter {
     }
 
     mutate {
-      add_field => { "risk_score" => "%{cvss}" }
+      convert => { "cvss" => "float"}
-    }
-
-    mutate {
       convert => { "cvss_base" => "float"}
       convert => { "cvss_temporal" => "float"}
-      convert => { "cvss" => "float"}
+      convert => { "cvss3" => "float"}
       convert => { "cvss3_base" => "float"}
       convert => { "cvss3_temporal" => "float"}
-      convert => { "cvss3" => "float"}
       convert => { "id" => "integer"}
       convert => { "plugin_id" => "integer"}
       convert => { "risk_number" => "integer"}
-      convert => { "risk_score" => "float"}
       convert => { "total_times_detected" => "integer"}
     }
 
-    if [risk_score] == 0 {
+    if [cvss] == 0 {
       mutate {
-        add_field => { "risk_score_name" => "info" }
+        add_field => { "cvss_severity" => "info" }
       }
     }
-    if [risk_score] > 0 and [risk_score] < 3 {
+    if [cvss] > 0 and [cvss] < 3 {
       mutate {
-        add_field => { "risk_score_name" => "low" }
+        add_field => { "cvss_severity" => "low" }
       }
     }
-    if [risk_score] >= 3 and [risk_score] < 6 {
+    if [cvss] >= 3 and [cvss] < 6 {
       mutate {
-        add_field => { "risk_score_name" => "medium" }
+        add_field => { "cvss_severity" => "medium" }
       }
     }
-    if [risk_score] >=6 and [risk_score] < 9 {
+    if [cvss] >=6 and [cvss] < 9 {
       mutate {
-        add_field => { "risk_score_name" => "high" }
+        add_field => { "cvss_severity" => "high" }
       }
     }
-    if [risk_score] >= 9 {
+    if [cvss] >= 9 {
       mutate {
-        add_field => { "risk_score_name" => "critical" }
+        add_field => { "cvss_severity" => "critical" }
       }
     }
 

resources/elk6/pipeline/3000_openvas.conf

@@ -94,48 +94,44 @@ filter {
     }
 
     mutate {
-      add_field => { "risk_score" => "%{cvss}" }
+      convert => { "cvss" => "float"}
-    }
-
-    mutate {
       convert => { "cvss_base" => "float"}
       convert => { "cvss_temporal" => "float"}
-      convert => { "cvss" => "float"}
+      convert => { "cvss3" => "float"}
       convert => { "cvss3_base" => "float"}
       convert => { "cvss3_temporal" => "float"}
-      convert => { "cvss3" => "float"}
       convert => { "id" => "integer"}
       convert => { "plugin_id" => "integer"}
       convert => { "risk_number" => "integer"}
-      convert => { "risk_score" => "float"}
       convert => { "total_times_detected" => "integer"}
     }
 
-    if [risk_score] == 0 {
+    if [cvss] == 0 {
       mutate {
-        add_field => { "risk_score_name" => "info" }
+        add_field => { "cvss_severity" => "info" }
       }
     }
-    if [risk_score] > 0 and [risk_score] < 3 {
+    if [cvss] > 0 and [cvss] < 3 {
       mutate {
-        add_field => { "risk_score_name" => "low" }
+        add_field => { "cvss_severity" => "low" }
       }
     }
-    if [risk_score] >= 3 and [risk_score] < 6 {
+    if [cvss] >= 3 and [cvss] < 6 {
       mutate {
-        add_field => { "risk_score_name" => "medium" }
+        add_field => { "cvss_severity" => "medium" }
       }
     }
-    if [risk_score] >=6 and [risk_score] < 9 {
+    if [cvss] >=6 and [cvss] < 9 {
       mutate {
-        add_field => { "risk_score_name" => "high" }
+        add_field => { "cvss_severity" => "high" }
       }
     }
-    if [risk_score] >= 9 {
+    if [cvss] >= 9 {
       mutate {
-        add_field => { "risk_score_name" => "critical" }
+        add_field => { "cvss_severity" => "critical" }
       }
     }
+
     # Add your critical assets by subnet or by hostname. Comment this field out if you don't want to tag any, but the asset panel will break.
     if [asset] =~ "^10\.0\.100\." {
       mutate {

vulnwhisp/frameworks/nessus.py

@@ -217,6 +217,7 @@ class NessusAPI(object):
         self.logger.debug('Changing case of fields')
         df['cve'] = df['cve'].str.upper()
         df['protocol'] = df['protocol'].str.lower()
+        df['risk'] = df['risk'].str.lower()
 
         # Copy asset to IP
         df['ip'] = df['asset']
@@ -225,27 +226,5 @@ class NessusAPI(object):
         self.logger.debug('Mapping risk to severity number')
         df['risk_number'] = df['risk'].str.lower().map(self.SEVERITY_MAPPING)
 
-        if self.profile == 'tenable':
-            self.logger.debug('Combinging CVSS vectors for tenable')
-            # Combine CVSS vectors
-            df['cvss_vector'] = (
-                df[['cvss_vector', 'cvss_temporal_vector']]
-                .apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1)
-                .str.rstrip('/nan')
-            )
-            df['cvss3_vector'] = (
-                df[['cvss3_vector', 'cvss3_temporal_vector']]
-                .apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1)
-                .str.rstrip('/nan')
-            )
-
-            df.drop(['cvss_temporal_vector', 'cvss3_temporal_vector'], axis=1, inplace=True)
-
-            # CVSS score = cvss3_temporal or cvss3_base or cvss_temporal or cvss_base
-            df['cvss'] = df['cvss_base']
-            df.loc[df['cvss_temporal'].notnull(), 'cvss'] = df['cvss_temporal']
-            df['cvss3'] = df['cvss3_base']
-            df.loc[df['cvss3_temporal'].notnull(), 'cvss3'] = df['cvss3_temporal']
-
         df.fillna('', inplace=True)
         return df

vulnwhisp/frameworks/qualys_vuln.py

@@ -90,6 +90,8 @@ class qualysVulnScan:
         'title': 'plugin_name'
     }
 
+    SEVERITY_MAPPING = {0: 'none', 1: 'low', 2: 'medium', 3: 'high',4: 'critical'}
+
     def __init__(
         self,
         config=None,
@@ -184,23 +186,9 @@ class qualysVulnScan:
             .apply(lambda x: x[0])
         )
 
-        # Combine base and temporal
-        df['cvss_vector'] = (
-            df[['cvss_vector', 'cvss_temporal_vector']]
-            .apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1)
-            .str.rstrip('/nan')
-        )
-
-        df.drop('cvss_temporal_vector', axis=1, inplace=True)
-
         # Convert Qualys severity to standardised risk number
         df['risk_number'] =  df['severity'].astype(int)-1
-
+        df['risk'] = df['risk_number'].map(self.SEVERITY_MAPPING)
-        # CVSS score = cvss3_temporal or cvss3_base or cvss_temporal or cvss_base
-        df['cvss'] = df['cvss_base']
-        df.loc[df['cvss_temporal'].notnull(), 'cvss'] = df['cvss_temporal']
-        df['cvss3'] = df['cvss3_base']
-        df.loc[df['cvss3_temporal'].notnull(), 'cvss3'] = df['cvss3_temporal']
 
         df.fillna('', inplace=True)
 

vulnwhisp/vulnwhisp.py

@@ -242,6 +242,52 @@ class vulnWhispererBase(object):
                 scan_names = []
 
         return results
+    def common_normalise(self, df):
+        """Map and transform common data values""" 
+        self.logger.info('Start common mapping')
+        if 'cvss_base' in df:
+            self.logger.info('Normalising CVSS')
+            # CVSS = cvss_temporal or cvss_base
+            df['cvss'] = df['cvss_base']
+            df.loc[df['cvss_temporal'].notnull(), 'cvss'] = df['cvss_temporal']
+            # Map CVSS to severity name
+            df.loc[df['cvss'] == 0, 'cvss_severity'] = 'info'
+            df.loc[(df['cvss'] > 0) & (df['cvss'] < 3), 'cvss_severity'] = 'info'
+            df.loc[(df['cvss'] >= 3) & (df['cvss'] < 6), 'cvss_severity'] = 'medium'
+            df.loc[(df['cvss'] >= 6) & (df['cvss'] < 9), 'cvss_severity'] = 'high'
+            df.loc[df['cvss'] > 9, 'cvss_severity'] = 'critical'
+
+        if 'cvss3_base' in df:
+            self.logger.info('Normalising CVSS3')
+            # CVSS3 = cvss3_temporal or cvss3_base
+            df['cvss3'] = df['cvss3_base']
+            df.loc[df['cvss3_temporal'].notnull(), 'cvss3'] = df['cvss3_temporal']  
+            # Map CVSS to severity name
+            df.loc[df['cvss3'] == 0, 'cvss3_severity'] = 'info'
+            df.loc[(df['cvss3'] > 0) & (df['cvss3'] < 3), 'cvss3_severity'] = 'info'
+            df.loc[(df['cvss3'] >= 3) & (df['cvss3'] < 6), 'cvss3_severity'] = 'medium'
+            df.loc[(df['cvss3'] >= 6) & (df['cvss3'] < 9), 'cvss3_severity'] = 'high'
+            df.loc[df['cvss3'] > 9, 'cvss3_severity'] = 'critical'
+
+        # Combine CVSS and CVSS3 vectors
+        if 'cvss_vector' in df and 'cvss_temporal_vector' in df:
+            self.logger.info('Normalising CVSS Vector')
+            df['cvss_vector'] = (
+                df[['cvss_vector', 'cvss_temporal_vector']]
+                .apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1)
+                .str.rstrip('/nan')
+            )
+            df.drop('cvss_temporal_vector', axis=1, inplace=True)
+        if 'cvss3_vector' in df and 'cvss3_temporal_vector' in df:
+            self.logger.info('Normalising CVSS Vector')
+            df['cvss3_vector'] = (
+                df[['cvss3_vector', 'cvss3_temporal_vector']]
+                .apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1)
+                .str.rstrip('/nan')
+            )
+            df.drop('cvss3_temporal_vector', axis=1, inplace=True)
+        return df
+
 
 class vulnWhispererNessus(vulnWhispererBase):
 
@@ -444,22 +490,23 @@ class vulnWhispererNessus(vulnWhispererBase):
                             self.exit_code += 1
                             continue
                             
-                        clean_csv = pd.read_csv(io.StringIO(file_req.decode('utf-8')))
+                        vuln_ready = pd.read_csv(io.StringIO(file_req.decode('utf-8')))
-                        if len(clean_csv) > 2:
+                        if len(vuln_ready) > 2:
                             self.logger.info('Processing {}/{} for scan: {}'.format(scan_count, len(scan_list), scan_name.encode('utf8')))
 
                             # Map and transform fields
-                            clean_csv = self.nessus.normalise(clean_csv)
+                            vuln_ready = self.nessus.normalise(vuln_ready)
+                            vuln_ready = self.common_normalise(vuln_ready)
 
                             # Set common fields
-                            clean_csv['scan_name'] = scan_name.encode('utf8')
+                            vuln_ready['scan_name'] = scan_name.encode('utf8')
-                            clean_csv['scan_id'] = uuid
+                            vuln_ready['scan_id'] = uuid
 
                             # Add timestamp and convert to milliseconds
-                            clean_csv['_timestamp'] = norm_time
+                            vuln_ready['_timestamp'] = norm_time
-                            clean_csv['scan_source'] = self.CONFIG_SECTION
+                            vuln_ready['scan_source'] = self.CONFIG_SECTION
 
-                            clean_csv.to_json(relative_path_name, orient='records', lines=True)
+                            vuln_ready.to_json(relative_path_name, orient='records', lines=True)
 
                             record_meta = (
                                 scan_name,
@@ -467,14 +514,14 @@ class vulnWhispererNessus(vulnWhispererBase):
                                 norm_time,
                                 file_name,
                                 time.time(),
-                                clean_csv.shape[0],
+                                vuln_ready.shape[0],
                                 self.CONFIG_SECTION,
                                 uuid,
                                 1,
                                 0,
                             )
                             self.record_insert(record_meta)
-                            self.logger.info('{filename} records written to {path} '.format(filename=clean_csv.shape[0],
+                            self.logger.info('{filename} records written to {path} '.format(filename=vuln_ready.shape[0],
                                                                                             path=file_name.encode('utf8')))
                         else:
                             record_meta = (
@@ -483,7 +530,7 @@ class vulnWhispererNessus(vulnWhispererBase):
                                 norm_time,
                                 file_name,
                                 time.time(),
-                                clean_csv.shape[0],
+                                vuln_ready.shape[0],
                                 self.CONFIG_SECTION,
                                 uuid,
                                 1,
@@ -623,6 +670,7 @@ class vulnWhispererQualys(vulnWhispererBase):
                     vuln_ready = self.qualys_scan.process_data(path=self.write_path, file_id=str(generated_report_id))
                     # Map and transform fields
                     vuln_ready = self.qualys_scan.normalise(vuln_ready)
+                    vuln_ready = self.common_normalise(vuln_ready)
                     # TODO remove the line below once normalising complete
                     vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
 
@@ -795,6 +843,7 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
                 vuln_ready = self.openvas_api.process_report(report_id=report_id)
                 # Map and transform fields
                 vuln_ready = self.openvas_api.normalise(vuln_ready)
+                vuln_ready = self.common_normalise(vuln_ready)
                 # TODO move the following to the openvas_api.transform_values 
                 vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
                 vuln_ready.port = vuln_ready.port.fillna(0).astype(int)
@@ -900,6 +949,7 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
                     vuln_ready = self.qualys_scan.process_data(scan_id=report_id)
                     # Map and transform fields
                     vuln_ready = self.qualys_scan.normalise(vuln_ready)
+                    vuln_ready = self.common_normalise(vuln_ready)
 
                     # Set common fields
                     vuln_ready['scan_name'] = scan_name.encode('utf8')