Add normalisation, mapping and transform stubs

vulnwhisp/frameworks/openvas.py

@@ -190,3 +190,17 @@ class OpenVAS_API(object):
         self.processed_reports += 1
         merged_df = pd.merge(report_df, self.openvas_reports, on='report_ids').reset_index().drop('index', axis=1)
         return merged_df
+
+    def normalise(self, dataframe):
+        self.logger.debug('Normalising data')
+        self.map_fields(dataframe)
+        self.transform_values(dataframe)
+        return dataframe
+
+    def map_fields(self, dataframe):
+        self.logger.debug('Mapping fields')
+        return dataframe
+    
+    def transform_values(self, dataframe):
+        self.logger.debug('Transforming values')
+        return dataframe

vulnwhisp/frameworks/qualys_web.py

@@ -463,3 +463,17 @@ class qualysScanReport:
         merged_data.sort_index(axis=1, inplace=True)
 
         return merged_data
+
+    def normalise(self, dataframe):
+        self.logger.debug('Normalising data')
+        self.map_fields(dataframe)
+        self.transform_values(dataframe)
+        return dataframe
+
+    def map_fields(self, dataframe):
+        self.logger.debug('Mapping fields')
+        return dataframe
+    
+    def transform_values(self, dataframe):
+        self.logger.debug('Transforming values')
+        return dataframe

vulnwhisp/vulnwhisp.py

@@ -610,6 +610,9 @@ class vulnWhispererQualys(vulnWhispererBase):
 
                     vuln_ready['scan_name'] = scan_name
                     vuln_ready['scan_reference'] = scan_reference
+                    # Map and transform fields
+                    vuln_ready = self.qualys_scan.normalise(vuln_ready)
+                    # TODO remove the line below once normalising complete
                     vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
 
                     record_meta = (
@@ -627,9 +630,7 @@ class vulnWhispererQualys(vulnWhispererBase):
                     self.record_insert(record_meta)
 
                     if output_format == 'json':
-                        with open(relative_path_name, 'w') as f:
+                        vuln_ready.to_json(relative_path_name, orient='records', lines=True)
-                            f.write(vuln_ready.to_json(orient='records', lines=True))
-                            f.write('\n')
 
                     elif output_format == 'csv':
                        vuln_ready.to_csv(relative_path_name, index=False, header=True)  # add when timestamp occured
@@ -779,10 +780,9 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
                 vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
                 vuln_ready.port = vuln_ready.port.fillna(0).astype(int)
                 vuln_ready.fillna('', inplace=True)
-                if output_format == 'json':
+                # Map and transform fields
-                    with open(relative_path_name, 'w') as f:
+                vuln_ready = self.openvas_api.normalise(vuln_ready)
-                        f.write(vuln_ready.to_json(orient='records', lines=True))
+                vuln_ready.to_json(relative_path_name, orient='records', lines=True)
-                        f.write('\n')
                 self.logger.info('Report written to {}'.format(report_name))
 
         return report