Nessus JSON output with normalisation

vulnwhisp/frameworks/nessus.py

@@ -23,8 +23,24 @@ class NessusAPI(object):
     EXPORT_FILE_DOWNLOAD = EXPORT + '/{file_id}/download'
     EXPORT_STATUS = EXPORT + '/{file_id}/status'
     EXPORT_HISTORY = EXPORT + '?history_id={history_id}'
+    # All column mappings should be lowercase
+    COLUMN_MAPPING = {
+        'cvss base score': 'cvss',
+        'cvss temporal score': 'cvss_temporal',
+        'cvss temporal vector': 'cvss_temporal_vector',
+        'cvss3 base score': 'cvss3',
+        'cvss3 temporal score': 'cvss3_temporal',
+        'cvss3 temporal vector': 'cvss3_temporal_vector',
+        'fqdn': 'dns',
+        'host': 'asset',
+        'name': 'plugin_name',
+        'os': 'operating_system',
+        'system type': 'category',
+        'vulnerability state': 'state'
+    }
+    SEVERITY_MAPPING = {'none': 0, 'low': 1, 'medium': 2, 'high': 3, 'critical': 4}
 
-    def __init__(self, hostname=None, port=None, username=None, password=None, verbose=True):
+    def __init__(self, hostname=None, port=None, username=None, password=None, verbose=True, profile=None):
         self.logger = logging.getLogger('NessusAPI')
         if verbose:
             self.logger.setLevel(logging.DEBUG)
@@ -35,6 +51,7 @@ class NessusAPI(object):
         self.password = password
         self.base = 'https://{hostname}:{port}'.format(hostname=hostname, port=port)
         self.verbose = verbose
+        self.profile = profile
 
         self.session = requests.Session()
         self.session.verify = False
@@ -114,7 +131,7 @@ class NessusAPI(object):
         data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json_output=True)
         return data['history']
 
-    def download_scan(self, scan_id=None, history=None, export_format="", profile=""):
+    def download_scan(self, scan_id=None, history=None, export_format=''):
         running = True
         counter = 0
 
@@ -137,13 +154,13 @@ class NessusAPI(object):
             report_status = self.request(self.EXPORT_STATUS.format(scan_id=scan_id, file_id=file_id), method='GET',
                                          json_output=True)
             running = report_status['status'] != 'ready'
-            sys.stdout.write(".")
+            sys.stdout.write('.')
             sys.stdout.flush()
             # FIXME: why? can this be removed in favour of a counter?
             if counter % 60 == 0:
-                self.logger.info("Completed: {}".format(counter))
+                self.logger.info('Completed: {}'.format(counter))
-        self.logger.info("Done: {}".format(counter))
+        self.logger.info('Done: {}'.format(counter))
-        if profile == 'tenable':
+        if self.profile == 'tenable':
             content = self.request(self.EXPORT_FILE_DOWNLOAD.format(scan_id=scan_id, file_id=file_id), method='GET', download=True)
         else:
             content = self.request(self.EXPORT_TOKEN_DOWNLOAD.format(token_id=token_id), method='GET', download=True)
@@ -169,3 +186,62 @@ class NessusAPI(object):
                     'Pacific Standard Time': 'US/Pacific',
                     'None': 'US/Central'}
         return time_map.get(tz, None)
+
+    def normalise(self, dataframe):
+        self.logger.debug('Normalising data')
+        self.map_fields(dataframe)
+        self.transform_values(dataframe)
+        return dataframe
+
+    def map_fields(self, dataframe):
+        self.logger.debug('Mapping fields')
+
+        # Any specific mappings here
+        if self.profile == 'tenable':
+            # Prefer CVSS Base Score over CVSS for tenable
+            self.logger.debug('Dropping redundant tenable fields')
+            dataframe.drop('CVSS', axis=1, inplace=True)
+            dataframe.drop('IP Address', axis=1, inplace=True)
+
+        # Map fields from COLUMN_MAPPING
+        fields = [x.lower() for x in dataframe.columns]
+        for field, replacement in self.COLUMN_MAPPING.iteritems():
+            if field in fields:
+                self.logger.debug('Renaming "{}" to "{}"'.format(field, replacement))
+                fields[fields.index(field)] = replacement
+
+        fields = [x.replace(' ', '_') for x in fields]
+        dataframe.columns = fields
+
+        return dataframe
+    
+    def transform_values(self, dataframe):
+        self.logger.debug('Transforming values')
+
+        # upper/lowercase fields
+        self.logger.debug('Changing case of fields')
+        dataframe['cve'] = dataframe['cve'].str.upper()
+        dataframe['protocol'] = dataframe['protocol'].str.lower()
+
+        # Map risk to a SEVERITY MAPPING value
+        self.logger.debug('Mapping risk to severity number')
+        dataframe['risk_number'] = dataframe['risk'].str.lower()
+        dataframe['risk_number'].replace(self.SEVERITY_MAPPING, inplace=True)
+
+        if self.profile == 'tenable':
+            self.logger.debug('Combinging CVSS vectors for tenable')
+            # Combine CVSS vectors
+            dataframe['cvss_vector'] = (
+                dataframe[['cvss_vector', 'cvss_temporal_vector']]
+                .apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1)
+                .str.rstrip('/nan')
+            )
+            dataframe['cvss3_vector'] = (
+                dataframe[['cvss3_vector', 'cvss3_temporal_vector']]
+                .apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1)
+                .str.rstrip('/nan')
+            )
+
+        dataframe.fillna('', inplace=True)
+
+        return dataframe

vulnwhisp/vulnwhisp.py

@@ -129,11 +129,6 @@ class vulnWhispererBase(object):
             self.delete_table()
         self.create_table()
 
-    def cleanser(self, _data):
-        repls = (('\n', r'\n'), ('\r', r'\r'))
-        data = reduce(lambda a, kv: a.replace(*kv), repls, _data)
-        return data
-
     def path_check(self, _data):
         if self.write_path:
             if '/' or '\\' in _data[-1]:
@@ -288,7 +283,9 @@ class vulnWhispererNessus(vulnWhispererBase):
                         NessusAPI(hostname=self.hostname,
                                   port=self.nessus_port,
                                   username=self.username,
-                                  password=self.password)
+                                  password=self.password,
+                                  profile=self.CONFIG_SECTION
+                                  )
                     self.nessus_connect = True
                     self.logger.info('Connected to nessus on {host}:{port}'.format(host=self.hostname,
                                                                                    port=str(self.nessus_port)))
@@ -435,21 +432,20 @@ class vulnWhispererNessus(vulnWhispererBase):
                         try:
                             file_req = \
                                 self.nessus.download_scan(scan_id=scan_id, history=history_id,
-                                                        export_format='csv', profile=self.CONFIG_SECTION)
+                                                        export_format='csv')
                         except Exception as e:
                             self.logger.error('Could not download {} scan {}: {}'.format(self.CONFIG_SECTION, scan_id, str(e)))
                             self.exit_code += 1
                             continue
                             
-                        clean_csv = \
+                        clean_csv = pd.read_csv(io.StringIO(file_req.decode('utf-8')))
-                            pd.read_csv(io.StringIO(file_req.decode('utf-8')))
                         if len(clean_csv) > 2:
                             self.logger.info('Processing {}/{} for scan: {}'.format(scan_count, len(scan_list), scan_name.encode('utf8')))
-                            columns_to_cleanse = ['CVSS','CVE','Description','Synopsis','Solution','See Also','Plugin Output']
 
-                            for col in columns_to_cleanse:
+                            # Map and transform fields
-                                clean_csv[col] = clean_csv[col].astype(str).apply(self.cleanser)
+                            clean_csv = self.nessus.normalise(clean_csv)
 
+                            clean_csv.to_json(relative_path_name.replace('csv', 'json'), orient='records', lines=True)
                             clean_csv.to_csv(relative_path_name, index=False)
                             record_meta = (
                                 scan_name,