Remove nessus csv completely

resources/elk5-old_compatibility/docker/1000_nessus_process_file.conf

@@ -6,12 +6,6 @@
 
 
 input {
-  file {
-    path => "/opt/VulnWhisperer/data/nessus/**/*.csv"
-    start_position => "beginning"
-    tags => "nessus"
-    type => "nessus"
-  }
   file {
     path => "/opt/VulnWhisperer/data/nessus/**/*.json"
     start_position => "beginning"
@@ -19,12 +13,6 @@ input {
     type => "nessus"
     codec => json
   }
-  file {
-    path => "/opt/VulnWhisperer/data/tenable/*.csv"
-    start_position => "beginning"
-    tags => "tenable"
-    type => "tenable"
-  }
   file {
     path => "/opt/VulnWhisperer/data/tenable/*.json"
     start_position => "beginning"
@@ -37,36 +25,6 @@ input {
 filter {
   if "nessus" in [tags] or "tenable" in [tags] {
 
-    if [path] =~ /\.csv$/ {}
-      # Drop the header column
-      if [message] =~ "^(Plugin ID|plugin_id,)" { drop {} }
-
-      csv {
-        # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"]
-        columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"]
-        separator => ","
-        source => "message"
-      }
-
-      ruby {
-          code => "if event.get('description')
-                      event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                  end
-                  if event.get('synopsis')
-                      event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                  end
-                  if event.get('solution')
-                      event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                  end
-                  if event.get('see_also')
-                      event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                  end
-                  if event.get('plugin_output')
-                      event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                  end"
-      }
-  }
-
     #If using filebeats as your source, you will need to replace the "path" field to "source"
     grok {
       match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.(csv|json)$" }

resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf

@@ -6,12 +6,6 @@
 
 
 input {
-  file {
-    path => "/opt/VulnWhisperer/data/nessus/**/*.csv"
-    start_position => "beginning"
-    tags => "nessus"
-    type => "nessus"
-  }
   file {
     path => "/opt/VulnWhisperer/data/nessus/**/*.json"
     start_position => "beginning"
@@ -19,12 +13,6 @@ input {
     type => "nessus"
     codec => json
   }
-  file {
-    path => "/opt/VulnWhisperer/data/tenable/*.csv"
-    start_position => "beginning"
-    tags => "tenable"
-    type => "tenable"
-  }
   file {
     path => "/opt/VulnWhisperer/data/tenable/*.json"
     start_position => "beginning"
@@ -37,36 +25,6 @@ input {
 filter {
   if "nessus" in [tags] or "tenable" in [tags] {
 
-    if [path] =~ /\.csv$/ {}
-      # Drop the header column
-      if [message] =~ "^(Plugin ID|plugin_id,)" { drop {} }
-
-      csv {
-        # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"]
-        columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"]
-        separator => ","
-        source => "message"
-      }
-
-      ruby {
-          code => "if event.get('description')
-                      event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                  end
-                  if event.get('synopsis')
-                      event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                  end
-                  if event.get('solution')
-                      event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                  end
-                  if event.get('see_also')
-                      event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                  end
-                  if event.get('plugin_output')
-                      event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                  end"
-      }
-    }
-
     #If using filebeats as your source, you will need to replace the "path" field to "source"
     grok {
       match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.(csv|json)$" }

resources/elk6/pipeline/1000_nessus_process_file.conf

@@ -6,13 +6,6 @@
 
 
 input {
-  file {
-    path => "/opt/vulnwhisperer/data/nessus/**/*.csv"
-    mode => "read"
-    start_position => "beginning"
-    file_completed_action => "delete"
-    tags => "nessus"
-  }
   file {
     path => "/opt/VulnWhisperer/data/nessus/**/*.json"
     start_position => "beginning"
@@ -20,13 +13,6 @@ input {
     tags => "nessus"
     codec => json
   }
-  file {
-    path => "/opt/vulnwhisperer/data/tenable/*.csv"
-    mode => "read"
-    start_position => "beginning"
-    file_completed_action => "delete"
-    tags => "tenable"
-  }
   file {
     path => "/opt/VulnWhisperer/data/tenable/*.json"
     start_position => "beginning"
@@ -39,36 +25,6 @@ input {
 filter {
   if "nessus" in [tags] or "tenable" in [tags] {
 
-    if [path] =~ /\.csv$/ {}
-      # Drop the header column
-      if [message] =~ "^(Plugin ID|plugin_id,)" { drop {} }
-
-      csv {
-        # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"]
-        columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"]
-        separator => ","
-        source => "message"
-      }
-
-      ruby {
-          code => "if event.get('description')
-                      event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                  end
-                  if event.get('synopsis')
-                      event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                  end
-                  if event.get('solution')
-                      event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                  end
-                  if event.get('see_also')
-                      event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                  end
-                  if event.get('plugin_output')
-                      event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                  end"
-      }
-    }
-
     #If using filebeats as your source, you will need to replace the "path" field to "source"
     grok {
       match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.(csv|json)$" }

vulnwhisp/vulnwhisp.py

@@ -406,7 +406,7 @@ class vulnWhispererNessus(vulnWhispererBase):
                     folder_name = next(f['name'] for f in folders if f['id'] == folder_id)
                 if status in ['completed', 'imported']:
                     file_name = '%s_%s_%s_%s.%s' % (scan_name, scan_id,
-                                                    history_id, norm_time, 'csv')
+                                                    history_id, norm_time, 'json')
                     repls = (('\\', '_'), ('/', '_'), (' ', '_'))
                     file_name = reduce(lambda a, kv: a.replace(*kv), repls, file_name)
                     relative_path_name = self.path_check(folder_name + '/' + file_name)
@@ -448,8 +448,8 @@ class vulnWhispererNessus(vulnWhispererBase):
                             clean_csv['scan_name'] = scan_name.encode('utf8')
                             clean_csv['scan_id'] = uuid
 
-                            clean_csv.to_json(relative_path_name.replace('csv', 'json'), orient='records', lines=True)
+                            clean_csv.to_json(relative_path_name, orient='records', lines=True)
-                            clean_csv.to_csv(relative_path_name, index=False)
+
                             record_meta = (
                                 scan_name,
                                 scan_id,