Update nessus logstash configs

resources/elk5-old_compatibility/docker/1000_nessus_process_file.conf

@@ -7,27 +7,43 @@
 
 input {
   file {
-    path => "/opt/VulnWhisperer/data/nessus/**/*"
+    path => "/opt/VulnWhisperer/data/nessus/**/*.csv"
     start_position => "beginning"
     tags => "nessus"
     type => "nessus"
   }
+  file {
+    path => "/opt/VulnWhisperer/data/nessus/**/*.json"
+    start_position => "beginning"
+    tags => "nessus"
+    type => "nessus"
+    codec => json
+  }
   file {
     path => "/opt/VulnWhisperer/data/tenable/*.csv"
     start_position => "beginning"
     tags => "tenable"
     type => "tenable"
   }
+  file {
+    path => "/opt/VulnWhisperer/data/tenable/*.json"
+    start_position => "beginning"
+    tags => "nessus"
+    type => "nessus"
+    codec => json
+  }
 }
 
 filter {
   if "nessus" in [tags] or "tenable" in [tags] {
+
+    if [path] =~ /\.csv$/ {}
       # Drop the header column
-    if [message] =~ "^Plugin ID" { drop {} }
+      if [message] =~ "^(Plugin ID|plugin_id,)" { drop {} }
 
       csv {
         # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"]
-      columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss_vector", "system_type", "host_start", "host_end"]
+        columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"]
         separator => ","
         source => "message"
       }
@@ -49,10 +65,11 @@ filter {
                       event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
                   end"
       }
+  }
 
     #If using filebeats as your source, you will need to replace the "path" field to "source"
     grok {
-      match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.csv$" }
+      match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.(csv|json)$" }
       tag_on_failure => []
     }
 

resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf

@@ -7,23 +7,39 @@
 
 input {
   file {
-    path => "/opt/vulnwhisperer/nessus/**/*"
+    path => "/opt/VulnWhisperer/data/nessus/**/*.csv"
     start_position => "beginning"
     tags => "nessus"
     type => "nessus"
   }
   file {
-    path => "/opt/vulnwhisperer/tenable/*.csv"
+    path => "/opt/VulnWhisperer/data/nessus/**/*.json"
+    start_position => "beginning"
+    tags => "nessus"
+    type => "nessus"
+    codec => json
+  }
+  file {
+    path => "/opt/VulnWhisperer/data/tenable/*.csv"
     start_position => "beginning"
     tags => "tenable"
     type => "tenable"
   }
+  file {
+    path => "/opt/VulnWhisperer/data/tenable/*.json"
+    start_position => "beginning"
+    tags => "nessus"
+    type => "nessus"
+    codec => json
+  }
 }
 
 filter {
   if "nessus" in [tags] or "tenable" in [tags] {
+
+    if [path] =~ /\.csv$/ {}
       # Drop the header column
-    if [message] =~ "^Plugin ID" { drop {} }
+      if [message] =~ "^(Plugin ID|plugin_id,)" { drop {} }
 
       csv {
         # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"]
@@ -49,10 +65,11 @@ filter {
                       event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
                   end"
       }
+    }
 
     #If using filebeats as your source, you will need to replace the "path" field to "source"
     grok {
-      match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.csv$" }
+      match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.(csv|json)$" }
       tag_on_failure => []
     }
 

resources/elk6/pipeline/1000_nessus_process_file.conf

@@ -7,12 +7,19 @@
 
 input {
   file {
-    path => "/opt/vulnwhisperer/data/nessus/**/*"
+    path => "/opt/vulnwhisperer/data/nessus/**/*.csv"
     mode => "read"
     start_position => "beginning"
     file_completed_action => "delete"
     tags => "nessus"
   }
+  file {
+    path => "/opt/VulnWhisperer/data/nessus/**/*.json"
+    start_position => "beginning"
+    file_completed_action => "delete"
+    tags => "nessus"
+    codec => json
+  }
   file {
     path => "/opt/vulnwhisperer/data/tenable/*.csv"
     mode => "read"
@@ -20,12 +27,21 @@ input {
     file_completed_action => "delete"
     tags => "tenable"
   }
+  file {
+    path => "/opt/VulnWhisperer/data/tenable/*.json"
+    start_position => "beginning"
+    file_completed_action => "delete"
+    tags => "tenable"
+    codec => json
+  }
 }
 
 filter {
   if "nessus" in [tags] or "tenable" in [tags] {
+
+    if [path] =~ /\.csv$/ {}
       # Drop the header column
-    if [message] =~ "^Plugin ID" { drop {} }
+      if [message] =~ "^(Plugin ID|plugin_id,)" { drop {} }
 
       csv {
         # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"]
@@ -51,10 +67,11 @@ filter {
                       event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
                   end"
       }
+    }
 
     #If using filebeats as your source, you will need to replace the "path" field to "source"
     grok {
-      match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.csv$" }
+      match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.(csv|json)$" }
       tag_on_failure => []
     }