remove unnecessary groks

resources/elk6/pipeline/1000_nessus_process_file.conf

@@ -33,13 +33,6 @@ filter {
       remove_field => ["scan_time"]
     }
 
-    #If using filebeats as your source, you will need to replace the "path" field to "source"
-    # Remove when scan name is included in event (current method is error prone)
-    grok {
-      match => { "path" => "([a-zA-Z0-9_.\-]+)_%{INT}_%{INT:history_id}_%{INT}.json$" }
-      tag_on_failure => []
-    }
-
     mutate {
       convert => { "cvss" => "float"}
       convert => { "cvss_base" => "float"}

resources/elk6/pipeline/2000_qualys_web_scans.conf

@@ -6,11 +6,19 @@
 
 input {
   file {
-    path => [ "/opt/VulnWhisperer/data/qualys/*.json" , "/opt/VulnWhisperer/data/qualys_web/*.json", "/opt/VulnWhisperer/data/qualys_vuln/*.json"] 
+    path => [ "/opt/VulnWhisperer/data/qualys_vuln/*.json" ] 
-    type => json
     codec => json
     start_position => "beginning"
-    tags => [ "qualys" ]
+    tags => [ "qualys_vuln" ]
+    mode => "read"
+    start_position => "beginning"
+    file_completed_action => "delete"
+  }
+  file {
+    path => [ "/opt/VulnWhisperer/data/qualys_web/*.json" ] 
+    codec => json
+    start_position => "beginning"
+    tags => [ "qualys_web" ]
     mode => "read"
     start_position => "beginning"
     file_completed_action => "delete"
@@ -25,11 +33,6 @@ filter {
       remove_field => ["scan_time"]
     }
 
-    grok {
-      match => { "path" => [ "(?<tags>qualys_vuln)_scan_%{DATA}_%{INT}.json$", "(?<tags>qualys_web)_%{INT:app_id}_%{INT}.json$" ] }
-      tag_on_failure => []
-    }
-
     if "qualys_web" in [tags] {
       mutate {
         add_field => { "asset" => "%{web_application_name}" }

resources/elk6/pipeline/3000_openvas.conf

@@ -7,7 +7,6 @@
 input {
   file {
     path => "/opt/VulnWhisperer/data/openvas/*.json"
-    type => json
     codec => json
     start_position => "beginning"
     tags => [ "openvas_scan", "openvas" ]
@@ -26,11 +25,6 @@ filter {
       remove_field => ["scan_time"]
     }
 
-    grok {
-        match => { "path" => "openvas_scan_%{DATA}_%{INT}.json$" }
-        tag_on_failure => []
-    }
-
     # TODO - move this mapping into the vulnwhisperer module
     translate {
       field => "[risk_number]"