From 0c3200567e874850bc817ffc1509dca2db976679 Mon Sep 17 00:00:00 2001
From: pemontto <pemontto@gmail.com>
Date: Mon, 22 Apr 2019 11:38:41 +1000
Subject: [PATCH] remove unnecessary groks

---
 .../pipeline/1000_nessus_process_file.conf    |  7 -------
 .../elk6/pipeline/2000_qualys_web_scans.conf  | 19 +++++++++++--------
 resources/elk6/pipeline/3000_openvas.conf     |  6 ------
 3 files changed, 11 insertions(+), 21 deletions(-)

diff --git a/resources/elk6/pipeline/1000_nessus_process_file.conf b/resources/elk6/pipeline/1000_nessus_process_file.conf
index c0c4f27..91c3dd5 100644
--- a/resources/elk6/pipeline/1000_nessus_process_file.conf
+++ b/resources/elk6/pipeline/1000_nessus_process_file.conf
@@ -33,13 +33,6 @@ filter {
       remove_field => ["scan_time"]
     }
 
-    #If using filebeats as your source, you will need to replace the "path" field to "source"
-    # Remove when scan name is included in event (current method is error prone)
-    grok {
-      match => { "path" => "([a-zA-Z0-9_.\-]+)_%{INT}_%{INT:history_id}_%{INT}.json$" }
-      tag_on_failure => []
-    }
-
     mutate {
       convert => { "cvss" => "float"}
       convert => { "cvss_base" => "float"}
diff --git a/resources/elk6/pipeline/2000_qualys_web_scans.conf b/resources/elk6/pipeline/2000_qualys_web_scans.conf
index aad34f1..652e48a 100644
--- a/resources/elk6/pipeline/2000_qualys_web_scans.conf
+++ b/resources/elk6/pipeline/2000_qualys_web_scans.conf
@@ -6,11 +6,19 @@
 
 input {
   file {
-    path => [ "/opt/VulnWhisperer/data/qualys/*.json" , "/opt/VulnWhisperer/data/qualys_web/*.json", "/opt/VulnWhisperer/data/qualys_vuln/*.json"] 
-    type => json
+    path => [ "/opt/VulnWhisperer/data/qualys_vuln/*.json" ] 
     codec => json
     start_position => "beginning"
-    tags => [ "qualys" ]
+    tags => [ "qualys_vuln" ]
+    mode => "read"
+    start_position => "beginning"
+    file_completed_action => "delete"
+  }
+  file {
+    path => [ "/opt/VulnWhisperer/data/qualys_web/*.json" ] 
+    codec => json
+    start_position => "beginning"
+    tags => [ "qualys_web" ]
     mode => "read"
     start_position => "beginning"
     file_completed_action => "delete"
@@ -25,11 +33,6 @@ filter {
       remove_field => ["scan_time"]
     }
 
-    grok {
-      match => { "path" => [ "(?<tags>qualys_vuln)_scan_%{DATA}_%{INT}.json$", "(?<tags>qualys_web)_%{INT:app_id}_%{INT}.json$" ] }
-      tag_on_failure => []
-    }
-
     if "qualys_web" in [tags] {
       mutate {
         add_field => { "asset" => "%{web_application_name}" }
diff --git a/resources/elk6/pipeline/3000_openvas.conf b/resources/elk6/pipeline/3000_openvas.conf
index 47aed47..00ef7cc 100644
--- a/resources/elk6/pipeline/3000_openvas.conf
+++ b/resources/elk6/pipeline/3000_openvas.conf
@@ -7,7 +7,6 @@
 input {
   file {
     path => "/opt/VulnWhisperer/data/openvas/*.json"
-    type => json
     codec => json
     start_position => "beginning"
     tags => [ "openvas_scan", "openvas" ]
@@ -26,11 +25,6 @@ filter {
       remove_field => ["scan_time"]
     }
 
-    grok {
-        match => { "path" => "openvas_scan_%{DATA}_%{INT}.json$" }
-        tag_on_failure => []
-    }
-
     # TODO - move this mapping into the vulnwhisperer module
     translate {
       field => "[risk_number]"