add unique document id

resources/elk6/pipeline/1000_nessus_process_file.conf

@@ -43,14 +43,29 @@ filter {
       convert => { "risk_number" => "integer"}
       convert => { "total_times_detected" => "integer"}
     }
+
+    if [_unique] {
+      # Set document ID from _unique
+      mutate {
+        rename => { "_unique" => "[@metadata][id]" }
+      }
+    }
   }
 }
 
 output {
   if "nessus" in [tags] or "tenable" in [tags]{
-    elasticsearch {
-      hosts => [ "elasticsearch:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+    if [@metadata][id] {
+      elasticsearch {
+        hosts => [ "elasticsearch:9200" ]
+        index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+        document_id => "%{[@metadata][id]}"
+      }
+    } else {
+      elasticsearch {
+        hosts => [ "elasticsearch:9200" ]
+        index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+      }
     }
   }
 }

resources/elk6/pipeline/2000_qualys_web_scans.conf

@@ -6,7 +6,7 @@
 
 input {
   file {
-    path => [ "/opt/VulnWhisperer/data/qualys_vuln/*.json" ] 
+    path => [ "/opt/VulnWhisperer/data/qualys_vuln/*.json" ]
     codec => json
     start_position => "beginning"
     tags => [ "qualys_vuln" ]
@@ -15,7 +15,7 @@ input {
     file_completed_action => "delete"
   }
   file {
-    path => [ "/opt/VulnWhisperer/data/qualys_web/*.json" ] 
+    path => [ "/opt/VulnWhisperer/data/qualys_web/*.json" ]
     codec => json
     start_position => "beginning"
     tags => [ "qualys_web" ]
@@ -79,13 +79,27 @@ filter {
     #     add_tag => [ "critical_asset" ]
     #   }
     # }
+    if [_unique] {
+      # Set document ID from _unique
+      mutate {
+        rename => { "_unique" => "[@metadata][id]" }
+      }
+    }
   }
 }
 output {
   if "qualys_vuln" in [tags] or "qualys_web" in  [tags] {
-    elasticsearch {
-      hosts => [ "elasticsearch:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+    if [@metadata][id] {
+      elasticsearch {
+        hosts => [ "elasticsearch:9200" ]
+        index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+        document_id => "%{[@metadata][id]}"
+      }
+    } else {
+      elasticsearch {
+        hosts => [ "elasticsearch:9200" ]
+        index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+      }
     }
   }
 }

resources/elk6/pipeline/3000_openvas.conf

@@ -100,18 +100,32 @@ filter {
     }
 
     # Add your critical assets by subnet or by hostname. Comment this field out if you don't want to tag any, but the asset panel will break.
-    if [asset] =~ "^10\.0\.100\." {
+    # if [asset] =~ "^10\.0\.100\." {
+    #   mutate {
+    #     add_tag => [ "critical_asset" ]
+    #   }
+    # }
+    if [_unique] {
+      # Set document ID from _unique
       mutate {
-        add_tag => [ "critical_asset" ]
+        rename => { "_unique" => "[@metadata][id]" }
       }
     }
   }
 }
 output {
   if "openvas" in [tags] {
-    elasticsearch {
-      hosts => [ "elasticsearch:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+    if [@metadata][id] {
+      elasticsearch {
+        hosts => [ "elasticsearch:9200" ]
+        index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+        document_id => "%{[@metadata][id]}"
+      }
+    } else {
+      elasticsearch {
+        hosts => [ "elasticsearch:9200" ]
+        index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+      }
     }
   }
 }