Updates for normalised fields and json output

resources/elk6/logstash-vulnwhisperer-template.json

@@ -1,5 +1,6 @@
 {
   "index_patterns": "logstash-vulnwhisperer-*",
+  "version": 2019041701,
   "mappings": {
     "doc": {
       "properties": {
@@ -22,9 +23,6 @@
         "asset_uuid": {
           "type": "keyword"
         },
-        "assign_ip": {
-          "type": "ip"
-        },
         "category": {
           "type": "keyword"
         },
@@ -34,7 +32,7 @@
         "cvss_base": {
           "type": "float"
         },
-        "cvss_temporal_vector": {
+        "cvss_severity": {
           "type": "keyword"
         },
         "cvss_temporal": {
@@ -49,7 +47,7 @@
         "cvss3_base": {
           "type": "float"
         },
-        "cvss3_temporal_vector": {
+        "cvss3_severity": {
           "type": "keyword"
         },
         "cvss3_temporal": {
@@ -117,24 +115,14 @@
         "host_start": {
           "type": "date"
         },
-        "impact": {
-          "fields": {
-            "keyword": {
-              "ignore_above": 256,
-              "type": "keyword"
-            }
-          },
-          "norms": false,
-          "type": "text"
-        },
-        "ip_status": {
-          "type": "keyword"
-        },
         "ip": {
           "type": "ip"
         },
-        "last_updated": {
+        "mac_address": {
-          "type": "date"
+          "type": "keyword"
+        },
+        "netbios": {
+          "type": "keyword"
         },
         "operating_system": {
           "type": "keyword"
@@ -170,18 +158,9 @@
         "protocol": {
           "type": "keyword"
         },
-        "results": {
-          "type": "text"
-        },
         "risk_number": {
           "type": "integer"
         },
-        "risk_score_name": {
-          "type": "keyword"
-        },
-        "risk_score": {
-          "type": "float"
-        },
         "risk": {
           "type": "keyword"
         },
@@ -191,41 +170,32 @@
         "scan_name": {
           "type": "keyword"
         },
-        "scan_reference": {
+        "scan_source": {
           "type": "keyword"
         },
-        "see_also": {
+        "severity": {
           "type": "keyword"
         },
         "solution": {
           "type": "keyword"
         },
-        "source": {
+        "ssl": {
           "type": "keyword"
         },
-        "ssl": {
+        "state": {
           "type": "keyword"
         },
         "synopsis": {
           "type": "keyword"
         },
-        "system_type": {
-          "type": "keyword"
-        },
         "tags": {
           "type": "keyword"
         },
-        "threat": {
-          "type": "text"
-        },
         "type": {
           "type": "keyword"
         },
         "vendor_reference": {
           "type": "keyword"
-        },
-        "vulnerability_state": {
-          "type": "keyword"
         }
       }
     }

tests/test-docker.sh

@@ -29,13 +29,13 @@ done
 green "✅ Elasticsearch status is green..."
 
 count=0
-until [[ $(curl -s "$logstash_url/_node/stats" | jq '.events.out') -ge 1236 ]]; do
+until [[ $(curl -s "$logstash_url/_node/stats" | jq '.events.out') -ge 1232 ]]; do
-    yellow "Waiting for Logstash load to finish...  $(curl -s "$logstash_url/_node/stats" | jq '.events.out') of 1236 (attempt $count of 60)"
+    yellow "Waiting for Logstash load to finish...  $(curl -s "$logstash_url/_node/stats" | jq '.events.out') of 1232 (attempt $count of 60)"
     ((count++)) && ((count==60)) && break
     sleep 5
 done
 
-if [[ count -le 60 && $(curl -s "$logstash_url/_node/stats" | jq '.events.out') -ge 1236 ]]; then
+if [[ count -le 60 && $(curl -s "$logstash_url/_node/stats" | jq '.events.out') -ge 1232 ]]; then
     green "✅ Logstash load finished..."
 else
     red "❌ Logstash load didn't complete... $(curl -s "$logstash_url/_node/stats" | jq '.events.out')"
@@ -49,7 +49,7 @@ until [[ $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" |
     sleep 2
 done
 if [[ count -le 50 && $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq '.count') -ge 1232 ]]; then
-    green "✅ logstash-vulnwhisperer-2019.03 document count >= 1232"
+    green "✅ logstash-vulnwhisperer-2019.03 document count $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq '.count') >= 1232"
 else
     red "❌ TIMED OUT waiting for logstash-vulnwhisperer-2019.03 document count: $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq) != 1232"
 fi
@@ -63,10 +63,10 @@ fi
 
 # Test Nessus plugin_name:Backported Security Patch Detection (FTP)
 nessus_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_search?q=plugin_name:%22Backported%20Security%20Patch%20Detection%20(FTP)%22%20AND%20asset:176.28.50.164%20AND%20tags:nessus" | jq '.hits.hits[]._source')
-if echo $nessus_doc | jq '.risk' | grep -q "None"; then
+if echo $nessus_doc | jq '.risk' | grep -q "none"; then
-    green "✅ Passed: Nessus risk == None"
+    green "✅ Passed: Nessus risk == none"
 else
-    red "❌ Failed: Nessus risk == None was: $(echo $nessus_doc | jq '.risk') instead"
+    red "❌ Failed: Nessus risk == none was: $(echo $nessus_doc | jq '.risk') instead"
     ((return_code = return_code + 1))
 fi
 
@@ -99,10 +99,10 @@ else
 fi
 
 # Test @XXXX
-if echo $qualys_vuln_doc | jq '.cvss' | grep -q '6.8'; then
+if echo $qualys_vuln_doc | jq '.cvss' | grep -q '5.6'; then
-    green "✅ Passed: Qualys VM cvss == 6.8"
+    green "✅ Passed: Qualys VM cvss == 5.6"
 else
-    red "❌ Failed: Qualys VM cvss == 6.8 was: $(echo $qualys_vuln_doc | jq '.cvss') instead"
+    red "❌ Failed: Qualys VM cvss == 5.6 was: $(echo $qualys_vuln_doc | jq '.cvss') instead"
     ((return_code = return_code + 1))
 fi