Merge pull request #177 from pemontto/feature-nessus-json-latest

Beta 2.0 initial commit

A number of changes in this PR. Many fixes and cleanups, some bug fixes, and the first steps towards a vulnerability standard.
- Nessus and Tenable output as JSON
- Start of vulnerability standard (mapping and transform methods for all modules)
- Removed ELK5
- Overhauled Logstash configs
- Support for alternative Qualys WAS CSV headers
- More unicode fixes
- Mock tests for Qualys WAS and OpenVAS

Todo:
- Continue standardising modules, bring nessus/tenable inline with others
- Write better end-to-end tests and include Qualys WAS and OpenVAS

.gitmodules

@@ -1,3 +1,4 @@
 [submodule "tests/data"]
 	path = tests/data
 	url = https://github.com/HASecuritySolutions/VulnWhisperer-tests.git
+	branch = master

.travis.yml

@@ -22,13 +22,13 @@ before_install:
 install:
     - pip install -r requirements.txt
     - pip install flake8  # pytest  # add another testing frameworks later
+    - python setup.py install
 before_script:
     # stop the build if there are Python syntax errors or undefined names
     - flake8 . --count --exclude=deps/qualysapi --select=E901,E999,F821,F822,F823 --show-source --statistics
     # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
     - flake8 . --count --exit-zero --exclude=deps/qualysapi --max-complexity=10 --max-line-length=127 --statistics
 script:
-    - python setup.py install
     - bash tests/test-vuln_whisperer.sh
     - bash tests/test-docker.sh
 notifications:

bin/vuln_whisperer

@@ -3,13 +3,14 @@
 __author__ = 'Austin Taylor'
 
 
-from vulnwhisp.vulnwhisp import vulnWhisperer
+import argparse
+import logging
+import os
+import sys
+
 from vulnwhisp.base.config import vwConfig
 from vulnwhisp.test.mock import mockAPI
-import os
+from vulnwhisp.vulnwhisp import vulnWhisperer
-import argparse
-import sys
-import logging
 
 
 def isFileValid(parser, arg):

configs/frameworks_example.ini

@@ -2,6 +2,8 @@
 enabled=true
 hostname=localhost
 port=8834
+access_key=
+secret_key=
 username=nessus_username
 password=nessus_password
 write_path=/opt/VulnWhisperer/data/nessus/
@@ -13,6 +15,8 @@ verbose=true
 enabled=true
 hostname=cloud.tenable.com
 port=443
+access_key=
+secret_key=
 username=tenable.io_username
 password=tenable.io_password
 write_path=/opt/VulnWhisperer/data/tenable/

configs/test.ini

@@ -2,6 +2,8 @@
 enabled=true
 hostname=nessus
 port=443
+access_key=
+secret_key=
 username=nessus_username
 password=nessus_password
 write_path=/opt/VulnWhisperer/data/nessus/
@@ -13,6 +15,8 @@ verbose=true
 enabled=true
 hostname=tenable
 port=443
+access_key=
+secret_key=
 username=tenable.io_username
 password=tenable.io_password
 write_path=/opt/VulnWhisperer/data/tenable/
@@ -22,7 +26,7 @@ verbose=true
 
 [qualys_web]
 #Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
-enabled = false
+enabled = true
 hostname = qualys_web
 username = exampleuser
 password = examplepass
@@ -34,7 +38,7 @@ verbose=true
 #Note, this applies only to failed connections and timeouts, never to requests where the server returns a response.
 max_retries = 10
 # Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID.
-template_id = 126024
+template_id = 289109
 
 [qualys_vuln]
 #Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
@@ -59,7 +63,7 @@ db_path = /opt/VulnWhisperer/data/database
 verbose = true
 
 [openvas]
-enabled = false
+enabled = true
 hostname = openvas
 port = 4000
 username = exampleuser

docker-compose-test.yml

@@ -9,6 +9,7 @@ services:
       - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
       - xpack.security.enabled=false
       - cluster.routing.allocation.disk.threshold_enabled=false
+      - path.repo=/snapshots
     ulimits:
       memlock:
         soft: -1

docker-compose.v6.yml

@@ -8,6 +8,7 @@ services:
       - bootstrap.memory_lock=true
       - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
       - xpack.security.enabled=false
+      - path.repo=/snapshots
     ulimits:
       memlock:
         soft: -1
@@ -18,6 +19,7 @@ services:
     mem_limit: 8g
     volumes:
       - esdata1:/usr/share/elasticsearch/data
+      - ./data/es_snapshots:/snapshots
     ports:
       - 9200:9200
     #restart: always

resources/elk5-old_compatibility/docker-compose_ELK5_unsupported.yml

@@ -1,72 +0,0 @@
-version: '2'
-services:
-  vulnwhisp-es1:
-    image: docker.elastic.co/elasticsearch/elasticsearch:5.6.2
-    container_name: vulnwhisp-es1
-    environment:
-      - cluster.name=vulnwhisperer
-      - bootstrap.memory_lock=true
-      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-    ulimits:
-      memlock:
-        soft: -1
-        hard: -1
-      nofile:
-        soft: 65536
-        hard: 65536
-    mem_limit: 8g
-    volumes:
-      - esdata1:/usr/share/elasticsearch/data
-    ports:
-      - 9200:9200
-    environment:
-      - xpack.security.enabled=false
-    #restart: always
-    networks:
-      esnet:
-        aliases:
-          - vulnwhisp-es1.local
-  vulnwhisp-ks1:
-    image: docker.elastic.co/kibana/kibana:5.6.2
-    environment:
-      SERVER_NAME: vulnwhisp-ks1
-      ELASTICSEARCH_URL: http://vulnwhisp-es1:9200
-    ports:
-      - 5601:5601
-    depends_on:
-      - vulnwhisp-es1
-    networks:
-      esnet:
-        aliases:
-          - vulnwhisp-ks1.local
-  vulnwhisp-ls1:
-    image: docker.elastic.co/logstash/logstash:5.6.2
-    container_name: vulnwhisp-ls1
-    volumes:
-      - ./docker/1000_nessus_process_file.conf:/usr/share/logstash/pipeline/1000_nessus_process_file.conf
-      - ./docker/2000_qualys_web_scans.conf:/usr/share/logstash/pipeline/2000_qualys_web_scans.conf
-      - ./docker/3000_openvas.conf:/usr/share/logstash/pipeline/3000_openvas.conf
-      - ./docker/4000_jira.conf:/usr/share/logstash/pipeline/4000_jira.conf
-      - ./docker/logstash.yml:/usr/share/logstash/config/logstash.yml
-      - ./data/:/opt/VulnWhisperer/data
-    environment:
-      - xpack.monitoring.enabled=false
-    depends_on:
-      - vulnwhisp-es1
-    networks:
-      esnet:
-        aliases:
-          - vulnwhisp-ls1.local
-  vulnwhisp-vulnwhisperer:
-    image: hasecuritysolutions/vulnwhisperer:latest
-    container_name: vulnwhisp-vulnwhisperer
-    volumes:
-      - ./data/:/opt/VulnWhisperer/data
-      - ./configs/frameworks_example.ini:/opt/VulnWhisperer/frameworks_example.ini
-    network_mode: host
-volumes:
-  esdata1:
-    driver: local
-
-networks:
-  esnet:

resources/elk5-old_compatibility/docker/1000_nessus_process_file.conf

@@ -1,220 +0,0 @@
-# Author: Austin Taylor and Justin Henderson
-# Email: email@austintaylor.io
-# Last Update: 12/20/2017
-# Version 0.3
-# Description: Take in nessus reports from vulnWhisperer and pumps into logstash
-
-
-input {
-  file {
-    path => "/opt/VulnWhisperer/data/nessus/**/*"
-    start_position => "beginning"
-    tags => "nessus"
-    type => "nessus"
-  }
-  file {
-    path => "/opt/VulnWhisperer/data/tenable/*.csv"
-    start_position => "beginning"
-    tags => "tenable"
-    type => "tenable"
-  }
-}
-
-filter {
-  if "nessus" in [tags] or "tenable" in [tags] {
-    # Drop the header column
-    if [message] =~ "^Plugin ID" { drop {} }
-
-    csv {
-      # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"]
-      columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss_vector", "system_type", "host_start", "host_end"]
-      separator => ","
-      source => "message"
-    }
-
-    ruby {
-        code => "if event.get('description')
-                    event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('synopsis')
-                    event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('solution')
-                    event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('see_also')
-                    event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('plugin_output')
-                    event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end"
-    }
-
-    #If using filebeats as your source, you will need to replace the "path" field to "source"
-    grok {
-      match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.csv$" }
-      tag_on_failure => []
-    }
-
-    date {
-      match => [ "last_updated", "UNIX" ]
-      target => "@timestamp"
-      remove_field => ["last_updated"]
-    }
-
-    if [risk] == "None" {
-      mutate { add_field => { "risk_number" => 0 }}
-    }
-    if [risk] == "Low" {
-      mutate { add_field => { "risk_number" => 1 }}
-    }
-    if [risk] == "Medium" {
-      mutate { add_field => { "risk_number" => 2 }}
-    }
-    if [risk] == "High" {
-      mutate { add_field => { "risk_number" => 3 }}
-    }
-    if [risk] == "Critical" {
-      mutate { add_field => { "risk_number" => 4 }}
-    }
-
-    if ![cve] or [cve] == "nan" {
-      mutate { remove_field => [ "cve" ] }
-    }
-    if ![cvss] or [cvss] == "nan" {
-      mutate { remove_field => [ "cvss" ] }
-    }
-    if ![cvss_base] or [cvss_base] == "nan" {
-      mutate { remove_field => [ "cvss_base" ] }
-    }
-    if ![cvss_temporal] or [cvss_temporal] == "nan" {
-      mutate { remove_field => [ "cvss_temporal" ] }
-    }
-    if ![cvss_temporal_vector] or [cvss_temporal_vector] == "nan" {
-      mutate { remove_field => [ "cvss_temporal_vector" ] }
-    }
-    if ![cvss_vector] or [cvss_vector] == "nan" {
-      mutate { remove_field => [ "cvss_vector" ] }
-    }
-    if ![cvss3_base] or [cvss3_base] == "nan" {
-      mutate { remove_field => [ "cvss3_base" ] }
-    }
-    if ![cvss3_temporal] or [cvss3_temporal] == "nan" {
-      mutate { remove_field => [ "cvss3_temporal" ] }
-    }
-    if ![cvss3_temporal_vector] or [cvss3_temporal_vector] == "nan" {
-      mutate { remove_field => [ "cvss3_temporal_vector" ] }
-    }
-    if ![description] or [description] == "nan" {
-      mutate { remove_field => [ "description" ] }
-    }
-    if ![mac_address] or [mac_address] == "nan" {
-      mutate { remove_field => [ "mac_address" ] }
-    }
-    if ![netbios] or [netbios] == "nan" {
-      mutate { remove_field => [ "netbios" ] }
-    }
-    if ![operating_system] or [operating_system] == "nan" {
-      mutate { remove_field => [ "operating_system" ] }
-    }
-    if ![plugin_output] or [plugin_output] == "nan" {
-      mutate { remove_field => [ "plugin_output" ] }
-    }
-    if ![see_also] or [see_also] == "nan" {
-      mutate { remove_field => [ "see_also" ] }
-    }
-    if ![synopsis] or [synopsis] == "nan" {
-      mutate { remove_field => [ "synopsis" ] }
-    }
-    if ![system_type] or [system_type] == "nan" {
-      mutate { remove_field => [ "system_type" ] }
-    }
-
-    mutate {
-      remove_field => [ "message" ]
-      add_field => { "risk_score" => "%{cvss}" }
-    }
-    mutate {
-      convert => { "risk_score" => "float" }
-    }
-    if [risk_score] == 0 {
-      mutate {
-        add_field => { "risk_score_name" => "info" }
-      }
-    }
-    if [risk_score] > 0 and [risk_score] < 3 {
-      mutate {
-        add_field => { "risk_score_name" => "low" }
-      }
-    }
-    if [risk_score] >= 3 and [risk_score] < 6 {
-      mutate {
-        add_field => { "risk_score_name" => "medium" }
-      }
-    }
-    if [risk_score] >=6 and [risk_score] < 9 {
-      mutate {
-        add_field => { "risk_score_name" => "high" }
-      }
-    }
-    if [risk_score] >= 9 {
-      mutate {
-        add_field => { "risk_score_name" => "critical" }
-      }
-    }
-
-    # Compensating controls - adjust risk_score
-    # Adobe and Java are not allowed to run in browser unless whitelisted
-    # Therefore, lower score by dividing by 3 (score is subjective to risk)
-
-    #Modify and uncomment when ready to use
-    #if [risk_score] != 0 {
-    #  if [plugin_name] =~ "Adobe" and [risk_score] > 6 or [plugin_name] =~ "Java" and [risk_score] > 6 {
-    #    ruby {
-    #      code => "event.set('risk_score', event.get('risk_score') / 3)"
-    #    }
-    #    mutate {
-    #      add_field => { "compensating_control" => "Adobe and Flash removed from browsers unless whitelisted site." }
-    #    }
-    #  }
-    #}
-
-    # Add tags for reporting based on assets or criticality
-
-    if [asset] == "dc01" or [asset] == "dc02" or [asset] == "pki01" or [asset] == "192.168.0.54" or [asset] =~ "^192\.168\.0\." or [asset] =~ "^42.42.42." {
-      mutate {
-        add_tag => [ "critical_asset" ]
-      }
-    }
-    #if [asset] =~ "^192\.168\.[45][0-9][0-9]\.1$" or [asset] =~ "^192.168\.[50]\.[0-9]{1,2}\.1$"{
-    #  mutate {
-    #    add_tag => [ "has_hipaa_data" ]
-    #  }
-    #}
-    #if [asset] =~ "^192\.168\.[45][0-9][0-9]\." {
-    #  mutate {
-    #    add_tag => [ "hipaa_asset" ]
-    #  }
-    #}
-    if [asset] =~ "^hr" {
-      mutate {
-        add_tag => [ "pci_asset" ]
-      }
-    }
-    #if [asset] =~ "^10\.0\.50\." {
-    #  mutate {
-    #    add_tag => [ "web_servers" ]
-    #  }
-    #}
-  }
-}
-
-output {
-  if "nessus" in [tags] or "tenable" in [tags] or [type] in [ "nessus", "tenable" ] {
-    # stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => [ "vulnwhisp-es1.local:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
-    }
-  }
-}

resources/elk5-old_compatibility/docker/2000_qualys_web_scans.conf

@@ -1,153 +0,0 @@
-# Author: Austin Taylor and Justin Henderson
-# Email: austin@hasecuritysolutions.com
-# Last Update: 12/30/2017
-# Version 0.3
-# Description: Take in qualys web scan reports from vulnWhisperer and pumps into logstash
-
-input {
-  file {
-    path => "/opt/VulnWhisperer/data/qualys/*.json"
-    type => json
-    codec => json
-    start_position => "beginning"
-    tags => [ "qualys" ]
-  }
-}
-
-filter {
-  if "qualys" in [tags] {
-    grok {
-      match => { "path" => [ "(?<tags>qualys_vuln)_scan_%{DATA}_%{INT:last_updated}.json$", "(?<tags>qualys_web)_%{INT:app_id}_%{INT:last_updated}.json$" ] }
-      tag_on_failure => []
-    }
-    
-    mutate {
-      replace => [ "message", "%{message}" ]
-      #gsub => [
-      #  "message", "\|\|\|", " ",
-      #  "message", "\t\t", " ",
-      #  "message", "    ", " ",
-      #  "message", "   ", " ",
-      #  "message", "  ", " ",
-      #  "message", "nan", " ",
-      #  "message",'\n',''
-      #]
-    }
-
-    if "qualys_web" in [tags] {
-    mutate {
-      add_field => { "asset" => "%{web_application_name}" }
-      add_field => { "risk_score" => "%{cvss}" }
-      }
-    } else if "qualys_vuln" in [tags] {
-      mutate {
-        add_field => { "asset" => "%{ip}" }
-        add_field => { "risk_score" => "%{cvss}" }
-      }
-    }
-
-    if [risk] == "1" {
-      mutate { add_field => { "risk_number" => 0 }}
-      mutate { replace => { "risk" => "info" }}
-    }
-    if [risk] == "2" {
-      mutate { add_field => { "risk_number" => 1 }}
-      mutate { replace => { "risk" => "low" }}
-    }
-    if [risk] == "3" {
-      mutate { add_field => { "risk_number" => 2 }}
-      mutate { replace => { "risk" => "medium" }}
-    }
-    if [risk] == "4" {
-      mutate { add_field => { "risk_number" => 3 }}
-      mutate { replace => { "risk" => "high" }}
-    }
-    if [risk] == "5" {
-      mutate { add_field => { "risk_number" => 4 }}
-      mutate { replace => { "risk" => "critical" }}
-    }
-
-    mutate {
-      remove_field => "message"
-    }
-
-    if [first_time_detected] {
-      date {
-        match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "first_time_detected"
-      }
-    }
-    if [first_time_tested] {
-      date {
-        match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "first_time_tested"
-      }
-    }
-    if [last_time_detected] {
-      date {
-        match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "last_time_detected"
-      }
-    }
-    if [last_time_tested] {
-      date {
-        match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "last_time_tested"
-      }
-    }
-    date {
-      match => [ "last_updated", "UNIX" ]
-      target => "@timestamp"
-      remove_field => "last_updated"
-    }
-    mutate {
-      convert => { "plugin_id" => "integer"}
-      convert => { "id" => "integer"}
-      convert => { "risk_number" => "integer"}
-      convert => { "risk_score" => "float"}
-      convert => { "total_times_detected" => "integer"}
-      convert => { "cvss_temporal" => "float"}
-      convert => { "cvss" => "float"}
-    }
-    if [risk_score] == 0 {
-      mutate {
-        add_field => { "risk_score_name" => "info" }
-      }
-    }
-    if [risk_score] > 0 and [risk_score] < 3 {
-      mutate {
-        add_field => { "risk_score_name" => "low" }
-      }
-    }
-    if [risk_score] >= 3 and [risk_score] < 6 {
-      mutate {
-        add_field => { "risk_score_name" => "medium" }
-      }
-    }
-    if [risk_score] >=6 and [risk_score] < 9 {
-      mutate {
-        add_field => { "risk_score_name" => "high" }
-      }
-    }
-    if [risk_score] >= 9 {
-      mutate {
-        add_field => { "risk_score_name" => "critical" }
-      }
-    }
-
-    if [asset] =~ "\.yourdomain\.(com|net)$" {
-      mutate {
-        add_tag => [ "critical_asset" ]
-      }
-    }
-  }
-}
-output {
-  if "qualys" in [tags] {
-    stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => [ "vulnwhisp-es1.local:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
-    }
-  }
-}

resources/elk5-old_compatibility/docker/3000_openvas.conf

@@ -1,146 +0,0 @@
-# Author: Austin Taylor and Justin Henderson
-# Email: austin@hasecuritysolutions.com
-# Last Update: 03/04/2018
-# Version 0.3
-# Description: Take in qualys web scan reports from vulnWhisperer and pumps into logstash
-
-input {
-  file {
-    path => "/opt/VulnWhisperer/data/openvas/*.json"
-    type => json
-    codec => json
-    start_position => "beginning"
-    tags => [ "openvas_scan", "openvas" ]
-  }
-}
-
-filter {
-  if "openvas_scan" in [tags] {
-    mutate {
-      replace => [ "message", "%{message}" ]
-      gsub => [
-        "message", "\|\|\|", " ",
-        "message", "\t\t", " ",
-        "message", "    ", " ",
-        "message", "   ", " ",
-        "message", "  ", " ",
-        "message", "nan", " ",
-        "message",'\n',''
-      ]
-    }
-
-
-    grok {
-        match => { "path" => "openvas_scan_%{DATA:scan_id}_%{INT:last_updated}.json$" }
-        tag_on_failure => []
-    }
-
-    mutate {
-      add_field => { "risk_score" => "%{cvss}" }
-    }
-
-    if [risk] == "1" {
-      mutate { add_field => { "risk_number" => 0 }}
-      mutate { replace => { "risk" => "info" }}
-    }
-    if [risk] == "2" {
-      mutate { add_field => { "risk_number" => 1 }}
-      mutate { replace => { "risk" => "low" }}
-    }
-    if [risk] == "3" {
-      mutate { add_field => { "risk_number" => 2 }}
-      mutate { replace => { "risk" => "medium" }}
-    }
-    if [risk] == "4" {
-      mutate { add_field => { "risk_number" => 3 }}
-      mutate { replace => { "risk" => "high" }}
-    }
-    if [risk] == "5" {
-      mutate { add_field => { "risk_number" => 4 }}
-      mutate { replace => { "risk" => "critical" }}
-    }
-
-    mutate {
-      remove_field => "message"
-    }
-
-    if [first_time_detected] {
-      date {
-        match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "first_time_detected"
-      }
-    }
-    if [first_time_tested] {
-      date {
-        match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "first_time_tested"
-      }
-    }
-    if [last_time_detected] {
-      date {
-        match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "last_time_detected"
-      }
-    }
-    if [last_time_tested] {
-      date {
-        match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "last_time_tested"
-      }
-    }
-    date {
-      match => [ "last_updated", "UNIX" ]
-      target => "@timestamp"
-      remove_field => "last_updated"
-    }
-    mutate {
-      convert => { "plugin_id" => "integer"}
-      convert => { "id" => "integer"}
-      convert => { "risk_number" => "integer"}
-      convert => { "risk_score" => "float"}
-      convert => { "total_times_detected" => "integer"}
-      convert => { "cvss_temporal" => "float"}
-      convert => { "cvss" => "float"}
-    }
-    if [risk_score] == 0 {
-      mutate {
-        add_field => { "risk_score_name" => "info" }
-      }
-    }
-    if [risk_score] > 0 and [risk_score] < 3 {
-      mutate {
-        add_field => { "risk_score_name" => "low" }
-      }
-    }
-    if [risk_score] >= 3 and [risk_score] < 6 {
-      mutate {
-        add_field => { "risk_score_name" => "medium" }
-      }
-    }
-    if [risk_score] >=6 and [risk_score] < 9 {
-      mutate {
-        add_field => { "risk_score_name" => "high" }
-      }
-    }
-    if [risk_score] >= 9 {
-      mutate {
-        add_field => { "risk_score_name" => "critical" }
-      }
-    }
-    # Add your critical assets by subnet or by hostname. Comment this field out if you don't want to tag any, but the asset panel will break.
-    if [asset] =~ "^10\.0\.100\." {
-      mutate {
-        add_tag => [ "critical_asset" ]
-      }
-    }
-  }
-}
-output {
-  if "openvas" in [tags] {
-    stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => [ "vulnwhisp-es1.local:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
-    }
-  }
-}

resources/elk5-old_compatibility/docker/4000_jira.conf

@@ -1,21 +0,0 @@
-# Description: Take in jira tickets from vulnWhisperer and pumps into logstash
-
-input {
-  file {
-    path => "/opt/Vulnwhisperer/jira/*.json"
-    type => json
-    codec => json
-    start_position => "beginning"
-    tags => [ "jira" ]
-  }
-}
-
-output {
-  if "jira" in [tags] {
-    stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => [ "vulnwhisp-es1.local:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
-    }
-  }
-}

resources/elk5-old_compatibility/docker/logstash.yml

@@ -1,5 +0,0 @@
-path.config: /usr/share/logstash/pipeline/
-xpack.monitoring.elasticsearch.password: changeme
-xpack.monitoring.elasticsearch.url: vulnwhisp-es1.local:9200
-xpack.monitoring.elasticsearch.username: elastic
-xpack.monitoring.enabled: false

resources/elk5-old_compatibility/elasticsearch/logstash-vulnwhisperer-template.json

@@ -1,122 +0,0 @@
-{
-  "order": 0,
-  "template": "logstash-vulnwhisperer-*",
-  "settings": {
-    "index": {
-      "routing": {
-        "allocation": {
-          "total_shards_per_node": "2"
-        }
-      },
-      "mapping": {
-        "total_fields": {
-          "limit": "3000"
-        }
-      },
-      "refresh_interval": "5s",
-      "number_of_shards": "1",
-      "number_of_replicas": "0"
-    }
-  },
-  "mappings": {
-    "_default_": {
-      "_all": {
-        "enabled": false
-      },
-      "dynamic_templates": [
-        {
-          "message_field": {
-            "path_match": "message",
-            "match_mapping_type": "string",
-            "mapping": {
-              "type": "text",
-              "norms": false
-            }
-          }
-        },
-        {
-          "string_fields": {
-            "match": "*",
-            "match_mapping_type": "string",
-            "mapping": {
-              "type": "text",
-              "norms": false,
-              "fields": {
-                "keyword": {
-                  "type": "keyword",
-                  "ignore_above": 256
-                }
-              }
-            }
-          }
-        }
-      ],
-      "properties": {
-        "plugin_id": {
-          "type": "float"
-        },
-        "last_updated": {
-          "type": "date"
-        },
-        "geoip": {
-          "dynamic": true,
-          "type": "object",
-          "properties": {
-            "ip": {
-              "type": "ip"
-            },
-            "latitude": {
-              "type": "float"
-            },
-            "location": {
-              "type": "geo_point"
-            },
-            "longitude": {
-              "type": "float"
-            }
-          }
-        },
-        "risk_score": {
-          "type": "float"
-        },
-        "source": {          
-          "type": "keyword"
-        },
-        "synopsis": {
-          "type": "keyword"
-        },
-        "see_also": {
-          "type": "keyword"
-        },
-        "@timestamp": {
-          "type": "date"
-        },
-        "cve": {
-          "type": "keyword"
-        },
-        "solution": {
-          "type": "keyword"
-        },
-        "port": {
-          "type": "integer"
-        },
-        "host": {
-          "type": "text"
-        },
-        "@version": {
-          "type": "keyword"
-        },
-        "risk": {
-          "type": "keyword"
-        },
-        "assign_ip": {
-          "type": "ip"
-        },
-        "cvss": {
-          "type": "float"
-        }
-      }
-    }
-  },
-  "aliases": {}
-}

resources/elk5-old_compatibility/filebeat/filebeat.yml

@@ -1,116 +0,0 @@
-###################### Filebeat Configuration Example #########################
-
-# This file is an example configuration file highlighting only the most common
-# options. The filebeat.full.yml file from the same directory contains all the
-# supported options with more comments. You can use it as a reference.
-#
-# You can find the full configuration reference here:
-# https://www.elastic.co/guide/en/beats/filebeat/index.html
-
-#=========================== Filebeat prospectors =============================
-
-filebeat.prospectors:
-
-# Each - is a prospector. Most options can be set at the prospector level, so
-# you can use different prospectors for various configurations.
-# Below are the prospector specific configurations.
-
-- input_type: log
-  # Paths that should be crawled and fetched. Glob based paths.
-  paths:
-    # Linux Example
-    #- /var/log/*.log
-
-    #Windows Example
-    - c:\nessus\My Scans\*
-
-  # Exclude lines. A list of regular expressions to match. It drops the lines that are
-  # matching any regular expression from the list.
-  #exclude_lines: ["^DBG"]
-
-  # Include lines. A list of regular expressions to match. It exports the lines that are
-  # matching any regular expression from the list.
-  #include_lines: ["^ERR", "^WARN"]
-
-  # Exclude files. A list of regular expressions to match. Filebeat drops the files that
-  # are matching any regular expression from the list. By default, no files are dropped.
-  #exclude_files: [".gz$"]
-
-  # Optional additional fields. These field can be freely picked
-  # to add additional information to the crawled log files for filtering
-  #fields:
-  #  level: debug
-  #  review: 1
-
-  ### Multiline options
-
-  # Mutiline can be used for log messages spanning multiple lines. This is common
-  # for Java Stack Traces or C-Line Continuation
-
-  # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [
-  #multiline.pattern: ^\[
-
-  # Defines if the pattern set under pattern should be negated or not. Default is false.
-  #multiline.negate: false
-
-  # Match can be set to "after" or "before". It is used to define if lines should be append to a pattern
-  # that was (not) matched before or after or as long as a pattern is not matched based on negate.
-  # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash
-  #multiline.match: after
-
-
-#================================ General =====================================
-
-# The name of the shipper that publishes the network data. It can be used to group
-# all the transactions sent by a single shipper in the web interface.
-#name:
-
-# The tags of the shipper are included in their own field with each
-# transaction published.
-#tags: ["service-X", "web-tier"]
-
-# Optional fields that you can specify to add additional information to the
-# output.
-#fields:
-#  env: staging
-
-#================================ Outputs =====================================
-
-# Configure what outputs to use when sending the data collected by the beat.
-# Multiple outputs may be used.
-
-#-------------------------- Elasticsearch output ------------------------------
-#output.elasticsearch:
-  # Array of hosts to connect to.
-#  hosts: ["logstash01:9200"]
-
-  # Optional protocol and basic auth credentials.
-  #protocol: "https"
-  #username: "elastic"
-  #password: "changeme"
-
-#----------------------------- Logstash output --------------------------------
-output.logstash:
-  # The Logstash hosts
-  hosts: ["logstashserver1:5044", "logstashserver2:5044", "logstashserver3:5044"]
-
-  # Optional SSL. By default is off.
-  # List of root certificates for HTTPS server verifications
-  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
-
-  # Certificate for SSL client authentication
-  #ssl.certificate: "/etc/pki/client/cert.pem"
-
-  # Client Certificate Key
-  #ssl.key: "/etc/pki/client/cert.key"
-
-#================================ Logging =====================================
-
-# Sets log level. The default log level is info.
-# Available log levels are: critical, error, warning, info, debug
-#logging.level: debug
-
-# At debug level, you can selectively enable logging only for some components.
-# To enable all selectors use ["*"]. Examples of other selectors are "beat",
-# "publish", "service".
-#logging.selectors: ["*"]

resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/1000_vulnWhispererBaseVisuals.json

@@ -1,450 +0,0 @@
-[
-  {
-    "_id": "80158c90-57c1-11e7-b484-a970fc9d150a",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - HIPAA TL",
-      "visState": "{\"type\":\"timelion\",\"title\":\"VulnWhisperer - HIPAA TL\",\"params\":{\"expression\":\".es(index=logstash-vulnwhisperer-*,q='risk_score:>9 AND tags:pci_asset').label(\\\"PCI Assets\\\"),.es(index=logstash-vulnwhisperer-*,q='risk_score:>9 AND tags:has_hipaa_data').label(\\\"Has HIPAA Data\\\"),.es(index=logstash-vulnwhisperer-*,q='risk_score:>9 AND tags:hipaa_asset').label(\\\"HIPAA Assets\\\")\",\"interval\":\"auto\"}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{}"
-      }
-    }
-  },
-  {
-    "_id": "479deab0-8a39-11e7-a58a-9bfcb3761a3d",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - TL - TaggedAssetsPluginNames",
-      "visState": "{\"title\":\"VulnWhisperer - TL - TaggedAssetsPluginNames\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*', q='tags:critical_asset OR tags:hipaa_asset OR tags:pci_asset', split=\\\"plugin_name.keyword:10\\\").bars(width=4).label(regex=\\\".*:(.+)>.*\\\",label=\\\"$1\\\")\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "84f5c370-8a38-11e7-a58a-9bfcb3761a3d",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - TL - CriticalAssetsPluginNames",
-      "visState": "{\"title\":\"VulnWhisperer - TL - CriticalAssetsPluginNames\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*', q='tags:critical_asset', split=\\\"plugin_name.keyword:10\\\").bars(width=4).label(regex=\\\".*:(.+)>.*\\\",label=\\\"$1\\\")\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "307cdae0-8a38-11e7-a58a-9bfcb3761a3d",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - TL - PluginNames",
-      "visState": "{\"title\":\"VulnWhisperer - TL - PluginNames\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*', split=\\\"plugin_name.keyword:25\\\").bars(width=4).label(regex=\\\".*:(.+)>.*\\\",label=\\\"$1\\\")\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "5093c620-44e9-11e7-8014-ede06a7e69f8",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Mitigation Readme",
-      "visState": "{\"title\":\"VulnWhisperer - Mitigation Readme\",\"type\":\"markdown\",\"params\":{\"markdown\":\"** Legend **\\n\\n* [Common Vulnerability Scoring System (CVSS)](https://nvd.nist.gov/vuln-metrics/cvss) is the NIST vulnerability scoring system\\n* Risk Number is residual risk score calculated from CVSS, which is adjusted to be specific to the netowrk owner, which accounts for services not in use such as Java and Flash\\n* Vulnerabilities by Tag are systems tagged with HIPAA and PCI identification.\\n\\n\\n** Workflow **\\n* Select 10.0 under Risk Number to identify Critical Vulnerabilities. \\n* For more information about a CVE, scroll down and click the CVE link.\\n* To filter by tags, use one of the following filters:\\n** tags:has_hipaa_data, tags:pci_asset, tags:hipaa_asset, tags:critical_asset**\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "7e7fbc90-3df2-11e7-a44e-c79ca8efb780",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer-PluginID",
-      "visState": "{\"title\":\"VulnWhisperer-PluginID\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"plugin_id\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "5a3c0340-3eb3-11e7-a192-93f36fbd9d05",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer-CVSSHeatmap",
-      "visState": "{\"title\":\"VulnWhisperer-CVSSHeatmap\",\"type\":\"heatmap\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"enableHover\":false,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":4,\"colorSchema\":\"Yellow to Red\",\"setColorRange\":false,\"colorsRange\":[],\"invertColors\":false,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":false,\"rotate\":0,\"color\":\"#555\"}}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"host.keyword\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"cvss.keyword\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"_term\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 3500\":\"rgb(255,255,204)\",\"3500 - 7000\":\"rgb(254,217,118)\",\"7000 - 10500\":\"rgb(253,141,60)\",\"10500 - 14000\":\"rgb(227,27,28)\"}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "1de9e550-3df1-11e7-a44e-c79ca8efb780",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer-Description",
-      "visState": "{\"title\":\"VulnWhisperer-Description\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"description.keyword\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Description\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "13c7d4e0-3df3-11e7-a44e-c79ca8efb780",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer-Solution",
-      "visState": "{\"title\":\"VulnWhisperer-Solution\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"solution.keyword\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Solution\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "297df800-3f7e-11e7-bd24-6903e3283192",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Plugin Name",
-      "visState": "{\"title\":\"VulnWhisperer - Plugin Name\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"plugin_name.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Plugin Name\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "de1a5f40-3f85-11e7-97f9-3777d794626d",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - ScanName",
-      "visState": "{\"title\":\"VulnWhisperer - ScanName\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"scan_name.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Scan Name\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "ecbb99c0-3f84-11e7-97f9-3777d794626d",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Total",
-      "visState": "{\"title\":\"VulnWhisperer - Total\",\"type\":\"metric\",\"params\":{\"handleNoResults\":true,\"fontSize\":60},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Total\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "471a3580-3f6b-11e7-88e7-df1abe6547fb",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Vulnerabilities by Tag",
-      "visState": "{\"title\":\"VulnWhisperer - Vulnerabilities by Tag\",\"type\":\"table\",\"params\":{\"perPage\":3,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"bucket\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"tags:has_hipaa_data\",\"analyze_wildcard\":true}}},\"label\":\"Systems with HIPAA data\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"tags:pci_asset\",\"analyze_wildcard\":true}}},\"label\":\"PCI Systems\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"tags:hipaa_asset\",\"analyze_wildcard\":true}}},\"label\":\"HIPAA Systems\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "35b6d320-3f7f-11e7-bd24-6903e3283192",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Residual Risk",
-      "visState": "{\"title\":\"VulnWhisperer - Residual Risk\",\"type\":\"table\",\"params\":{\"perPage\":15,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"risk_score\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Risk Number\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "a9225930-3df2-11e7-a44e-c79ca8efb780",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer-Risk",
-      "visState": "{\"title\":\"VulnWhisperer-Risk\",\"type\":\"table\",\"params\":{\"perPage\":4,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"risk\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Risk Severity\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "2f979030-44b9-11e7-a818-f5f80dfc3590",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - ScanBarChart",
-      "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customLabel\":\"Scan Name\",\"field\":\"plugin_name.keyword\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":10},\"schema\":\"segment\",\"type\":\"terms\"}],\"listeners\":{},\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"defaultYExtents\":false,\"legendPosition\":\"right\",\"mode\":\"stacked\",\"scale\":\"linear\",\"setYExtents\":false,\"times\":[]},\"title\":\"VulnWhisperer - ScanBarChart\",\"type\":\"histogram\"}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "a6508640-897a-11e7-bbc0-33592ce0be1e",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Critical Assets Aggregated",
-      "visState": "{\"title\":\"VulnWhisperer - Critical Assets Aggregated\",\"type\":\"heatmap\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"enableHover\":true,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":4,\"colorSchema\":\"Green to Red\",\"setColorRange\":true,\"colorsRange\":[{\"from\":0,\"to\":3},{\"from\":3,\"to\":7},{\"from\":7,\"to\":9},{\"from\":9,\"to\":11}],\"invertColors\":false,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":true,\"rotate\":0,\"color\":\"white\"}}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"risk_score\",\"customLabel\":\"Residual Risk Score\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Date\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"host\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Critical Asset IP\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"plugin_name.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"row\":true}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"colors\":{\"0 - 3\":\"#7EB26D\",\"3 - 7\":\"#EAB839\",\"7 - 9\":\"#EF843C\",\"8 - 10\":\"#BF1B00\",\"9 - 11\":\"#BF1B00\"},\"defaultColors\":{\"0 - 3\":\"rgb(0,104,55)\",\"3 - 7\":\"rgb(135,203,103)\",\"7 - 9\":\"rgb(255,255,190)\",\"9 - 11\":\"rgb(249,142,82)\"},\"legendOpen\":false}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":\"Critical Asset\",\"disabled\":false,\"index\":\"logstash-vulnwhisperer-*\",\"key\":\"tags\",\"negate\":false,\"type\":\"phrase\",\"value\":\"critical_asset\"},\"query\":{\"match\":{\"tags\":{\"query\":\"critical_asset\",\"type\":\"phrase\"}}}}]}"
-      }
-    }
-  },
-  {
-    "_id": "099a3820-3f68-11e7-a6bd-e764d950e506",
-    "_type": "visualization",
-    "_source": {
-      "title": "Timelion VulnWhisperer Example",
-      "visState": "{\"type\":\"timelion\",\"title\":\"Timelion VulnWhisperer Example\",\"params\":{\"expression\":\".es(index=logstash-vulnwhisperer-*,q=risk:high).label(\\\"Current High Risk\\\"),.es(index=logstash-vulnwhisperer-*,q=risk:high,offset=-1y).label(\\\"Last 1 Year High Risk\\\"),.es(index=logstash-vulnwhisperer-*,q=risk:medium).label(\\\"Current Medium Risk\\\"),.es(index=logstash-vulnwhisperer-*,q=risk:medium,offset=-1y).label(\\\"Last 1 Year Medium Risk\\\")\",\"interval\":\"auto\"}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{}"
-      }
-    }
-  },
-  {
-    "_id": "67d432e0-44ec-11e7-a05f-d9719b331a27",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - TL-Critical Risk",
-      "visState": "{\"title\":\"VulnWhisperer - TL-Critical Risk\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*',q='(risk_score:>=9 AND risk_score:<=10)').label(\\\"Original\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=9 AND risk_score:<=10)',offset=-1w).label(\\\"One week offset\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=9 AND risk_score:<=10)').subtract(.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=9 AND risk_score:<=10)',offset=-1w)).label(\\\"Difference\\\").lines(steps=3,fill=2,width=1)\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "a91b9fe0-44ec-11e7-a05f-d9719b331a27",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - TL-Medium Risk",
-      "visState": "{\"title\":\"VulnWhisperer - TL-Medium Risk\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*',q='(risk_score:>=4 AND risk_score:<7)').label(\\\"Original\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=4 AND risk_score:<7)',offset=-1w).label(\\\"One week offset\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=4 AND risk_score:<7)').subtract(.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=4 AND risk_score:<7)',offset=-1w)).label(\\\"Difference\\\").lines(steps=3,fill=2,width=1)\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "8d9592d0-44ec-11e7-a05f-d9719b331a27",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - TL-High Risk",
-      "visState": "{\"title\":\"VulnWhisperer - TL-High Risk\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*',q='(risk_score:>=7 AND risk_score:<9)').label(\\\"Original\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=7 AND risk_score:<9)',offset=-1w).label(\\\"One week offset\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=7 AND risk_score:<9)').subtract(.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=7 AND risk_score:<9)',offset=-1w)).label(\\\"Difference\\\").lines(steps=3,fill=2,width=1)\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "a2d66660-44ec-11e7-a05f-d9719b331a27",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - TL-Low Risk",
-      "visState": "{\"title\":\"VulnWhisperer - TL-Low Risk\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*',q='(risk_score:>0 AND risk_score:<4)').label(\\\"Original\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>0 AND risk_score:<4)',offset=-1w).label(\\\"One week offset\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>0 AND risk_score:<4)').subtract(.es(index='logstash-vulnwhisperer-*',q='(risk_score:>0 AND risk_score:<4)',offset=-1w)).label(\\\"Difference\\\").lines(steps=3,fill=2,width=1)\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "fb6eb020-49ab-11e7-8f8c-57ad64ec48a6",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Critical Risk Score for Tagged Assets",
-      "visState": "{\"title\":\"VulnWhisperer - Critical Risk Score for Tagged Assets\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index=logstash-vulnwhisperer-*,q='risk_score:>9 AND tags:hipaa_asset').label(\\\"HIPAA Assets\\\"),.es(index=logstash-vulnwhisperer-*,q='risk_score:>9 AND tags:pci_asset').label(\\\"PCI Systems\\\"),.es(index=logstash-vulnwhisperer-*,q='risk_score:>9 AND tags:has_hipaa_data').label(\\\"Has HIPAA Data\\\")\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "b2f2adb0-897f-11e7-a2d2-c57bca21b3aa",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer  - Risk: Total",
-      "visState": "{\"title\":\"VulnWhisperer  - Risk: Total\",\"type\":\"goal\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"gauge\":{\"autoExtend\":false,\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"gaugeColorMode\":\"Background\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Metric\",\"invertColors\":false,\"labels\":{\"color\":\"black\",\"show\":false},\"orientation\":\"vertical\",\"percentageMode\":false,\"scale\":{\"color\":\"#333\",\"labels\":false,\"show\":true,\"width\":2},\"style\":{\"bgColor\":true,\"bgFill\":\"white\",\"fontSize\":\"34\",\"labelColor\":false,\"subText\":\"Risk\"},\"type\":\"simple\",\"useRanges\":false,\"verticalSplit\":false},\"type\":\"gauge\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Total\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}},\"label\":\"Critical\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"colors\":{\"0 - 10000\":\"#64B0C8\"},\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "465c5820-8977-11e7-857e-e1d56b17746d",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Critical Assets",
-      "visState": "{\"title\":\"VulnWhisperer - Critical Assets\",\"type\":\"heatmap\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"enableHover\":true,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":4,\"colorSchema\":\"Green to Red\",\"setColorRange\":true,\"colorsRange\":[{\"from\":0,\"to\":3},{\"from\":3,\"to\":7},{\"from\":7,\"to\":9},{\"from\":9,\"to\":11}],\"invertColors\":false,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":false,\"rotate\":0,\"color\":\"white\"}}],\"type\":\"heatmap\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"risk_score\",\"customLabel\":\"Residual Risk Score\"}},{\"id\":\"2\",\"enabled\":false,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"risk_score\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"row\":true}},{\"id\":\"3\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Date\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"asset.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Critical Asset\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 3\":\"rgb(0,104,55)\",\"3 - 7\":\"rgb(135,203,103)\",\"7 - 9\":\"rgb(255,255,190)\",\"9 - 11\":\"rgb(249,142,82)\"},\"colors\":{\"8 - 10\":\"#BF1B00\",\"9 - 11\":\"#BF1B00\",\"7 - 9\":\"#EF843C\",\"3 - 7\":\"#EAB839\",\"0 - 3\":\"#7EB26D\"},\"legendOpen\":false}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[{\"meta\":{\"index\":\"logstash-vulnwhisperer-*\",\"negate\":false,\"disabled\":false,\"alias\":\"Critical Asset\",\"type\":\"phrase\",\"key\":\"tags\",\"value\":\"critical_asset\"},\"query\":{\"match\":{\"tags\":{\"query\":\"critical_asset\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}]}"
-      }
-    }
-  },
-  {
-    "_id": "852816e0-3eb1-11e7-90cb-918f9cb01e3d",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer-CVSS",
-      "visState": "{\"title\":\"VulnWhisperer-CVSS\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"cvss.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"CVSS Score\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"asset.keyword\",\"customLabel\":\"# of Assets\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "d048c220-80b3-11e7-8790-73b60225f736",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer  - Risk: High",
-      "visState": "{\"title\":\"VulnWhisperer  - Risk: High\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Background\",\"colorsRange\":[{\"from\":0,\"to\":1000}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"bgFill\":\"white\",\"bgColor\":true,\"labelColor\":false,\"subText\":\"\",\"fontSize\":\"34\"},\"extendRange\":true}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"High Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:high\"}}},\"label\":\"\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 10000\":\"#EF843C\",\"0 - 1000\":\"#E0752D\"}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "db55bce0-80b3-11e7-8790-73b60225f736",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer  - Risk: Critical",
-      "visState": "{\"title\":\"VulnWhisperer  - Risk: Critical\",\"type\":\"goal\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"gauge\":{\"autoExtend\":false,\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"gaugeColorMode\":\"Background\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Metric\",\"invertColors\":false,\"labels\":{\"color\":\"black\",\"show\":false},\"orientation\":\"vertical\",\"percentageMode\":false,\"scale\":{\"color\":\"#333\",\"labels\":false,\"show\":true,\"width\":2},\"style\":{\"bgColor\":true,\"bgFill\":\"white\",\"fontSize\":\"34\",\"labelColor\":false,\"subText\":\"Risk\"},\"type\":\"simple\",\"useRanges\":false,\"verticalSplit\":false},\"type\":\"gauge\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Critical Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:critical\"}}},\"label\":\"Critical\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"colors\":{\"0 - 10000\":\"#BF1B00\"},\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "56f0f5f0-3ebe-11e7-a192-93f36fbd9d05",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer-RiskOverTime",
-      "visState": "{\"title\":\"VulnWhisperer-RiskOverTime\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per 12 hours\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"orderBucketsBySum\":false,\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"mode\":\"normal\",\"show\":\"true\",\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}],\"type\":\"line\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:info\"}}},\"label\":\"Info\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:low\"}}},\"label\":\"Low\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:medium\"}}},\"label\":\"Medium\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:high\"}}},\"label\":\"High\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:critical\"}}},\"label\":\"Critical\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"colors\":{\"Critical\":\"#962D82\",\"High\":\"#BF1B00\",\"Low\":\"#629E51\",\"Medium\":\"#EAB839\",\"Info\":\"#65C5DB\"}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "c1361da0-80b3-11e7-8790-73b60225f736",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer  - Risk: Medium",
-      "visState": "{\"title\":\"VulnWhisperer  - Risk: Medium\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Background\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"bgFill\":\"white\",\"bgColor\":true,\"labelColor\":false,\"subText\":\"\",\"fontSize\":\"34\"},\"extendRange\":false},\"isDisplayWarning\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Medium Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:medium\"}}},\"label\":\"Medium Risk\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 10000\":\"#EAB839\"}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "e46ff7f0-897d-11e7-934b-67cec0a7da65",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer  - Risk: Low",
-      "visState": "{\"title\":\"VulnWhisperer  - Risk: Low\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Background\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"bgFill\":\"white\",\"bgColor\":true,\"labelColor\":false,\"subText\":\"\",\"fontSize\":\"34\"},\"extendRange\":false}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Low Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:low\"}}},\"label\":\"Low Risk\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 10000\":\"#629E51\"}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "995e2280-3df3-11e7-a44e-c79ca8efb780",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer-Asset",
-      "visState": "{\"title\":\"VulnWhisperer-Asset\",\"type\":\"table\",\"params\":{\"perPage\":15,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"asset.keyword\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Asset\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  }
-]

resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/1001_vulnWhisperer_ReportingMitigationDashboard.json

@@ -1,43 +0,0 @@
-[
-  {
-    "_id": "72051530-448e-11e7-a818-f5f80dfc3590",
-    "_type": "dashboard",
-    "_source": {
-      "title": "VulnWhisperer - Reporting",
-      "hits": 0,
-      "description": "",
-      "panelsJSON": "[{\"col\":1,\"id\":\"2f979030-44b9-11e7-a818-f5f80dfc3590\",\"panelIndex\":5,\"row\":12,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"8d9592d0-44ec-11e7-a05f-d9719b331a27\",\"panelIndex\":12,\"row\":8,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"67d432e0-44ec-11e7-a05f-d9719b331a27\",\"panelIndex\":14,\"row\":4,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":10,\"id\":\"297df800-3f7e-11e7-bd24-6903e3283192\",\"panelIndex\":15,\"row\":8,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"471a3580-3f6b-11e7-88e7-df1abe6547fb\",\"panelIndex\":20,\"row\":8,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":11,\"id\":\"995e2280-3df3-11e7-a44e-c79ca8efb780\",\"panelIndex\":22,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":9,\"id\":\"b2f2adb0-897f-11e7-a2d2-c57bca21b3aa\",\"panelIndex\":23,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"db55bce0-80b3-11e7-8790-73b60225f736\",\"panelIndex\":25,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"d048c220-80b3-11e7-8790-73b60225f736\",\"panelIndex\":26,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"e46ff7f0-897d-11e7-934b-67cec0a7da65\",\"panelIndex\":27,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":3,\"id\":\"c1361da0-80b3-11e7-8790-73b60225f736\",\"panelIndex\":28,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"479deab0-8a39-11e7-a58a-9bfcb3761a3d\",\"panelIndex\":29,\"row\":4,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"}]",
-      "optionsJSON": "{\"darkTheme\":false}",
-      "uiStateJSON": "{\"P-15\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-20\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-21\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-22\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-23\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-24\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-25\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-26\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-27\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-28\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-5\":{\"vis\":{\"legendOpen\":false}}}",
-      "version": 1,
-      "timeRestore": true,
-      "timeTo": "now",
-      "timeFrom": "now-1y",
-      "refreshInterval": {
-        "display": "Off",
-        "pause": false,
-        "value": 0
-      },
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"filter\":[{\"query\":{\"match_all\":{}}}],\"highlightAll\":true,\"version\":true}"
-      }
-    }
-  },
-  {
-    "_id": "AWCUqesWib22Ai8JwW3u",
-    "_type": "dashboard",
-    "_source": {
-      "title": "VulnWhisperer - Risk Mitigation",
-      "hits": 0,
-      "description": "",
-      "panelsJSON": "[{\"col\":11,\"id\":\"995e2280-3df3-11e7-a44e-c79ca8efb780\",\"panelIndex\":20,\"row\":8,\"size_x\":2,\"size_y\":6,\"type\":\"visualization\"},{\"col\":1,\"id\":\"852816e0-3eb1-11e7-90cb-918f9cb01e3d\",\"panelIndex\":21,\"row\":10,\"size_x\":3,\"size_y\":5,\"type\":\"visualization\"},{\"col\":4,\"id\":\"297df800-3f7e-11e7-bd24-6903e3283192\",\"panelIndex\":27,\"row\":8,\"size_x\":3,\"size_y\":5,\"type\":\"visualization\"},{\"col\":9,\"id\":\"35b6d320-3f7f-11e7-bd24-6903e3283192\",\"panelIndex\":28,\"row\":8,\"size_x\":2,\"size_y\":6,\"type\":\"visualization\"},{\"col\":11,\"id\":\"471a3580-3f6b-11e7-88e7-df1abe6547fb\",\"panelIndex\":30,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"de1a5f40-3f85-11e7-97f9-3777d794626d\",\"panelIndex\":31,\"row\":8,\"size_x\":2,\"size_y\":5,\"type\":\"visualization\"},{\"col\":10,\"id\":\"5093c620-44e9-11e7-8014-ede06a7e69f8\",\"panelIndex\":37,\"row\":4,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"columns\":[\"host\",\"risk\",\"risk_score\",\"cve\",\"plugin_name\",\"solution\",\"plugin_output\"],\"id\":\"54648700-3f74-11e7-852e-69207a3d0726\",\"panelIndex\":38,\"row\":15,\"size_x\":12,\"size_y\":6,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"fb6eb020-49ab-11e7-8f8c-57ad64ec48a6\",\"panelIndex\":39,\"row\":8,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":5,\"id\":\"465c5820-8977-11e7-857e-e1d56b17746d\",\"panelIndex\":40,\"row\":4,\"size_x\":5,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"56f0f5f0-3ebe-11e7-a192-93f36fbd9d05\",\"panelIndex\":46,\"row\":4,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"e46ff7f0-897d-11e7-934b-67cec0a7da65\",\"panelIndex\":47,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":3,\"id\":\"c1361da0-80b3-11e7-8790-73b60225f736\",\"panelIndex\":48,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"d048c220-80b3-11e7-8790-73b60225f736\",\"panelIndex\":49,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"db55bce0-80b3-11e7-8790-73b60225f736\",\"panelIndex\":50,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":9,\"id\":\"b2f2adb0-897f-11e7-a2d2-c57bca21b3aa\",\"panelIndex\":51,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"}]",
-      "optionsJSON": "{\"darkTheme\":false}",
-      "uiStateJSON": "{\"P-11\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-2\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-20\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-21\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}},\"P-27\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-28\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}},\"P-3\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"asc\"}}}},\"P-30\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-31\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-40\":{\"vis\":{\"defaultColors\":{\"0 - 3\":\"rgb(0,104,55)\",\"3 - 7\":\"rgb(135,203,103)\",\"7 - 9\":\"rgb(255,255,190)\",\"9 - 11\":\"rgb(249,142,82)\"}}},\"P-41\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-42\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-43\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"}}},\"P-44\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-45\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-46\":{\"vis\":{\"legendOpen\":true}},\"P-47\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-48\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-49\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-5\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-50\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-51\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-6\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-8\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}",
-      "version": 1,
-      "timeRestore": false,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"filter\":[{\"query\":{\"match_all\":{}}}],\"highlightAll\":true,\"version\":true}"
-      }
-    }
-  }
-]

resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/2000_vulnWhisperer_QualysVisuals (required with Dashboard).json

@@ -1,170 +0,0 @@
-[
-  {
-    "_id": "AWCUo-jRib22Ai8JwW1N",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer  - Risk: High Qualys Scoring",
-      "visState": "{\"title\":\"VulnWhisperer  - Risk: High Qualys Scoring\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Background\",\"colorsRange\":[{\"from\":0,\"to\":1000}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"bgFill\":\"white\",\"bgColor\":true,\"labelColor\":false,\"subText\":\"\",\"fontSize\":\"34\"},\"extendRange\":true}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"High Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:high\"}}},\"label\":\"\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 10000\":\"#EF843C\",\"0 - 1000\":\"#E0752D\"}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCUozGBib22Ai8JwW1B",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer  - Risk: Medium Qualys Scoring",
-      "visState": "{\"title\":\"VulnWhisperer  - Risk: Medium Qualys Scoring\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Background\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"bgFill\":\"white\",\"bgColor\":true,\"labelColor\":false,\"subText\":\"\",\"fontSize\":\"34\"},\"extendRange\":false}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Medium Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:medium\"}}},\"label\":\"Medium Risk\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 10000\":\"#EAB839\"}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCUpE3Kib22Ai8JwW1c",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer  - Risk: Critical Qualys Scoring",
-      "visState": "{\"title\":\"VulnWhisperer  - Risk: Critical Qualys Scoring\",\"type\":\"goal\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"gauge\":{\"autoExtend\":false,\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"gaugeColorMode\":\"Background\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Metric\",\"invertColors\":false,\"labels\":{\"color\":\"black\",\"show\":false},\"orientation\":\"vertical\",\"percentageMode\":false,\"scale\":{\"color\":\"#333\",\"labels\":false,\"show\":true,\"width\":2},\"style\":{\"bgColor\":true,\"bgFill\":\"white\",\"fontSize\":\"34\",\"labelColor\":false,\"subText\":\"Risk\"},\"type\":\"simple\",\"useRanges\":false,\"verticalSplit\":false},\"type\":\"gauge\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Critical Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:critical\"}}},\"label\":\"Critical\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"colors\":{\"0 - 10000\":\"#BF1B00\"},\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCUyeHGib22Ai8JwX62",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer-RiskOverTime Qualys Scoring",
-      "visState": "{\"title\":\"VulnWhisperer-RiskOverTime Qualys Scoring\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per 12 hours\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"orderBucketsBySum\":false,\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"mode\":\"normal\",\"show\":\"true\",\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}],\"type\":\"line\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:info\"}}},\"label\":\"Info\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:low\"}}},\"label\":\"Low\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:medium\"}}},\"label\":\"Medium\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:high\"}}},\"label\":\"High\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:critical\"}}},\"label\":\"Critical\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"colors\":{\"Critical\":\"#962D82\",\"High\":\"#BF1B00\",\"Low\":\"#629E51\",\"Medium\":\"#EAB839\",\"Info\":\"#65C5DB\"}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCUos-Fib22Ai8JwW0y",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer  - Risk: Low Qualys Scoring",
-      "visState": "{\"title\":\"VulnWhisperer  - Risk: Low Qualys Scoring\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Background\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"bgFill\":\"white\",\"bgColor\":true,\"labelColor\":false,\"subText\":\"\",\"fontSize\":\"34\"},\"extendRange\":false}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Low Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:low\"}}},\"label\":\"Low Risk\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 10000\":\"#629E51\"}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCg9Wsfib22Ai8Jww3v",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Qualys: Category Description",
-      "visState": "{\"title\":\"VulnWhisperer - Qualys: Category Description\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"category_description.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Category Description\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"match_all\":{}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCg88f1ib22Ai8Jww3C",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - QualysOS",
-      "visState": "{\"title\":\"VulnWhisperer - QualysOS\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"operating_system.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"match_all\":{}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCg9JUAib22Ai8Jww3Y",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - QualysOwner",
-      "visState": "{\"title\":\"VulnWhisperer - QualysOwner\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"owner.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"match_all\":{}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCg9tE6ib22Ai8Jww4R",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Qualys: Impact",
-      "visState": "{\"title\":\"VulnWhisperer - Qualys: Impact\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"impact.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Impact\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"match_all\":{}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCg9igvib22Ai8Jww36",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Qualys: Level",
-      "visState": "{\"title\":\"VulnWhisperer - Qualys: Level\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"level.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Level\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"match_all\":{}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCUsp_3ib22Ai8JwW7R",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - TL-Critical Risk Qualys Scoring",
-      "visState": "{\"title\":\"VulnWhisperer - TL-Critical Risk Qualys Scoring\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*',q='(risk:critical)').label(\\\"Original\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk:critical)',offset=-1w).label(\\\"One week offset\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk:critical)').subtract(.es(index='logstash-vulnwhisperer-*',q='(risk:critical)',offset=-1w)).label(\\\"Difference\\\").lines(steps=3,fill=2,width=1)\",\"interval\":\"auto\",\"type\":\"timelion\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCUtHETib22Ai8JwW79",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - TL-High Risk Qualys Scoring",
-      "visState": "{\"title\":\"VulnWhisperer - TL-High Risk Qualys Scoring\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*',q='(risk:high)').label(\\\"Original\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk:high)',offset=-1w).label(\\\"One week offset\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk:high)').subtract(.es(index='logstash-vulnwhisperer-*',q='(risk:high)',offset=-1w)).label(\\\"Difference\\\").lines(steps=3,fill=2,width=1)\",\"interval\":\"auto\",\"type\":\"timelion\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  }
-]

resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/2001_vulnWhisperer_ReportingMitigationDashboardQualysRisk.json

@@ -1,50 +0,0 @@
-[
-  {
-    "_id": "AWCUrIBqib22Ai8JwW43",
-    "_type": "dashboard",
-    "_source": {
-      "title": "VulnWhisperer - Reporting Qualys Scoring",
-      "hits": 0,
-      "description": "",
-      "panelsJSON": "[{\"col\":1,\"id\":\"2f979030-44b9-11e7-a818-f5f80dfc3590\",\"panelIndex\":5,\"row\":11,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":10,\"id\":\"297df800-3f7e-11e7-bd24-6903e3283192\",\"panelIndex\":15,\"row\":7,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"471a3580-3f6b-11e7-88e7-df1abe6547fb\",\"panelIndex\":20,\"row\":7,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":11,\"id\":\"995e2280-3df3-11e7-a44e-c79ca8efb780\",\"panelIndex\":22,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":9,\"id\":\"b2f2adb0-897f-11e7-a2d2-c57bca21b3aa\",\"panelIndex\":23,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"479deab0-8a39-11e7-a58a-9bfcb3761a3d\",\"panelIndex\":29,\"row\":4,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"size_x\":6,\"size_y\":3,\"panelIndex\":30,\"type\":\"visualization\",\"id\":\"AWCUtHETib22Ai8JwW79\",\"col\":1,\"row\":8},{\"size_x\":6,\"size_y\":3,\"panelIndex\":31,\"type\":\"visualization\",\"id\":\"AWCUsp_3ib22Ai8JwW7R\",\"col\":7,\"row\":4},{\"size_x\":2,\"size_y\":3,\"panelIndex\":33,\"type\":\"visualization\",\"id\":\"AWCUozGBib22Ai8JwW1B\",\"col\":3,\"row\":1},{\"size_x\":2,\"size_y\":3,\"panelIndex\":34,\"type\":\"visualization\",\"id\":\"AWCUo-jRib22Ai8JwW1N\",\"col\":5,\"row\":1},{\"size_x\":2,\"size_y\":3,\"panelIndex\":35,\"type\":\"visualization\",\"id\":\"AWCUpE3Kib22Ai8JwW1c\",\"col\":7,\"row\":1},{\"size_x\":2,\"size_y\":3,\"panelIndex\":36,\"type\":\"visualization\",\"id\":\"AWCUos-Fib22Ai8JwW0y\",\"col\":1,\"row\":1}]",
-      "optionsJSON": "{\"darkTheme\":false}",
-      "uiStateJSON": "{\"P-15\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-20\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-21\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-22\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-23\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-24\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-5\":{\"vis\":{\"legendOpen\":false}},\"P-33\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-34\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-35\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-27\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-28\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-26\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"}}},\"P-25\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-32\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-36\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}}}",
-      "version": 1,
-      "timeRestore": true,
-      "timeTo": "now",
-      "timeFrom": "now-30d",
-      "refreshInterval": {
-        "display": "Off",
-        "pause": false,
-        "value": 0
-      },
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"-vulnerability_category:\\\"INFORMATION_GATHERED\\\"\"}}}],\"highlightAll\":true,\"version\":true}"
-      }
-    }
-  },
-  {
-    "_id": "5dba30c0-3df3-11e7-a44e-c79ca8efb780",
-    "_type": "dashboard",
-    "_source": {
-      "title": "VulnWhisperer - Risk Mitigation Qualys Web Scoring",
-      "hits": 0,
-      "description": "",
-      "panelsJSON": "[{\"col\":11,\"id\":\"995e2280-3df3-11e7-a44e-c79ca8efb780\",\"panelIndex\":20,\"row\":8,\"size_x\":2,\"size_y\":7,\"type\":\"visualization\"},{\"col\":1,\"id\":\"852816e0-3eb1-11e7-90cb-918f9cb01e3d\",\"panelIndex\":21,\"row\":10,\"size_x\":3,\"size_y\":5,\"type\":\"visualization\"},{\"col\":4,\"id\":\"297df800-3f7e-11e7-bd24-6903e3283192\",\"panelIndex\":27,\"row\":8,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":9,\"id\":\"35b6d320-3f7f-11e7-bd24-6903e3283192\",\"panelIndex\":28,\"row\":8,\"size_x\":2,\"size_y\":7,\"type\":\"visualization\"},{\"col\":11,\"id\":\"471a3580-3f6b-11e7-88e7-df1abe6547fb\",\"panelIndex\":30,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"de1a5f40-3f85-11e7-97f9-3777d794626d\",\"panelIndex\":31,\"row\":8,\"size_x\":2,\"size_y\":4,\"type\":\"visualization\"},{\"col\":10,\"id\":\"5093c620-44e9-11e7-8014-ede06a7e69f8\",\"panelIndex\":37,\"row\":4,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"columns\":[\"host\",\"risk\",\"risk_score\",\"cve\",\"plugin_name\",\"solution\",\"plugin_output\"],\"id\":\"54648700-3f74-11e7-852e-69207a3d0726\",\"panelIndex\":38,\"row\":15,\"size_x\":12,\"size_y\":6,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"fb6eb020-49ab-11e7-8f8c-57ad64ec48a6\",\"panelIndex\":39,\"row\":8,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":5,\"id\":\"465c5820-8977-11e7-857e-e1d56b17746d\",\"panelIndex\":40,\"row\":4,\"size_x\":5,\"size_y\":4,\"type\":\"visualization\"},{\"col\":9,\"id\":\"b2f2adb0-897f-11e7-a2d2-c57bca21b3aa\",\"panelIndex\":45,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AWCUos-Fib22Ai8JwW0y\",\"panelIndex\":47,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":3,\"id\":\"AWCUozGBib22Ai8JwW1B\",\"panelIndex\":48,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"AWCUo-jRib22Ai8JwW1N\",\"panelIndex\":49,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"AWCUpE3Kib22Ai8JwW1c\",\"panelIndex\":50,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AWCUyeHGib22Ai8JwX62\",\"panelIndex\":51,\"row\":4,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":4,\"id\":\"AWCg88f1ib22Ai8Jww3C\",\"panelIndex\":52,\"row\":12,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"AWCg9JUAib22Ai8Jww3Y\",\"panelIndex\":53,\"row\":12,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"}]",
-      "optionsJSON": "{\"darkTheme\":false}",
-      "uiStateJSON": "{\"P-11\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-2\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-20\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-21\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}},\"P-27\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-28\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}},\"P-3\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"asc\"}}}},\"P-30\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-31\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-40\":{\"vis\":{\"defaultColors\":{\"0 - 3\":\"rgb(0,104,55)\",\"3 - 7\":\"rgb(135,203,103)\",\"7 - 9\":\"rgb(255,255,190)\",\"9 - 11\":\"rgb(249,142,82)\"}}},\"P-41\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-42\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-43\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"}}},\"P-44\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-45\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-47\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-48\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-49\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-5\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-50\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-6\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-8\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-52\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-53\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}",
-      "version": 1,
-      "timeRestore": true,
-      "timeTo": "now",
-      "timeFrom": "now-30d",
-      "refreshInterval": {
-        "display": "Off",
-        "pause": false,
-        "value": 0
-      },
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"-vulnerability_category:\\\"INFORMATION_GATHERED\\\"\"}}}],\"highlightAll\":true,\"version\":true}"
-      }
-    }
-  }
-]

resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/9000_vulnWhisperer_SavedSearch.json

@@ -1,28 +0,0 @@
-[
-  {
-    "_id": "54648700-3f74-11e7-852e-69207a3d0726",
-    "_type": "search",
-    "_source": {
-      "title": "VulnWhisperer - Saved Search",
-      "description": "",
-      "hits": 0,
-      "columns": [
-        "host",
-        "risk",
-        "risk_score",
-        "cve",
-        "plugin_name",
-        "solution",
-        "plugin_output"
-      ],
-      "sort": [
-        "@timestamp",
-        "desc"
-      ],
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}"
-      }
-    }
-  }
-]

resources/elk5-old_compatibility/logstash/0001_input_beats.conf

@@ -1,14 +0,0 @@
-input {
-  beats {
-    port => 5044
-    tags => "beats"
-  }
-}
-
-filter {
-  if [beat][hostname] == "filebeathost" {
-    mutate {
-      add_tag => ["nessus"]
-    }
-  }
-}

resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf

@@ -1,220 +0,0 @@
-# Author: Austin Taylor and Justin Henderson
-# Email: email@austintaylor.io
-# Last Update: 12/20/2017
-# Version 0.3
-# Description: Take in nessus reports from vulnWhisperer and pumps into logstash
-
-
-input {
-  file {
-    path => "/opt/VulnWhisperer/nessus/**/*"
-    start_position => "beginning"
-    tags => "nessus"
-    type => "nessus"
-  }
-  file {
-    path => "/opt/VulnWhisperer/tenable/*.csv"
-    start_position => "beginning"
-    tags => "tenable"
-    type => "tenable"
-  }
-}
-
-filter {
-  if "nessus" in [tags] or "tenable" in [tags] {
-    # Drop the header column
-    if [message] =~ "^Plugin ID" { drop {} }
-
-    csv {
-      # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"]
-      columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"]
-      separator => ","
-      source => "message"
-    }
-
-    ruby {
-        code => "if event.get('description')
-                    event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('synopsis')
-                    event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('solution')
-                    event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('see_also')
-                    event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('plugin_output')
-                    event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end"
-    }
-
-    #If using filebeats as your source, you will need to replace the "path" field to "source"
-    grok {
-      match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.csv$" }
-      tag_on_failure => []
-    }
-
-    date {
-      match => [ "last_updated", "UNIX" ]
-      target => "@timestamp"
-      remove_field => ["last_updated"]
-    }
-
-    if [risk] == "None" {
-      mutate { add_field => { "risk_number" => 0 }}
-    }
-    if [risk] == "Low" {
-      mutate { add_field => { "risk_number" => 1 }}
-    }
-    if [risk] == "Medium" {
-      mutate { add_field => { "risk_number" => 2 }}
-    }
-    if [risk] == "High" {
-      mutate { add_field => { "risk_number" => 3 }}
-    }
-    if [risk] == "Critical" {
-      mutate { add_field => { "risk_number" => 4 }}
-    }
-
-    if ![cve] or [cve] == "nan" {
-      mutate { remove_field => [ "cve" ] }
-    }
-    if ![cvss] or [cvss] == "nan" {
-      mutate { remove_field => [ "cvss" ] }
-    }
-    if ![cvss_base] or [cvss_base] == "nan" {
-      mutate { remove_field => [ "cvss_base" ] }
-    }
-    if ![cvss_temporal] or [cvss_temporal] == "nan" {
-      mutate { remove_field => [ "cvss_temporal" ] }
-    }
-    if ![cvss_temporal_vector] or [cvss_temporal_vector] == "nan" {
-      mutate { remove_field => [ "cvss_temporal_vector" ] }
-    }
-    if ![cvss_vector] or [cvss_vector] == "nan" {
-      mutate { remove_field => [ "cvss_vector" ] }
-    }
-    if ![cvss3_base] or [cvss3_base] == "nan" {
-      mutate { remove_field => [ "cvss3_base" ] }
-    }
-    if ![cvss3_temporal] or [cvss3_temporal] == "nan" {
-      mutate { remove_field => [ "cvss3_temporal" ] }
-    }
-    if ![cvss3_temporal_vector] or [cvss3_temporal_vector] == "nan" {
-      mutate { remove_field => [ "cvss3_temporal_vector" ] }
-    }
-    if ![description] or [description] == "nan" {
-      mutate { remove_field => [ "description" ] }
-    }
-    if ![mac_address] or [mac_address] == "nan" {
-      mutate { remove_field => [ "mac_address" ] }
-    }
-    if ![netbios] or [netbios] == "nan" {
-      mutate { remove_field => [ "netbios" ] }
-    }
-    if ![operating_system] or [operating_system] == "nan" {
-      mutate { remove_field => [ "operating_system" ] }
-    }
-    if ![plugin_output] or [plugin_output] == "nan" {
-      mutate { remove_field => [ "plugin_output" ] }
-    }
-    if ![see_also] or [see_also] == "nan" {
-      mutate { remove_field => [ "see_also" ] }
-    }
-    if ![synopsis] or [synopsis] == "nan" {
-      mutate { remove_field => [ "synopsis" ] }
-    }
-    if ![system_type] or [system_type] == "nan" {
-      mutate { remove_field => [ "system_type" ] }
-    }
-
-    mutate {
-      remove_field => [ "message" ]
-      add_field => { "risk_score" => "%{cvss}" }
-    }
-    mutate {
-      convert => { "risk_score" => "float" }
-    }
-    if [risk_score] == 0 {
-      mutate {
-        add_field => { "risk_score_name" => "info" }
-      }
-    }
-    if [risk_score] > 0 and [risk_score] < 3 {
-      mutate {
-        add_field => { "risk_score_name" => "low" }
-      }
-    }
-    if [risk_score] >= 3 and [risk_score] < 6 {
-      mutate {
-        add_field => { "risk_score_name" => "medium" }
-      }
-    }
-    if [risk_score] >=6 and [risk_score] < 9 {
-      mutate {
-        add_field => { "risk_score_name" => "high" }
-      }
-    }
-    if [risk_score] >= 9 {
-      mutate {
-        add_field => { "risk_score_name" => "critical" }
-      }
-    }
-
-    # Compensating controls - adjust risk_score
-    # Adobe and Java are not allowed to run in browser unless whitelisted
-    # Therefore, lower score by dividing by 3 (score is subjective to risk)
-
-    #Modify and uncomment when ready to use
-    #if [risk_score] != 0 {
-    #  if [plugin_name] =~ "Adobe" and [risk_score] > 6 or [plugin_name] =~ "Java" and [risk_score] > 6 {
-    #    ruby {
-    #      code => "event.set('risk_score', event.get('risk_score') / 3)"
-    #    }
-    #    mutate {
-    #      add_field => { "compensating_control" => "Adobe and Flash removed from browsers unless whitelisted site." }
-    #    }
-    #  }
-    #}
-
-    # Add tags for reporting based on assets or criticality
-
-    if [asset] == "dc01" or [asset] == "dc02" or [asset] == "pki01" or [asset] == "192.168.0.54" or [asset] =~ "^192\.168\.0\." or [asset] =~ "^42.42.42." {
-      mutate {
-        add_tag => [ "critical_asset" ]
-      }
-    }
-    #if [asset] =~ "^192\.168\.[45][0-9][0-9]\.1$" or [asset] =~ "^192.168\.[50]\.[0-9]{1,2}\.1$"{
-    #  mutate {
-    #    add_tag => [ "has_hipaa_data" ]
-    #  }
-    #}
-    #if [asset] =~ "^192\.168\.[45][0-9][0-9]\." {
-    #  mutate {
-    #    add_tag => [ "hipaa_asset" ]
-    #  }
-    #}
-    if [asset] =~ "^hr" {
-      mutate {
-        add_tag => [ "pci_asset" ]
-      }
-    }
-    #if [asset] =~ "^10\.0\.50\." {
-    #  mutate {
-    #    add_tag => [ "web_servers" ]
-    #  }
-    #}
-  }
-}
-
-output {
-  if "nessus" in [tags] or "tenable" in [tags] or [type] in [ "nessus", "tenable" ] {
-    # stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => [ "localhost:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
-    }
-  }
-}

resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf

@@ -1,153 +0,0 @@
-# Author: Austin Taylor and Justin Henderson
-# Email: austin@hasecuritysolutions.com
-# Last Update: 12/30/2017
-# Version 0.3
-# Description: Take in qualys web scan reports from vulnWhisperer and pumps into logstash
-
-input {
-  file {
-    path => [ "/opt/VulnWhisperer/data/qualys/*.json" , "/opt/VulnWhisperer/data/qualys_web/*.json", "/opt/VulnWhisperer/data/qualys_vuln/*.json" ]
-    type => json
-    codec => json
-    start_position => "beginning"
-    tags => [ "qualys" ]
-  }
-}
-
-filter {
-  if "qualys" in [tags] {
-    grok {
-      match => { "path" => [ "(?<tags>qualys_vuln)_scan_%{DATA}_%{INT:last_updated}.json$", "(?<tags>qualys_web)_%{INT:app_id}_%{INT:last_updated}.json$" ] }
-      tag_on_failure => []
-    }
-    
-    mutate {
-      replace => [ "message", "%{message}" ]
-      #gsub => [
-      #  "message", "\|\|\|", " ",
-      #  "message", "\t\t", " ",
-      #  "message", "    ", " ",
-      #  "message", "   ", " ",
-      #  "message", "  ", " ",
-      #  "message", "nan", " ",
-      #  "message",'\n',''
-      #]
-    }
-
-    if "qualys_web" in [tags] {
-    mutate {
-      add_field => { "asset" => "%{web_application_name}" }
-      add_field => { "risk_score" => "%{cvss}" }
-      }
-    } else if "qualys_vuln" in [tags] {
-      mutate {
-        add_field => { "asset" => "%{ip}" }
-        add_field => { "risk_score" => "%{cvss}" }
-      }
-    }
-
-    if [risk] == "1" {
-      mutate { add_field => { "risk_number" => 0 }}
-      mutate { replace => { "risk" => "info" }}
-    }
-    if [risk] == "2" {
-      mutate { add_field => { "risk_number" => 1 }}
-      mutate { replace => { "risk" => "low" }}
-    }
-    if [risk] == "3" {
-      mutate { add_field => { "risk_number" => 2 }}
-      mutate { replace => { "risk" => "medium" }}
-    }
-    if [risk] == "4" {
-      mutate { add_field => { "risk_number" => 3 }}
-      mutate { replace => { "risk" => "high" }}
-    }
-    if [risk] == "5" {
-      mutate { add_field => { "risk_number" => 4 }}
-      mutate { replace => { "risk" => "critical" }}
-    }
-
-    mutate {
-      remove_field => "message"
-    }
-
-    if [first_time_detected] {
-      date {
-        match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "first_time_detected"
-      }
-    }
-    if [first_time_tested] {
-      date {
-        match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "first_time_tested"
-      }
-    }
-    if [last_time_detected] {
-      date {
-        match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "last_time_detected"
-      }
-    }
-    if [last_time_tested] {
-      date {
-        match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "last_time_tested"
-      }
-    }
-    date {
-      match => [ "last_updated", "UNIX" ]
-      target => "@timestamp"
-      remove_field => "last_updated"
-    }
-    mutate {
-      convert => { "plugin_id" => "integer"}
-      convert => { "id" => "integer"}
-      convert => { "risk_number" => "integer"}
-      convert => { "risk_score" => "float"}
-      convert => { "total_times_detected" => "integer"}
-      convert => { "cvss_temporal" => "float"}
-      convert => { "cvss" => "float"}
-    }
-    if [risk_score] == 0 {
-      mutate {
-        add_field => { "risk_score_name" => "info" }
-      }
-    }
-    if [risk_score] > 0 and [risk_score] < 3 {
-      mutate {
-        add_field => { "risk_score_name" => "low" }
-      }
-    }
-    if [risk_score] >= 3 and [risk_score] < 6 {
-      mutate {
-        add_field => { "risk_score_name" => "medium" }
-      }
-    }
-    if [risk_score] >=6 and [risk_score] < 9 {
-      mutate {
-        add_field => { "risk_score_name" => "high" }
-      }
-    }
-    if [risk_score] >= 9 {
-      mutate {
-        add_field => { "risk_score_name" => "critical" }
-      }
-    }
-
-    if [asset] =~ "\.yourdomain\.(com|net)$" {
-      mutate {
-        add_tag => [ "critical_asset" ]
-      }
-    }
-  }
-}
-output {
-  if "qualys" in [tags] {
-    stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => [ "localhost:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
-    }
-  }
-}

resources/elk5-old_compatibility/logstash/3000_openvas.conf

@@ -1,146 +0,0 @@
-# Author: Austin Taylor and Justin Henderson
-# Email: austin@hasecuritysolutions.com
-# Last Update: 03/04/2018
-# Version 0.3
-# Description: Take in qualys web scan reports from vulnWhisperer and pumps into logstash
-
-input {
-  file {
-    path => "/opt/VulnWhisperer/openvas/*.json"
-    type => json
-    codec => json
-    start_position => "beginning"
-    tags => [ "openvas_scan", "openvas" ]
-  }
-}
-
-filter {
-  if "openvas_scan" in [tags] {
-    mutate {
-      replace => [ "message", "%{message}" ]
-      gsub => [
-        "message", "\|\|\|", " ",
-        "message", "\t\t", " ",
-        "message", "    ", " ",
-        "message", "   ", " ",
-        "message", "  ", " ",
-        "message", "nan", " ",
-        "message",'\n',''
-      ]
-    }
-
-
-    grok {
-        match => { "path" => "openvas_scan_%{DATA:scan_id}_%{INT:last_updated}.json$" }
-        tag_on_failure => []
-    }
-
-    mutate {
-      add_field => { "risk_score" => "%{cvss}" }
-    }
-
-    if [risk] == "1" {
-      mutate { add_field => { "risk_number" => 0 }}
-      mutate { replace => { "risk" => "info" }}
-    }
-    if [risk] == "2" {
-      mutate { add_field => { "risk_number" => 1 }}
-      mutate { replace => { "risk" => "low" }}
-    }
-    if [risk] == "3" {
-      mutate { add_field => { "risk_number" => 2 }}
-      mutate { replace => { "risk" => "medium" }}
-    }
-    if [risk] == "4" {
-      mutate { add_field => { "risk_number" => 3 }}
-      mutate { replace => { "risk" => "high" }}
-    }
-    if [risk] == "5" {
-      mutate { add_field => { "risk_number" => 4 }}
-      mutate { replace => { "risk" => "critical" }}
-    }
-
-    mutate {
-      remove_field => "message"
-    }
-
-    if [first_time_detected] {
-      date {
-        match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "first_time_detected"
-      }
-    }
-    if [first_time_tested] {
-      date {
-        match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "first_time_tested"
-      }
-    }
-    if [last_time_detected] {
-      date {
-        match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "last_time_detected"
-      }
-    }
-    if [last_time_tested] {
-      date {
-        match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "last_time_tested"
-      }
-    }
-    date {
-      match => [ "last_updated", "UNIX" ]
-      target => "@timestamp"
-      remove_field => "last_updated"
-    }
-    mutate {
-      convert => { "plugin_id" => "integer"}
-      convert => { "id" => "integer"}
-      convert => { "risk_number" => "integer"}
-      convert => { "risk_score" => "float"}
-      convert => { "total_times_detected" => "integer"}
-      convert => { "cvss_temporal" => "float"}
-      convert => { "cvss" => "float"}
-    }
-    if [risk_score] == 0 {
-      mutate {
-        add_field => { "risk_score_name" => "info" }
-      }
-    }
-    if [risk_score] > 0 and [risk_score] < 3 {
-      mutate {
-        add_field => { "risk_score_name" => "low" }
-      }
-    }
-    if [risk_score] >= 3 and [risk_score] < 6 {
-      mutate {
-        add_field => { "risk_score_name" => "medium" }
-      }
-    }
-    if [risk_score] >=6 and [risk_score] < 9 {
-      mutate {
-        add_field => { "risk_score_name" => "high" }
-      }
-    }
-    if [risk_score] >= 9 {
-      mutate {
-        add_field => { "risk_score_name" => "critical" }
-      }
-    }
-    # Add your critical assets by subnet or by hostname. Comment this field out if you don't want to tag any, but the asset panel will break.
-    if [asset] =~ "^10\.0\.100\." {
-      mutate {
-        add_tag => [ "critical_asset" ]
-      }
-    }
-  }
-}
-output {
-  if "openvas" in [tags] {
-    stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => [ "localhost:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
-    }
-  }
-}

resources/elk5-old_compatibility/logstash/4000_jira.conf

@@ -1,21 +0,0 @@
-# Description: Take in jira tickets from vulnWhisperer and pumps into logstash
-
-input {
-  file {
-    path => "/opt/VulnWhisperer/jira/*.json"
-    type => json
-    codec => json
-    start_position => "beginning"
-    tags => [ "jira" ]
-  }
-}
-
-output {
-  if "jira" in [tags] {
-    stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => [ "localhost:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
-    }
-  }
-}

resources/elk5-old_compatibility/logstash/9998_input_broker_rabbitmq.conf

@@ -1,13 +0,0 @@
-input {
-  rabbitmq {
-    key => "nessus"
-    queue => "nessus"
-    durable => true
-    exchange => "nessus"
-    user => "logstash"    
-    password => "yourpassword"
-    host => "buffer01"
-    port => 5672
-    tags => [ "queue_nessus", "rabbitmq" ]
-  }
-}

resources/elk5-old_compatibility/logstash/9998_output_broker_rabbitmq.conf

@@ -1,16 +0,0 @@
-output {
-  if "nessus" in [tags]{
-    rabbitmq {
-      key => "nessus"
-      exchange => "nessus"
-      exchange_type => "direct"
-      user => "logstash"
-      password => "yourbufferpassword"
-      host => "buffer01"
-      port => 5672
-      durable => true
-      persistent => true
-    }
-  }
-
-}

resources/elk6/logstash-vulnwhisperer-template.json

@@ -1,5 +1,6 @@
 {
   "index_patterns": "logstash-vulnwhisperer-*",
+  "version": 2019041701,
   "mappings": {
     "doc": {
       "properties": {
@@ -22,9 +23,6 @@
         "asset_uuid": {
           "type": "keyword"
         },
-        "assign_ip": {
-          "type": "ip"
-        },
         "category": {
           "type": "keyword"
         },
@@ -34,7 +32,7 @@
         "cvss_base": {
           "type": "float"
         },
-        "cvss_temporal_vector": {
+        "cvss_severity": {
           "type": "keyword"
         },
         "cvss_temporal": {
@@ -49,7 +47,7 @@
         "cvss3_base": {
           "type": "float"
         },
-        "cvss3_temporal_vector": {
+        "cvss3_severity": {
           "type": "keyword"
         },
         "cvss3_temporal": {
@@ -117,24 +115,14 @@
         "host_start": {
           "type": "date"
         },
-        "impact": {
-          "fields": {
-            "keyword": {
-              "ignore_above": 256,
-              "type": "keyword"
-            }
-          },
-          "norms": false,
-          "type": "text"
-        },
-        "ip_status": {
-          "type": "keyword"
-        },
         "ip": {
           "type": "ip"
         },
-        "last_updated": {
+        "mac_address": {
-          "type": "date"
+          "type": "keyword"
+        },
+        "netbios": {
+          "type": "keyword"
         },
         "operating_system": {
           "type": "keyword"
@@ -170,18 +158,9 @@
         "protocol": {
           "type": "keyword"
         },
-        "results": {
-          "type": "text"
-        },
         "risk_number": {
           "type": "integer"
         },
-        "risk_score_name": {
-          "type": "keyword"
-        },
-        "risk_score": {
-          "type": "float"
-        },
         "risk": {
           "type": "keyword"
         },
@@ -191,41 +170,32 @@
         "scan_name": {
           "type": "keyword"
         },
-        "scan_reference": {
+        "scan_source": {
           "type": "keyword"
         },
-        "see_also": {
+        "severity": {
           "type": "keyword"
         },
         "solution": {
           "type": "keyword"
         },
-        "source": {
+        "ssl": {
           "type": "keyword"
         },
-        "ssl": {
+        "state": {
           "type": "keyword"
         },
         "synopsis": {
           "type": "keyword"
         },
-        "system_type": {
-          "type": "keyword"
-        },
         "tags": {
           "type": "keyword"
         },
-        "threat": {
-          "type": "text"
-        },
         "type": {
           "type": "keyword"
         },
         "vendor_reference": {
           "type": "keyword"
-        },
-        "vulnerability_state": {
-          "type": "keyword"
         }
       }
     }

resources/elk6/pipeline/1000_nessus_process_file.conf

@@ -7,163 +7,47 @@
 
 input {
   file {
-    path => "/opt/VulnWhisperer/data/nessus/**/*"
+    path => "/opt/VulnWhisperer/data/nessus/**/*.json"
     mode => "read"
     start_position => "beginning"
     file_completed_action => "delete"
     tags => "nessus"
+    codec => json
   }
   file {
-    path => "/opt/VulnWhisperer/data/tenable/*.csv"
+    path => "/opt/VulnWhisperer/data/tenable/*.json"
     mode => "read"
     start_position => "beginning"
     file_completed_action => "delete"
     tags => "tenable"
+    codec => json
   }
 }
 
 filter {
   if "nessus" in [tags] or "tenable" in [tags] {
-    # Drop the header column
-    if [message] =~ "^Plugin ID" { drop {} }
 
-    csv {
-      # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"]
-      columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"]
-      separator => ","
-      source => "message"
-    }
-
-    ruby {
-        code => "if event.get('description')
-                    event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('synopsis')
-                    event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('solution')
-                    event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('see_also')
-                    event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('plugin_output')
-                    event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end"
-    }
-
-    #If using filebeats as your source, you will need to replace the "path" field to "source"
-    # Remove when scan name is included in event (current method is error prone)
-    grok {
-      match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.csv$" }
-      tag_on_failure => []
-    }
-
-    # TODO remove when @timestamp is included in event
     date {
-      match => [ "last_updated", "UNIX" ]
+      match => [ "scan_time", "UNIX" ]
       target => "@timestamp"
-      remove_field => ["last_updated"]
+      remove_field => ["scan_time"]
-    }
-
-    if [risk] == "None" {
-      mutate { add_field => { "risk_number" => 0 }}
-    }
-    if [risk] == "Low" {
-      mutate { add_field => { "risk_number" => 1 }}
-    }
-    if [risk] == "Medium" {
-      mutate { add_field => { "risk_number" => 2 }}
-    }
-    if [risk] == "High" {
-      mutate { add_field => { "risk_number" => 3 }}
-    }
-    if [risk] == "Critical" {
-      mutate { add_field => { "risk_number" => 4 }}
-    }
-
-    if ![cve] or [cve] == "nan" {
-      mutate { remove_field => [ "cve" ] }
-    }
-    if ![cvss] or [cvss] == "nan" {
-      mutate { remove_field => [ "cvss" ] }
-    }
-    if ![cvss_base] or [cvss_base] == "nan" {
-      mutate { remove_field => [ "cvss_base" ] }
-    }
-    if ![cvss_temporal] or [cvss_temporal] == "nan" {
-      mutate { remove_field => [ "cvss_temporal" ] }
-    }
-    if ![cvss_temporal_vector] or [cvss_temporal_vector] == "nan" {
-      mutate { remove_field => [ "cvss_temporal_vector" ] }
-    }
-    if ![cvss_vector] or [cvss_vector] == "nan" {
-      mutate { remove_field => [ "cvss_vector" ] }
-    }
-    if ![cvss3_base] or [cvss3_base] == "nan" {
-      mutate { remove_field => [ "cvss3_base" ] }
-    }
-    if ![cvss3_temporal] or [cvss3_temporal] == "nan" {
-      mutate { remove_field => [ "cvss3_temporal" ] }
-    }
-    if ![cvss3_temporal_vector] or [cvss3_temporal_vector] == "nan" {
-      mutate { remove_field => [ "cvss3_temporal_vector" ] }
-    }
-    if ![description] or [description] == "nan" {
-      mutate { remove_field => [ "description" ] }
-    }
-    if ![mac_address] or [mac_address] == "nan" {
-      mutate { remove_field => [ "mac_address" ] }
-    }
-    if ![netbios] or [netbios] == "nan" {
-      mutate { remove_field => [ "netbios" ] }
-    }
-    if ![operating_system] or [operating_system] == "nan" {
-      mutate { remove_field => [ "operating_system" ] }
-    }
-    if ![plugin_output] or [plugin_output] == "nan" {
-      mutate { remove_field => [ "plugin_output" ] }
-    }
-    if ![see_also] or [see_also] == "nan" {
-      mutate { remove_field => [ "see_also" ] }
-    }
-    if ![synopsis] or [synopsis] == "nan" {
-      mutate { remove_field => [ "synopsis" ] }
-    }
-    if ![system_type] or [system_type] == "nan" {
-      mutate { remove_field => [ "system_type" ] }
     }
 
     mutate {
-      remove_field => [ "message" ]
+      convert => { "cvss" => "float"}
-      add_field => { "risk_score" => "%{cvss}" }
+      convert => { "cvss_base" => "float"}
+      convert => { "cvss_temporal" => "float"}
+      convert => { "cvss3" => "float"}
+      convert => { "cvss3_base" => "float"}
+      convert => { "cvss3_temporal" => "float"}
+      convert => { "risk_number" => "integer"}
+      convert => { "total_times_detected" => "integer"}
     }
+
+    if [_unique] {
+      # Set document ID from _unique
       mutate {
-      convert => { "risk_score" => "float" }
+        rename => { "_unique" => "[@metadata][id]" }
-    }
-    if [risk_score] == 0 {
-      mutate {
-        add_field => { "risk_score_name" => "info" }
-      }
-    }
-    if [risk_score] > 0 and [risk_score] < 3 {
-      mutate {
-        add_field => { "risk_score_name" => "low" }
-      }
-    }
-    if [risk_score] >= 3 and [risk_score] < 6 {
-      mutate {
-        add_field => { "risk_score_name" => "medium" }
-      }
-    }
-    if [risk_score] >=6 and [risk_score] < 9 {
-      mutate {
-        add_field => { "risk_score_name" => "high" }
-      }
-    }
-    if [risk_score] >= 9 {
-      mutate {
-        add_field => { "risk_score_name" => "critical" }
       }
     }
   }
@@ -171,12 +55,17 @@ filter {
 
 output {
   if "nessus" in [tags] or "tenable" in [tags]{
-    stdout {
+    if [@metadata][id] {
-      codec => dots
+      elasticsearch {
+        hosts => [ "elasticsearch:9200" ]
+        index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+        document_id => "%{[@metadata][id]}"
       }
+    } else {
       elasticsearch {
         hosts => [ "elasticsearch:9200" ]
         index => "logstash-vulnwhisperer-%{+YYYY.MM}"
       }
     }
+  }
 }

resources/elk6/pipeline/2000_qualys_web_scans.conf

@@ -6,11 +6,19 @@
 
 input {
   file {
-    path => [ "/opt/VulnWhisperer/data/qualys/*.json" , "/opt/VulnWhisperer/data/qualys_web/*.json", "/opt/VulnWhisperer/data/qualys_vuln/*.json"] 
+    path => [ "/opt/VulnWhisperer/data/qualys_vuln/*.json" ]
-    type => json
     codec => json
     start_position => "beginning"
-    tags => [ "qualys" ]
+    tags => [ "qualys_vuln" ]
+    mode => "read"
+    start_position => "beginning"
+    file_completed_action => "delete"
+  }
+  file {
+    path => [ "/opt/VulnWhisperer/data/qualys_web/*.json" ]
+    codec => json
+    start_position => "beginning"
+    tags => [ "qualys_web" ]
     mode => "read"
     start_position => "beginning"
     file_completed_action => "delete"
@@ -18,60 +26,22 @@ input {
 }
 
 filter {
-  if "qualys" in [tags] {
+  if "qualys_vuln" in [tags] or "qualys_web" in  [tags] {
-    grok {
+    date {
-      match => { "path" => [ "(?<tags>qualys_vuln)_scan_%{DATA}_%{INT:last_updated}.json$", "(?<tags>qualys_web)_%{INT:app_id}_%{INT:last_updated}.json$" ] }
+      match => [ "scan_time", "UNIX" ]
-      tag_on_failure => []
+      target => "@timestamp"
+      remove_field => ["scan_time"]
     }
 
     mutate {
-      replace => [ "message", "%{message}" ]
+      convert => { "cvss" => "float"}
-      #gsub => [
+      convert => { "cvss_base" => "float"}
-      #  "message", "\|\|\|", " ",
+      convert => { "cvss_temporal" => "float"}
-      #  "message", "\t\t", " ",
+      convert => { "cvss3" => "float"}
-      #  "message", "    ", " ",
+      convert => { "cvss3_base" => "float"}
-      #  "message", "   ", " ",
+      convert => { "cvss3_temporal" => "float"}
-      #  "message", "  ", " ",
+      convert => { "risk_number" => "integer"}
-      #  "message", "nan", " ",
+      convert => { "total_times_detected" => "integer"}
-      #  "message",'\n',''
-      #]
-    }
-
-    if "qualys_web" in [tags] {
-    mutate {
-      add_field => { "asset" => "%{web_application_name}" }
-      add_field => { "risk_score" => "%{cvss}" }
-      }
-    } else if "qualys_vuln" in [tags] {
-      mutate {
-        add_field => { "asset" => "%{ip}" }
-        add_field => { "risk_score" => "%{cvss}" }
-      }
-    }
-
-    if [risk] == "1" {
-      mutate { add_field => { "risk_number" => 0 }}
-      mutate { replace => { "risk" => "info" }}
-    }
-    if [risk] == "2" {
-      mutate { add_field => { "risk_number" => 1 }}
-      mutate { replace => { "risk" => "low" }}
-    }
-    if [risk] == "3" {
-      mutate { add_field => { "risk_number" => 2 }}
-      mutate { replace => { "risk" => "medium" }}
-    }
-    if [risk] == "4" {
-      mutate { add_field => { "risk_number" => 3 }}
-      mutate { replace => { "risk" => "high" }}
-    }
-    if [risk] == "5" {
-      mutate { add_field => { "risk_number" => 4 }}
-      mutate { replace => { "risk" => "critical" }}
-    }
-
-    mutate {
-      remove_field => "message"
     }
 
     if [first_time_detected] {
@@ -98,63 +68,32 @@ filter {
         target => "last_time_tested"
       }
     }
-
+    # if [asset] =~ "\.yourdomain\.(com|net)$" {
-    # TODO remove when @timestamp is included in event
+    #   mutate {
-    date {
+    #     add_tag => [ "critical_asset" ]
-      match => [ "last_updated", "UNIX" ]
+    #   }
-      target => "@timestamp"
+    # }
-      remove_field => "last_updated"
+    if [_unique] {
-    }
+      # Set document ID from _unique
       mutate {
-      convert => { "plugin_id" => "integer"}
+        rename => { "_unique" => "[@metadata][id]" }
-      convert => { "id" => "integer"}
-      convert => { "risk_number" => "integer"}
-      convert => { "risk_score" => "float"}
-      convert => { "total_times_detected" => "integer"}
-      convert => { "cvss_temporal" => "float"}
-      convert => { "cvss" => "float"}
-    }
-    if [risk_score] == 0 {
-      mutate {
-        add_field => { "risk_score_name" => "info" }
-      }
-    }
-    if [risk_score] > 0 and [risk_score] < 3 {
-      mutate {
-        add_field => { "risk_score_name" => "low" }
-      }
-    }
-    if [risk_score] >= 3 and [risk_score] < 6 {
-      mutate {
-        add_field => { "risk_score_name" => "medium" }
-      }
-    }
-    if [risk_score] >=6 and [risk_score] < 9 {
-      mutate {
-        add_field => { "risk_score_name" => "high" }
-      }
-    }
-    if [risk_score] >= 9 {
-      mutate {
-        add_field => { "risk_score_name" => "critical" }
-      }
-    }
-
-    if [asset] =~ "\.yourdomain\.(com|net)$" {
-      mutate {
-        add_tag => [ "critical_asset" ]
       }
     }
   }
 }
 output {
-  if "qualys" in [tags] {
+  if "qualys_vuln" in [tags] or "qualys_web" in  [tags] {
-    stdout {
+    if [@metadata][id] {
-      codec => dots
+      elasticsearch {
+        hosts => [ "elasticsearch:9200" ]
+        index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+        document_id => "%{[@metadata][id]}"
       }
+    } else {
       elasticsearch {
         hosts => [ "elasticsearch:9200" ]
         index => "logstash-vulnwhisperer-%{+YYYY.MM}"
       }
     }
+  }
 }

resources/elk6/pipeline/3000_openvas.conf

@@ -7,7 +7,6 @@
 input {
   file {
     path => "/opt/VulnWhisperer/data/openvas/*.json"
-    type => json
     codec => json
     start_position => "beginning"
     tags => [ "openvas_scan", "openvas" ]
@@ -20,27 +19,23 @@ input {
 
 filter {
   if "openvas_scan" in [tags] {
-    mutate {
+    date {
-      replace => [ "message", "%{message}" ]
+      match => [ "scan_time", "UNIX" ]
-      gsub => [
+      target => "@timestamp"
-        "message", "\|\|\|", " ",
+      remove_field => ["scan_time"]
-        "message", "\t\t", " ",
-        "message", "    ", " ",
-        "message", "   ", " ",
-        "message", "  ", " ",
-        "message", "nan", " ",
-        "message",'\n',''
-      ]
     }
 
-
+    # TODO - move this mapping into the vulnwhisperer module
-    grok {
+    translate {
-        match => { "path" => "openvas_scan_%{DATA:scan_id}_%{INT:last_updated}.json$" }
+      field => "[risk_number]"
-        tag_on_failure => []
+      destination => "[risk]"
+      dictionary => {
+        "0" => "Info"
+        "1" => "Low"
+        "2" => "Medium"
+        "3" => "High"
+        "4" => "Critical"
       }
-
-    mutate {
-      add_field => { "risk_score" => "%{cvss}" }
     }
 
     if [risk] == "1" {
@@ -93,62 +88,44 @@ filter {
       }
     }
 
-    # TODO remove when @timestamp is included in event
-    date {
-      match => [ "last_updated", "UNIX" ]
-      target => "@timestamp"
-      remove_field => "last_updated"
-    }
     mutate {
-      convert => { "plugin_id" => "integer"}
-      convert => { "id" => "integer"}
-      convert => { "risk_number" => "integer"}
-      convert => { "risk_score" => "float"}
-      convert => { "total_times_detected" => "integer"}
-      convert => { "cvss_temporal" => "float"}
       convert => { "cvss" => "float"}
+      convert => { "cvss_base" => "float"}
+      convert => { "cvss_temporal" => "float"}
+      convert => { "cvss3" => "float"}
+      convert => { "cvss3_base" => "float"}
+      convert => { "cvss3_temporal" => "float"}
+      convert => { "risk_number" => "integer"}
+      convert => { "total_times_detected" => "integer"}
     }
-    if [risk_score] == 0 {
+
-      mutate {
-        add_field => { "risk_score_name" => "info" }
-      }
-    }
-    if [risk_score] > 0 and [risk_score] < 3 {
-      mutate {
-        add_field => { "risk_score_name" => "low" }
-      }
-    }
-    if [risk_score] >= 3 and [risk_score] < 6 {
-      mutate {
-        add_field => { "risk_score_name" => "medium" }
-      }
-    }
-    if [risk_score] >=6 and [risk_score] < 9 {
-      mutate {
-        add_field => { "risk_score_name" => "high" }
-      }
-    }
-    if [risk_score] >= 9 {
-      mutate {
-        add_field => { "risk_score_name" => "critical" }
-      }
-    }
     # Add your critical assets by subnet or by hostname. Comment this field out if you don't want to tag any, but the asset panel will break.
-    if [asset] =~ "^10\.0\.100\." {
+    # if [asset] =~ "^10\.0\.100\." {
+    #   mutate {
+    #     add_tag => [ "critical_asset" ]
+    #   }
+    # }
+    if [_unique] {
+      # Set document ID from _unique
       mutate {
-        add_tag => [ "critical_asset" ]
+        rename => { "_unique" => "[@metadata][id]" }
       }
     }
   }
 }
 output {
   if "openvas" in [tags] {
-    stdout {
+    if [@metadata][id] {
-      codec => dots
+      elasticsearch {
+        hosts => [ "elasticsearch:9200" ]
+        index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+        document_id => "%{[@metadata][id]}"
       }
+    } else {
       elasticsearch {
         hosts => [ "elasticsearch:9200" ]
         index => "logstash-vulnwhisperer-%{+YYYY.MM}"
       }
     }
+  }
 }

tests/test-docker.sh

@@ -29,49 +29,49 @@ done
 green "✅ Elasticsearch status is green..."
 
 count=0
-until [[ $(curl -s "$logstash_url/_node/stats" | jq '.events.out') -ge 1236 ]]; do
+until [[ $(curl -s "$logstash_url/_node/stats" | jq '.events.out') -ge 1617 ]]; do
-    yellow "Waiting for Logstash load to finish...  $(curl -s "$logstash_url/_node/stats" | jq '.events.out') of 1236 (attempt $count of 60)"
+    yellow "Waiting for Logstash load to finish...  $(curl -s "$logstash_url/_node/stats" | jq '.events.out') of 1617 (attempt $count of 60)"
     ((count++)) && ((count==60)) && break
     sleep 5
 done
 
-if [[ count -le 60 && $(curl -s "$logstash_url/_node/stats" | jq '.events.out') -ge 1236 ]]; then
+if [[ count -le 60 && $(curl -s "$logstash_url/_node/stats" | jq '.events.out') -ge 1617 ]]; then
-    green "✅ Logstash load finished..."
+    green "✅ Logstash load finished $(curl -s "$logstash_url/_node/stats" | jq '.events.out') logs processed..."
 else
-    red "❌ Logstash load didn't complete... $(curl -s "$logstash_url/_node/stats" | jq '.events.out')"
+    red "❌ Logstash load didn't complete  $(curl -s "$logstash_url/_node/stats" | jq '.events.out') logs processed... $(curl -s "$logstash_url/_node/stats" | jq '.events.out')"
 fi
 
 
 count=0
-until [[ $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq '.count') -ge 1232 ]] ; do
+until [[ $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_count" | jq '.count') -ge 1617 ]] ; do
-    yellow "Waiting for Elasticsearch index to sync... $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq '.count') of 1232 logs loaded (attempt $count of 150)"
+    yellow "Waiting for Elasticsearch index to sync... $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_count" | jq '.count') of 1617 logs loaded (attempt $count of 150)"
     ((count++)) && ((count==150)) && break
     sleep 2
 done
-if [[ count -le 50 && $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq '.count') -ge 1232 ]]; then
+if [[ count -le 50 && $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_count" | jq '.count') -ge 1617 ]]; then
-    green "✅ logstash-vulnwhisperer-2019.03 document count >= 1232"
+    green "✅ logstash-vulnwhisperer-* document count $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_count" | jq '.count') >= 1617"
 else
-    red "❌ TIMED OUT waiting for logstash-vulnwhisperer-2019.03 document count: $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq) != 1232"
+    red "❌ TIMED OUT waiting for logstash-vulnwhisperer-* document count: $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_count" | jq) != 1617"
 fi
 
-# if [[ $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq '.count') == 1232 ]]; then
+# if [[ $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_count" | jq '.count') == 1232 ]]; then
-#     green "✅ Passed: logstash-vulnwhisperer-2019.03 document count == 1232"
+#     green "✅ Passed: logstash-vulnwhisperer-* document count == 1232"
 # else
-#     red "❌ Failed: logstash-vulnwhisperer-2019.03 document count == 1232 was: $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count") instead"
+#     red "❌ Failed: logstash-vulnwhisperer-* document count == 1232 was: $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_count") instead"
 #     ((return_code = return_code + 1))
 # fi
 
 # Test Nessus plugin_name:Backported Security Patch Detection (FTP)
-nessus_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_search?q=plugin_name:%22Backported%20Security%20Patch%20Detection%20(FTP)%22%20AND%20asset:176.28.50.164%20AND%20tags:nessus" | jq '.hits.hits[]._source')
+nessus_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_search?q=plugin_name:%22Backported%20Security%20Patch%20Detection%20(FTP)%22%20AND%20asset:176.28.50.164%20AND%20tags:nessus" | jq '.hits.hits[]._source')
-if echo $nessus_doc | jq '.risk' | grep -q "None"; then
+if echo $nessus_doc | jq '.risk' | grep -q "none"; then
-    green "✅ Passed: Nessus risk == None"
+    green "✅ Passed: Nessus risk == none"
 else
-    red "❌ Failed: Nessus risk == None was: $(echo $nessus_doc | jq '.risk') instead"
+    red "❌ Failed: Nessus risk == none was: $(echo $nessus_doc | jq '.risk') instead"
     ((return_code = return_code + 1))
 fi
 
 # Test Tenable plugin_name:Backported Security Patch Detection (FTP)
-tenable_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_search?q=plugin_name:%22Backported%20Security%20Patch%20Detection%20(FTP)%22%20AND%20asset:176.28.50.164%20AND%20tags:tenable" | jq '.hits.hits[]._source')
+tenable_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_search?q=plugin_name:%22Backported%20Security%20Patch%20Detection%20(FTP)%22%20AND%20asset:176.28.50.164%20AND%20tags:tenable" | jq '.hits.hits[]._source')
 # Test asset
 if echo $tenable_doc | jq .asset | grep -q '176.28.50.164'; then
     green "✅ Passed: Tenable asset == 176.28.50.164"
@@ -89,7 +89,7 @@ else
 fi
 
 # Test Qualys plugin_name:OpenSSL Multiple Remote Security Vulnerabilities
-qualys_vuln_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_search?q=tags:qualys_vuln%20AND%20ip:%22176.28.50.164%22%20AND%20plugin_name:%22OpenSSL%20Multiple%20Remote%20Security%20Vulnerabilities%22%20AND%20port:465" | jq '.hits.hits[]._source')
+qualys_vuln_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_search?q=tags:qualys_vuln%20AND%20ip:%22176.28.50.164%22%20AND%20plugin_name:%22OpenSSL%20Multiple%20Remote%20Security%20Vulnerabilities%22%20AND%20port:465" | jq '.hits.hits[]._source')
 # Test @timestamp
 if echo $qualys_vuln_doc | jq '.["@timestamp"]' | grep -q '2019-03-30T10:17:41.000Z'; then
     green "✅ Passed: Qualys VM @timestamp == 2019-03-30T10:17:41.000Z"
@@ -99,10 +99,10 @@ else
 fi
 
 # Test @XXXX
-if echo $qualys_vuln_doc | jq '.cvss' | grep -q '6.8'; then
+if echo $qualys_vuln_doc | jq '.cvss' | grep -q '5.6'; then
-    green "✅ Passed: Qualys VM cvss == 6.8"
+    green "✅ Passed: Qualys VM cvss == 5.6"
 else
-    red "❌ Failed: Qualys VM cvss == 6.8 was: $(echo $qualys_vuln_doc | jq '.cvss') instead"
+    red "❌ Failed: Qualys VM cvss == 5.6 was: $(echo $qualys_vuln_doc | jq '.cvss') instead"
     ((return_code = return_code + 1))
 fi
 

tests/test-vuln_whisperer.sh

@@ -79,7 +79,7 @@ else
     ((return_code = return_code + 1))
 fi
 
-yellow "*********************************************"
+yellow "\n*********************************************"
 yellow "* Test only Qualys VM with one failed scan  *"
 yellow "*********************************************"
 rm -rf /opt/VulnWhisperer/*

vulnwhisp/frameworks/nessus.py

@@ -23,18 +23,40 @@ class NessusAPI(object):
     EXPORT_FILE_DOWNLOAD = EXPORT + '/{file_id}/download'
     EXPORT_STATUS = EXPORT + '/{file_id}/status'
     EXPORT_HISTORY = EXPORT + '?history_id={history_id}'
+    # All column mappings should be lowercase
+    COLUMN_MAPPING = {
+        'cvss base score': 'cvss_base',
+        'cvss temporal score': 'cvss_temporal',
+        'cvss temporal vector': 'cvss_temporal_vector',
+        'cvss3 base score': 'cvss3_base',
+        'cvss3 temporal score': 'cvss3_temporal',
+        'cvss3 temporal vector': 'cvss3_temporal_vector',
+        'fqdn': 'dns',
+        'host': 'asset',
+        'ip address': 'ip',
+        'name': 'plugin_name',
+        'os': 'operating_system',
+        'see also': 'exploitability',
+        'system type': 'category',
+        'vulnerability state': 'state'
+    }
+    SEVERITY_MAPPING = {'none': 0, 'low': 1, 'medium': 2, 'high': 3, 'critical': 4}
 
-    def __init__(self, hostname=None, port=None, username=None, password=None, verbose=True):
+    def __init__(self, hostname=None, port=None, username=None, password=None, verbose=True, profile=None, access_key=None, secret_key=None):
         self.logger = logging.getLogger('NessusAPI')
         if verbose:
             self.logger.setLevel(logging.DEBUG)
-        if username is None or password is None:
+        if not all((username, password)) and not all((access_key, secret_key)):
-            raise Exception('ERROR: Missing username or password.')
+            raise Exception('ERROR: Missing username, password or API keys.')
 
         self.user = username
         self.password = password
+        self.api_keys = False
+        self.access_key = access_key
+        self.secret_key = secret_key
         self.base = 'https://{hostname}:{port}'.format(hostname=hostname, port=port)
         self.verbose = verbose
+        self.profile = profile
 
         self.session = requests.Session()
         self.session.verify = False
@@ -52,7 +74,13 @@ class NessusAPI(object):
             'X-Cookie': None
         }
 
+        if all((self.access_key, self.secret_key)):
+            self.logger.debug('Using {} API keys'.format(self.profile))
+            self.api_keys = True
+            self.session.headers['X-ApiKeys'] = 'accessKey={}; secretKey={}'.format(self.access_key, self.secret_key)
+        else:
             self.login()
+
         self.scans = self.get_scans()
         self.scan_ids = self.get_scan_ids()
 
@@ -78,8 +106,10 @@ class NessusAPI(object):
                 if url == self.base + self.SESSION:
                     break
                 try:
-                    self.login()
                     timeout += 1
+                    if self.api_keys:
+                        continue
+                    self.login()
                     self.logger.info('Token refreshed')
                 except Exception as e:
                     self.logger.error('Could not refresh token\nReason: {}'.format(str(e)))
@@ -114,10 +144,8 @@ class NessusAPI(object):
         data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json_output=True)
         return data['history']
 
-    def download_scan(self, scan_id=None, history=None, export_format="", profile=""):
+    def download_scan(self, scan_id=None, history=None, export_format=''):
         running = True
-        counter = 0
-
         data = {'format': export_format}
         if not history:
             query = self.EXPORT.format(scan_id=scan_id)
@@ -127,23 +155,19 @@ class NessusAPI(object):
         req = self.request(query, data=json.dumps(data), method='POST', json_output=True)
         try:
             file_id = req['file']
+            if self.profile == 'nessus':
                 token_id = req['token'] if 'token' in req else req['temp_token']
         except Exception as e:
             self.logger.error('{}'.format(str(e)))
-        self.logger.info('Download for file id {}'.format(str(file_id)))
+        self.logger.info('Downloading file id {}'.format(str(file_id)))
         while running:
             time.sleep(2)
-            counter += 2
             report_status = self.request(self.EXPORT_STATUS.format(scan_id=scan_id, file_id=file_id), method='GET',
                                          json_output=True)
             running = report_status['status'] != 'ready'
-            sys.stdout.write(".")
+            sys.stdout.write('.')
             sys.stdout.flush()
-            # FIXME: why? can this be removed in favour of a counter?
+        if self.profile == 'tenable' or self.api_keys:
-            if counter % 60 == 0:
-                self.logger.info("Completed: {}".format(counter))
-        self.logger.info("Done: {}".format(counter))
-        if profile == 'tenable':
             content = self.request(self.EXPORT_FILE_DOWNLOAD.format(scan_id=scan_id, file_id=file_id), method='GET', download=True)
         else:
             content = self.request(self.EXPORT_TOKEN_DOWNLOAD.format(token_id=token_id), method='GET', download=True)
@@ -169,3 +193,47 @@ class NessusAPI(object):
                     'Pacific Standard Time': 'US/Pacific',
                     'None': 'US/Central'}
         return time_map.get(tz, None)
+
+    def normalise(self, df):
+        self.logger.debug('Normalising data')
+        df = self.map_fields(df)
+        df = self.transform_values(df)
+        return df
+
+    def map_fields(self, df):
+        self.logger.debug('Mapping fields')
+        # Any specific mappings here
+        if self.profile == 'tenable':
+            # Prefer CVSS Base Score over CVSS for tenable
+            self.logger.debug('Dropping redundant tenable fields')
+            df.drop('CVSS', axis=1, inplace=True, errors='ignore')
+
+        if self.profile == 'nessus':
+            # Set IP from Host field
+            df['ip'] = df['Host']
+
+        # Lowercase and map fields from COLUMN_MAPPING
+        df.columns = [x.lower() for x in df.columns]
+        df.rename(columns=self.COLUMN_MAPPING, inplace=True)
+        df.columns = [x.replace(' ', '_') for x in df.columns]
+
+        return df
+
+    def transform_values(self, df):
+        self.logger.debug('Transforming values')
+
+        df.fillna('', inplace=True)
+
+        # upper/lowercase fields
+        self.logger.debug('Changing case of fields')
+        df['cve'] = df['cve'].str.upper()
+        df['protocol'] = df['protocol'].str.lower()
+        df['risk'] = df['risk'].str.lower()
+
+        # Map risk to a SEVERITY MAPPING value
+        self.logger.debug('Mapping risk to severity number')
+        df['risk_number'] = df['risk'].map(self.SEVERITY_MAPPING)
+
+        df.fillna('', inplace=True)
+
+        return df

vulnwhisp/frameworks/openvas.py

@@ -110,6 +110,7 @@ class OpenVAS_API(object):
         ]
         token = requests.post(self.base + self.OMP, data=data, verify=False)
         return token
+
     def get_report_formats(self):
         params = (
             ('cmd', 'get_report_formats'),
@@ -190,3 +191,18 @@ class OpenVAS_API(object):
         self.processed_reports += 1
         merged_df = pd.merge(report_df, self.openvas_reports, on='report_ids').reset_index().drop('index', axis=1)
         return merged_df
+
+    def normalise(self, df):
+        self.logger.debug('Normalising data')
+        df = self.map_fields(df)
+        df = self.transform_values(df)
+        return df
+
+    def map_fields(self, df):
+        self.logger.debug('Mapping fields')
+        return df
+
+    def transform_values(self, df):
+        self.logger.debug('Transforming values')
+        df.fillna('', inplace=True)
+        return df

vulnwhisp/frameworks/qualys_vuln.py

@@ -28,7 +28,7 @@ class qualysWhisperAPI(object):
 
     def scan_xml_parser(self, xml):
         all_records = []
-        root = ET.XML(xml.encode("utf-8"))
+        root = ET.XML(xml.encode('utf-8'))
         for child in root.find('.//SCAN_LIST'):
             all_records.append({
                 'name': child.find('TITLE').text,
@@ -78,6 +78,19 @@ class qualysUtils:
 
 class qualysVulnScan:
 
+    COLUMN_MAPPING = {
+        'cve_id': 'cve',
+        'impact': 'synopsis',
+        'ip_status': 'state',
+        'os': 'operating_system',
+        'qid': 'plugin_id',
+        'results': 'plugin_output',
+        'threat': 'description',
+        'title': 'plugin_name'
+    }
+
+    SEVERITY_MAPPING = {0: 'none', 1: 'low', 2: 'medium', 3: 'high',4: 'critical'}
+
     def __init__(
         self,
         config=None,
@@ -122,3 +135,47 @@ class qualysVulnScan:
             return scan_report
 
         return scan_report
+
+    def normalise(self, df):
+        self.logger.debug('Normalising data')
+        df = self.map_fields(df)
+        df = self.transform_values(df)
+        return df
+
+    def map_fields(self, df):
+        self.logger.info('Mapping fields')
+
+        # Lowercase and map fields from COLUMN_MAPPING
+        df.columns = [x.lower() for x in df.columns]
+        df.rename(columns=self.COLUMN_MAPPING, inplace=True)
+        df.columns = [x.replace(' ', '_') for x in df.columns]
+
+        return df
+
+    def transform_values(self, df):
+        self.logger.info('Transforming values')
+
+        df.fillna('', inplace=True)
+
+        # upper/lowercase fields
+        self.logger.info('Changing case of fields')
+        df['cve'] = df['cve'].str.upper()
+        df['protocol'] = df['protocol'].str.lower()
+
+        # Contruct the CVSS vector
+        self.logger.info('Extracting CVSS components')
+        df['cvss_vector'] = df['cvss_base'].str.extract('\((.*)\)', expand=False)
+        df['cvss_base'] = df['cvss_base'].str.extract('^(\d+(?:\.\d+)?)', expand=False)
+        df['cvss_temporal_vector'] = df['cvss_temporal'].str.extract('\((.*)\)', expand=False)
+        df['cvss_temporal'] = df['cvss_temporal'].str.extract('^(\d+(?:\.\d+)?)', expand=False)
+
+        # Set asset to ip
+        df['asset'] = df['ip']
+
+        # Convert Qualys severity to standardised risk number
+        df['risk_number'] =  df['severity'].astype(int)-1
+        df['risk'] = df['risk_number'].map(self.SEVERITY_MAPPING)
+
+        df.fillna('', inplace=True)
+
+        return df

vulnwhisp/frameworks/qualys_web.py

@@ -2,18 +2,19 @@
 # -*- coding: utf-8 -*-
 __author__ = 'Austin Taylor'
 
-from lxml import objectify
+import csv
-from lxml.builder import E
+import logging
+import os
+import sys
 import xml.etree.ElementTree as ET
+
+import dateutil.parser as dp
 import pandas as pd
 import qualysapi
 import qualysapi.config as qcconf
 import requests
-import sys
+from lxml import objectify
-import os
+from lxml.builder import E
-import csv
-import logging
-import dateutil.parser as dp
 
 
 class qualysWhisperAPI(object):
@@ -109,7 +110,6 @@ class qualysWhisperAPI(object):
         _records = []
         try:
             total = int(self.get_was_scan_count(status=status))
-            self.logger.error('Already have WAS scan count')
             self.logger.info('Retrieving information for {} scans'.format(total))
             for i in range(0, total):
                 if i % limit == 0:
@@ -282,13 +282,26 @@ class qualysUtils:
     def iso_to_epoch(self, dt):
         return dp.parse(dt).strftime('%s')
 
-    def cleanser(self, _data):
-        repls = (('\n', '|||'), ('\r', '|||'), (',', ';'), ('\t', '|||'))
-        if _data:
-            _data = reduce(lambda a, kv: a.replace(*kv), repls, str(_data))
-        return _data
-
 class qualysScanReport:
+
+    COLUMN_MAPPING = {
+        'DescriptionCatSev': 'category_description',
+        'DescriptionSeverity': 'synopsis',
+        'Evidence #1': 'evidence',
+        'Payload #1': 'payload',
+        'QID': 'plugin_id',
+        'Request Headers #1': 'request_headers',
+        'Request Method #1': 'request_method',
+        'Request URL #1': 'request_url',
+        'Response #1': 'plugin_output',
+        'Title': 'plugin_name',
+        'Url': 'uri',
+        'URL': 'url',
+        'Vulnerability Category': 'type',
+    }
+
+    SEVERITY_MAPPING = {0: 'none', 1: 'low', 2: 'medium', 3: 'high', 4: 'critical'}
+
     # URL Vulnerability Information
     WEB_SCAN_VULN_BLOCK = list(qualysReportFields.VULN_BLOCK)
     WEB_SCAN_VULN_BLOCK.insert(WEB_SCAN_VULN_BLOCK.index('QID'), 'Detection ID')
@@ -297,6 +310,16 @@ class qualysScanReport:
     WEB_SCAN_VULN_HEADER[WEB_SCAN_VULN_BLOCK.index(qualysReportFields.CATEGORIES[0])] = \
         'Vulnerability Category'
 
+    # Add an alternative vulnerability header
+    WEB_SCAN_VULN_BLOCK_ALT = WEB_SCAN_VULN_BLOCK[:]
+    WEB_SCAN_VULN_BLOCK_ALT.insert(WEB_SCAN_VULN_BLOCK_ALT.index('First Time Detected'), 'Detection Date')
+    remove_fields = ['Last Time Tested', 'Times Detected', 'First Time Detected', 'Last Time Detected']
+    WEB_SCAN_VULN_BLOCK_ALT = [x for x in WEB_SCAN_VULN_BLOCK_ALT if x not in remove_fields]
+
+    WEB_SCAN_VULN_HEADER_ALT = WEB_SCAN_VULN_BLOCK_ALT[:]
+    WEB_SCAN_VULN_HEADER_ALT[WEB_SCAN_VULN_BLOCK_ALT.index(qualysReportFields.CATEGORIES[0])] = \
+        'Vulnerability Category'
+
     WEB_SCAN_SENSITIVE_HEADER = list(WEB_SCAN_VULN_HEADER)
     WEB_SCAN_SENSITIVE_HEADER.insert(WEB_SCAN_SENSITIVE_HEADER.index('Url'
                                                                      ), 'Content')
@@ -358,6 +381,17 @@ class qualysScanReport:
                                                                                            self.WEB_SCAN_INFO_BLOCK],
                                                                                        pop_last=True),
                                                                columns=self.WEB_SCAN_VULN_HEADER)
+            if len(dict_tracker['WEB_SCAN_VULN_BLOCK']) == 0:
+                # Try alternative headers
+                dict_tracker["WEB_SCAN_VULN_BLOCK"] = pd.DataFrame(
+                    self.utils.grab_section(
+                        report,
+                        self.WEB_SCAN_VULN_BLOCK_ALT,
+                        end=[self.WEB_SCAN_SENSITIVE_BLOCK, self.WEB_SCAN_INFO_BLOCK],
+                        pop_last=True,
+                    ),
+                    columns=self.WEB_SCAN_VULN_HEADER_ALT,
+                )
             dict_tracker['WEB_SCAN_SENSITIVE_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
                                                                                             self.WEB_SCAN_SENSITIVE_BLOCK,
                                                                                             end=[
@@ -423,9 +457,6 @@ class qualysScanReport:
                               'Request Headers #1', 'Response #1', 'Evidence #1',
                               'Description', 'Impact', 'Solution', 'Url', 'Content']
 
-        for col in columns_to_cleanse:
-            merged_df[col] = merged_df[col].apply(self.utils.cleanser)
-
         merged_df = merged_df.drop(['QID_y', 'QID_x'], axis=1)
         merged_df = merged_df.rename(columns={'Id': 'QID'})
 
@@ -444,7 +475,7 @@ class qualysScanReport:
         return merged_df
 
     def download_file(self, path='', file_id=None):
-        report = self.qw.download_report(file_id)
+        report = self.qw.download_report(file_id).encode('utf-8')
         filename = path + str(file_id) + '.csv'
         file_out = open(filename, 'w')
         for line in report.splitlines():
@@ -463,3 +494,41 @@ class qualysScanReport:
         merged_data.sort_index(axis=1, inplace=True)
 
         return merged_data
+
+    def normalise(self, df):
+        self.logger.debug('Normalising data')
+        df = self.map_fields(df)
+        df = self.transform_values(df)
+        return df
+
+    def map_fields(self, df):
+        self.logger.debug('Mapping fields')
+
+        df.rename(columns=self.COLUMN_MAPPING, inplace=True)
+
+        # Lowercase and map fields from COLUMN_MAPPING
+        df.columns = [x.lower() for x in df.columns]
+        df.columns = [x.replace(' ', '_') for x in df.columns]
+
+        return df
+
+    def transform_values(self, df):
+        self.logger.debug('Transforming values')
+        df.fillna('', inplace=True)
+
+        self.logger.info('Changing case of fields')
+        df['cwe'] = df['cwe'].str.upper()
+
+        # Convert Qualys severity to standardised risk number
+        df['risk_number'] =  df['severity'].astype(int)-1
+        df['risk'] = df['risk_number'].map(self.SEVERITY_MAPPING)
+
+        # Extract dns field from URL
+        df['dns'] = df['url'].str.extract('https?://([^/]+)', expand=False)
+        df.loc[df['uri'] != '','dns'] = df.loc[df['uri'] != '','uri'].str.extract('https?://([^/]+)', expand=False)
+
+        # Set asset to web_application_name
+        df['asset'] = df['web_application_name']
+
+        df.fillna('', inplace=True)
+        return df

vulnwhisp/test/mock.py

@@ -1,5 +1,6 @@
-import os
 import logging
+import os
+
 import httpretty
 
 
@@ -16,32 +17,15 @@ class mockAPI(object):
             self.logger.setLevel(logging.DEBUG)
 
         self.logger.info('mockAPI initialised, API requests will be mocked')
-        self.logger.debug('Test path resolved as {}'.format(self.mock_dir))
+        self.logger.info('Test path resolved as {}'.format(self.mock_dir))
 
     def get_directories(self, path):
         dir, subdirs, files = next(os.walk(path))
-        return subdirs
+        return sorted(subdirs)
 
     def get_files(self, path):
         dir, subdirs, files = next(os.walk(path))
-        return files
+        return sorted(files)
-
-    def qualys_vuln_callback(self, request, uri, response_headers):
-        self.logger.debug('Simulating response for {} ({})'.format(uri, request.body))
-        if 'list' in request.parsed_body['action']:
-            return [200,
-                    response_headers,
-                    open('{}/{}'.format(self.qualys_vuln_path, 'scans')).read()]
-        elif 'fetch' in request.parsed_body['action']:
-            try:
-                response_body = open('{}/{}'.format(
-                                      self.qualys_vuln_path, 
-                                      request.parsed_body['scan_ref'][0].replace('/', '_'))
-                                    ).read()
-            except:
-                # Can't find the file, just send an empty response
-                response_body = ''
-        return [200, response_headers, response_body] 
 
     def create_nessus_resource(self, framework):
         for filename in self.get_files('{}/{}'.format(self.mock_dir, framework)):
@@ -53,19 +37,81 @@ class mockAPI(object):
                 body=open('{}/{}/{}'.format(self.mock_dir, framework, filename)).read()
             )
 
+    def qualys_vuln_callback(self, request, uri, response_headers):
+        self.logger.debug('Simulating response for {} ({})'.format(uri, request.body))
+        if 'list' in request.parsed_body['action']:
+            return [200,
+                    response_headers,
+                    open(self.qualys_vuln_path + '/scans').read()]
+        elif 'fetch' in request.parsed_body['action']:
+            try:
+                response_body = open('{}/{}'.format(
+                                      self.qualys_vuln_path,
+                                      request.parsed_body['scan_ref'][0].replace('/', '_'))
+                                    ).read()
+            except:
+                # Can't find the file, just send an empty response
+                response_body = ''
+        return [200, response_headers, response_body]
+
     def create_qualys_vuln_resource(self, framework):
         # Create health check endpoint
-        self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, 'GET', 'msp/about.php'))
+        self.logger.debug('Adding mocked {} endpoint GET msp/about.php'.format(framework))
         httpretty.register_uri(
                 httpretty.GET,
-                'https://{}:443/{}'.format(framework, 'msp/about.php'),
+                'https://{}:443/msp/about.php'.format(framework),
                 body='')
 
         self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, 'POST', 'api/2.0/fo/scan'))
         httpretty.register_uri(
-            httpretty.POST, 'https://{}:443/{}'.format(framework, 'api/2.0/fo/scan/'),
+            httpretty.POST, 'https://{}:443/api/2.0/fo/scan/'.format(framework),
             body=self.qualys_vuln_callback)
 
+    def qualys_web_callback(self, request, uri, response_headers):
+        self.logger.debug('Simulating response for {} ({})'.format(uri, request.body))
+        report_id = request.parsed_body.split('<WasScan><id>')[1].split('<')[0]
+        response_body = open('{}/create_{}'.format(self.qualys_web_path, report_id)).read()
+        return [200, response_headers, response_body]
+
+    def create_qualys_web_resource(self, framework):
+        for filename in self.get_files('{}/{}'.format(self.mock_dir, framework)):
+            if filename.startswith('POST') or filename.startswith('GET'):
+                method, resource = filename.split('_', 1)
+                resource = resource.replace('_', '/')
+                self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, method, resource))
+                httpretty.register_uri(
+                    getattr(httpretty, method), 'https://{}:443/{}'.format(framework, resource),
+                    body=open('{}/{}/{}'.format(self.mock_dir, framework, filename)).read()
+                )
+
+        self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, 'POST', 'qps/rest/3.0/create/was/report'))
+        httpretty.register_uri(
+            httpretty.POST, 'https://{}:443/qps/rest/3.0/create/was/report'.format(framework),
+            body=self.qualys_web_callback)
+
+    def openvas_callback(self, request, uri, response_headers):
+        self.logger.debug('Simulating response for {} ({})'.format(uri, request.body))
+        if request.querystring['cmd'][0] in ['get_reports', 'get_report_formats']:
+            response_body = open('{}/{}'.format(self.openvas_path, request.querystring['cmd'][0])).read()
+
+        if request.querystring['cmd'][0] == 'get_report':
+                response_body = open('{}/report_{}'.format(self.openvas_path, request.querystring['report_id'][0])).read()
+
+        return [200, response_headers, response_body]
+
+    def create_openvas_resource(self, framework):
+        # Create login endpoint
+        httpretty.register_uri(
+            httpretty.POST, 'https://{}:4000/omp'.format(framework),
+            body=open('{}/{}/{}'.format(self.mock_dir, framework, 'login')).read()
+        )
+
+        # Create GET requests endpoint
+        httpretty.register_uri(
+            httpretty.GET, 'https://{}:4000/omp'.format(framework),
+            body=self.openvas_callback
+        )
+
     def mock_endpoints(self):
         for framework in self.get_directories(self.mock_dir):
             if framework in ['nessus', 'tenable']:
@@ -73,4 +119,10 @@ class mockAPI(object):
             elif framework == 'qualys_vuln':
                 self.qualys_vuln_path = self.mock_dir + '/' + framework
                 self.create_qualys_vuln_resource(framework)
+            elif framework == 'qualys_web':
+                self.qualys_web_path = self.mock_dir + '/' + framework
+                self.create_qualys_web_resource(framework)
+            elif framework == 'openvas':
+                self.openvas_path = self.mock_dir + '/' + framework
+                self.create_openvas_resource(framework)
         httpretty.enable()

vulnwhisp/vulnwhisp.py

@@ -2,22 +2,25 @@
 # -*- coding: utf-8 -*-
 __author__ = 'Austin Taylor'
 
-from base.config import vwConfig
-from frameworks.nessus import NessusAPI
-from frameworks.qualys_web import qualysScanReport
-from frameworks.qualys_vuln import qualysVulnScan
-from frameworks.openvas import OpenVAS_API
-from reporting.jira_api import JiraAPI
-import pandas as pd
-from lxml import objectify
-import sys
-import os
 import io
-import time
-import sqlite3
 import json
 import logging
+import os
 import socket
+import sqlite3
+import sys
+import time
+
+import numpy as np
+import pandas as pd
+from lxml import objectify
+
+from base.config import vwConfig
+from frameworks.nessus import NessusAPI
+from frameworks.openvas import OpenVAS_API
+from frameworks.qualys_vuln import qualysVulnScan
+from frameworks.qualys_web import qualysScanReport
+from reporting.jira_api import JiraAPI
 
 
 class vulnWhispererBase(object):
@@ -55,8 +58,12 @@ class vulnWhispererBase(object):
             except:
                 self.enabled = False
             self.hostname = self.config.get(self.CONFIG_SECTION, 'hostname')
+            try:
                 self.username = self.config.get(self.CONFIG_SECTION, 'username')
                 self.password = self.config.get(self.CONFIG_SECTION, 'password')
+            except:
+                self.username = None
+                self.password = None
             self.write_path = self.config.get(self.CONFIG_SECTION, 'write_path')
             self.db_path = self.config.get(self.CONFIG_SECTION, 'db_path')
             self.verbose = self.config.getbool(self.CONFIG_SECTION, 'verbose')
@@ -84,7 +91,7 @@ class vulnWhispererBase(object):
                 self.cur = self.conn.cursor()
                 self.logger.info('Connected to database at {loc}'.format(loc=self.database))
             except Exception as e:
-                self.logger.error('Could not connect to database at {loc}\nReason: {e} - Please ensure the path exist'.format(
+                self.logger.error('Could not connect to database at {loc}\nReason: {e} - Please ensure the path exists'.format(
                         e=e,
                         loc=self.database))
         else:
@@ -129,11 +136,6 @@ class vulnWhispererBase(object):
             self.delete_table()
         self.create_table()
 
-    def cleanser(self, _data):
-        repls = (('\n', r'\n'), ('\r', r'\r'))
-        data = reduce(lambda a, kv: a.replace(*kv), repls, _data)
-        return data
-
     def path_check(self, _data):
         if self.write_path:
             if '/' or '\\' in _data[-1]:
@@ -194,7 +196,7 @@ class vulnWhispererBase(object):
                 scan=self.write_path.encode('utf8')))
         else:
             os.path.exists(self.write_path)
-            self.logger.info('Directory already exist for {scan} - Skipping creation'.format(
+            self.logger.info('Directory already exists for {scan} - Skipping creation'.format(
                 scan=self.write_path.encode('utf8')))
 
     def get_latest_results(self, source, scan_name):
@@ -248,6 +250,56 @@ class vulnWhispererBase(object):
 
         return results
 
+    def common_normalise(self, df):
+        """Map and transform common data values"""
+        self.logger.info('Start common normalisation')
+
+        self.logger.info('Normalising CVSS')
+        for cvss_version in ['cvss', 'cvss3']:
+            if cvss_version + '_base' in df:
+                self.logger.info('Normalising {} base'.format(cvss_version))
+                # CVSS = cvss_temporal or cvss_base
+                df[cvss_version] = df[cvss_version + '_base']
+                df.loc[df[cvss_version + '_temporal'] != '', cvss_version] = df[cvss_version + '_temporal']
+
+            # Combine CVSS and CVSS3 vectors
+            if cvss_version + '_vector' in df and cvss_version + '_temporal_vector' in df:
+                self.logger.info('Normalising {} vector'.format(cvss_version))
+                df[cvss_version + '_vector'] = (
+                    df[[cvss_version + '_vector', cvss_version + '_temporal_vector']]
+                    .apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1)
+                    .str.rstrip('/nan')
+                )
+                df.drop(cvss_version + '_temporal_vector', axis=1, inplace=True)
+
+            if cvss_version in df:
+                self.logger.info('Normalising {} severity'.format(cvss_version))
+                # Map CVSS to severity name
+                df.loc[df[cvss_version].astype(str) == '', cvss_version] = None
+                df[cvss_version] = df[cvss_version].astype('float')
+                # df.loc[df[cvss_version].isnull(), cvss_version + '_severity'] = 'info'
+                df.loc[df[cvss_version] == 0, cvss_version + '_severity'] = 'info'
+                df.loc[(df[cvss_version] > 0) & (df[cvss_version] < 3), cvss_version + '_severity'] = 'low'
+                df.loc[(df[cvss_version] >= 3) & (df[cvss_version] < 6), cvss_version + '_severity'] = 'medium'
+                df.loc[(df[cvss_version] >= 6) & (df[cvss_version] < 9), cvss_version + '_severity'] = 'high'
+                df.loc[(df[cvss_version] > 9) & (df[cvss_version].notnull()), cvss_version + '_severity'] = 'critical'
+
+        self.logger.info('Creating Unique Document ID')
+        df['_unique'] = df.index.values
+        if 'history_id' in df:
+            df['_unique'] = df[['scan_id', 'history_id', '_unique']].apply(lambda x: '_'.join(x.astype(str)), axis=1)
+        else:
+            df['_unique'] = df[['scan_id', '_unique']].apply(lambda x: '_'.join(x.astype(str)), axis=1)
+
+        # Rename cvss to cvss2
+        # Make cvss with no suffix == cvss3 else cvss2
+        # cvss = cvss3 if cvss3 else cvss2
+        # cvss_severity = cvss3_severity if cvss3_severity else cvss2_severity
+        df.replace({'': np.nan}, inplace=True)
+
+        return df
+
+
 class vulnWhispererNessus(vulnWhispererBase):
 
     CONFIG_SECTION = None
@@ -274,6 +326,8 @@ class vulnWhispererNessus(vulnWhispererBase):
 
         self.develop = True
         self.purge = purge
+        self.access_key = None
+        self.secret_key = None
 
         if config is not None:
             try:
@@ -283,19 +337,30 @@ class vulnWhispererNessus(vulnWhispererBase):
                                                         'trash')
 
                 try:
-                    self.logger.info('Attempting to connect to nessus...')
+                    self.access_key = self.config.get(self.CONFIG_SECTION,'access_key')
+                    self.secret_key = self.config.get(self.CONFIG_SECTION,'secret_key')
+                except:
+                    pass
+
+                try:
+                    self.logger.info('Attempting to connect to {}...'.format(self.CONFIG_SECTION))
                     self.nessus = \
                         NessusAPI(hostname=self.hostname,
                                   port=self.nessus_port,
                                   username=self.username,
-                                  password=self.password)
+                                  password=self.password,
+                                  profile=self.CONFIG_SECTION,
+                                  access_key=self.access_key,
+                                  secret_key=self.secret_key
+                                  )
                     self.nessus_connect = True
-                    self.logger.info('Connected to nessus on {host}:{port}'.format(host=self.hostname,
+                    self.logger.info('Connected to {} on {host}:{port}'.format(self.CONFIG_SECTION, host=self.hostname,
                                                                                    port=str(self.nessus_port)))
                 except Exception as e:
                     self.logger.error('Exception: {}'.format(str(e)))
                     raise Exception(
-                        'Could not connect to nessus -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format(
+                        'Could not connect to {} -- Please verify your settings in {config} are correct and try again.\nReason: {e}'.format(
+                            self.CONFIG_SECTION,
                             config=self.config.config_in,
                             e=e))
             except Exception as e:
@@ -329,13 +394,13 @@ class vulnWhispererNessus(vulnWhispererBase):
                         record['uuid'] = h.get('uuid', '')
                         record['status'] = h.get('status', '')
                         record['history_id'] = h.get('history_id', '')
-                        record['last_modification_date'] = \
+                        record["last_modification_date"] = h.get(
-                            h.get('last_modification_date', '')
+                            "last_modification_date", ""
-                        record['norm_time'] = \
+                        )
-                            self.nessus.get_utc_from_local(int(record['last_modification_date'
+                        record["norm_time"] = self.nessus.get_utc_from_local(
-                                                               ]),
+                            int(record["last_modification_date"]),
-                                                           local_tz=self.nessus.tz_conv(record['timezone'
+                            local_tz=self.nessus.tz_conv(record["timezone"]),
-                                                                                        ]))
+                        )
                         scan_records.append(record.copy())
                 except Exception as e:
                     # Generates error each time nonetype is encountered.
@@ -347,24 +412,32 @@ class vulnWhispererNessus(vulnWhispererBase):
 
 
     def whisper_nessus(self):
-        if self.nessus_connect:
+        if not self.nessus_connect:
+            self.logger.error('Failed to use scanner at {host}:{port}'.format(host=self.hostname, port=self.nessus_port))
+            self.exit_code += 1
+            return self.exit_code
+
         scan_data = self.nessus.scans
         folders = scan_data['folders']
         scans = scan_data['scans'] if scan_data['scans'] else []
         all_scans = self.scan_count(scans)
         if self.uuids:
-                scan_list = [scan for scan in all_scans if scan['uuid']
+            scan_list = [
-                             not in self.uuids and scan['status'] in ['completed', 'imported']]
+                scan for scan in all_scans
+                if scan["uuid"] not in self.uuids
+                and scan["status"] in ["completed", "imported"]
+            ]
         else:
             scan_list = all_scans
-            self.logger.info('Identified {new} scans to be processed'.format(new=len(scan_list)))
+        self.logger.info(
+            "Identified {new} scans to be processed".format(new=len(scan_list))
+        )
 
         if not scan_list:
-                self.logger.warn('No new scans to process. Exiting...')
+            self.logger.warn("No new scans to process. Exiting...")
             return self.exit_code
 
         # Create scan subfolders
-
         for f in folders:
             if not os.path.exists(self.path_check(f['name'])):
                 if f['name'] == 'Trash' and self.nessus_trash:
@@ -373,11 +446,9 @@ class vulnWhispererNessus(vulnWhispererBase):
                     os.makedirs(self.path_check(f['name']))
             else:
                 os.path.exists(self.path_check(f['name']))
-                    self.logger.info('Directory already exist for {scan} - Skipping creation'.format(
+                self.logger.info('Directory already exists for {scan} - Skipping creation'.format(
                     scan=self.path_check(f['name']).encode('utf8')))
 
-            # try download and save scans into each folder the belong to
-
         scan_count = 0
 
         # TODO Rewrite this part to go through the scans that have aleady been processed
@@ -409,14 +480,14 @@ class vulnWhispererNessus(vulnWhispererBase):
                 folder_name = next(f['name'] for f in folders if f['id'] == folder_id)
             if status in ['completed', 'imported']:
                 file_name = '%s_%s_%s_%s.%s' % (scan_name, scan_id,
-                                                    history_id, norm_time, 'csv')
+                                                history_id, norm_time, 'json')
                 repls = (('\\', '_'), ('/', '_'), (' ', '_'))
                 file_name = reduce(lambda a, kv: a.replace(*kv), repls, file_name)
                 relative_path_name = self.path_check(folder_name + '/' + file_name).encode('utf8')
 
                 if os.path.isfile(relative_path_name):
                     if self.develop:
-                            csv_in = pd.read_csv(relative_path_name)
+                        csv_in = pd.read_json(relative_path_name, lines=True)
                         record_meta = (
                             scan_name,
                             scan_id,
@@ -430,116 +501,61 @@ class vulnWhispererNessus(vulnWhispererBase):
                             0,
                         )
                         self.record_insert(record_meta)
-                            self.logger.info('File {filename} already exist! Updating database'.format(filename=relative_path_name))
+                        self.logger.info('File {filename} already exists! Updating database'.format(filename=relative_path_name))
                 else:
                     try:
                         file_req = \
                             self.nessus.download_scan(scan_id=scan_id, history=history_id,
-                                                        export_format='csv', profile=self.CONFIG_SECTION)
+                                                    export_format='csv')
                     except Exception as e:
                         self.logger.error('Could not download {} scan {}: {}'.format(self.CONFIG_SECTION, scan_id, str(e)))
                         self.exit_code += 1
                         continue
 
-                        clean_csv = \
-                            pd.read_csv(io.StringIO(file_req.decode('utf-8')))
-                        if len(clean_csv) > 2:
                     self.logger.info('Processing {}/{} for scan: {}'.format(scan_count, len(scan_list), scan_name.encode('utf8')))
-                            columns_to_cleanse = ['CVSS','CVE','Description','Synopsis','Solution','See Also','Plugin Output']
+                    vuln_ready = pd.read_csv(io.StringIO(file_req.decode('utf-8')))
 
-                            for col in columns_to_cleanse:
+                    # Map and transform fields
-                                clean_csv[col] = clean_csv[col].astype(str).apply(self.cleanser)
+                    vuln_ready = self.nessus.normalise(vuln_ready)
+
+                    # Set common fields
+                    vuln_ready['history_id'] = history_id
+                    vuln_ready['scan_id'] = uuid
+                    vuln_ready['scan_name'] = scan_name.encode('utf8')
+                    vuln_ready['scan_source'] = self.CONFIG_SECTION
+                    vuln_ready['scan_time'] = norm_time
+
+                    vuln_ready = self.common_normalise(vuln_ready)
+
+                    vuln_ready.to_json(relative_path_name + '.tmp', orient='records', lines=True)
+                    os.rename(relative_path_name + '.tmp', relative_path_name)
+                    self.logger.info('{records} records written to {path} '.format(records=vuln_ready.shape[0],
+                                                                                   path=relative_path_name))
 
-                            clean_csv.to_csv(relative_path_name, index=False)
                     record_meta = (
                         scan_name,
                         scan_id,
                         norm_time,
                         file_name,
                         time.time(),
-                                clean_csv.shape[0],
+                        vuln_ready.shape[0],
                         self.CONFIG_SECTION,
                         uuid,
                         1,
                         0,
                     )
                     self.record_insert(record_meta)
-                            self.logger.info('{filename} records written to {path} '.format(filename=clean_csv.shape[0],
+                    self.logger.info('Scan {} ({}) written to database'.format(scan_name.encode('utf8'), uuid))
-                                                                                            path=file_name.encode('utf8')))
+
-                        else:
-                            record_meta = (
-                                scan_name,
-                                scan_id,
-                                norm_time,
-                                file_name,
-                                time.time(),
-                                clean_csv.shape[0],
-                                self.CONFIG_SECTION,
-                                uuid,
-                                1,
-                                0,
-                            )
-                            self.record_insert(record_meta)
-                            self.logger.warn('{} has no host available... Updating database and skipping!'.format(file_name))
         self.conn.close()
         self.logger.info('Scan aggregation complete! Connection to database closed.')
-        else:
+
-            self.logger.error('Failed to use scanner at {host}:{port}'.format(host=self.hostname, port=self.nessus_port))
-            self.exit_code += 1
         return self.exit_code
 
 
 class vulnWhispererQualys(vulnWhispererBase):
 
     CONFIG_SECTION = 'qualys_web'
-    COLUMN_MAPPING = {'Access Path': 'access_path',
-                     'Ajax Request': 'ajax_request',
-                     'Ajax Request ID': 'ajax_request_id',
-                     'Authentication': 'authentication',
-                     'CVSS Base': 'cvss',
-                     'CVSS Temporal': 'cvss_temporal',
-                     'CWE': 'cwe',
-                     'Category': 'category',
-                     'Content': 'content',
-                     'DescriptionSeverity': 'severity_description',
-                     'DescriptionCatSev': 'category_description',
-                     'Detection ID': 'detection_id',
-                     'Evidence #1': 'evidence_1',
-                     'First Time Detected': 'first_time_detected',
-                     'Form Entry Point': 'form_entry_point',
-                     'Function': 'function',
-                     'Groups': 'groups',
-                     'ID': 'id',
-                     'Ignore Comments': 'ignore_comments',
-                     'Ignore Date': 'ignore_date',
-                     'Ignore Reason': 'ignore_reason',
-                     'Ignore User': 'ignore_user',
-                     'Ignored': 'ignored',
-                     'Impact': 'impact',
-                     'Last Time Detected': 'last_time_detected',
-                     'Last Time Tested': 'last_time_tested',
-                     'Level': 'level',
-                     'OWASP': 'owasp',
-                     'Operating System': 'operating_system',
-                     'Owner': 'owner',
-                     'Param': 'param',
-                     'Payload #1': 'payload_1',
-                     'QID': 'plugin_id',
-                     'Request Headers #1': 'request_headers_1',
-                     'Request Method #1': 'request_method_1',
-                     'Request URL #1': 'request_url_1',
-                     'Response #1': 'response_1',
-                     'Scope': 'scope',
-                     'Severity': 'risk',
-                     'Severity Level': 'security_level',
-                     'Solution': 'solution',
-                     'Times Detected': 'times_detected',
-                     'Title': 'plugin_name',
-                     'URL': 'url',
-                     'Url': 'uri',
-                     'Vulnerability Category': 'vulnerability_category',
-                     'WASC': 'wasc',
-                     'Web Application Name': 'web_application_name'}
     def __init__(
             self,
             config=None,
@@ -599,7 +615,7 @@ class vulnWhispererQualys(vulnWhispererBase):
                     0,
                 )
                 self.record_insert(record_meta)
-                self.logger.info('File {filename} already exist! Updating database'.format(filename=relative_path_name))
+                self.logger.info('File {filename} already exists! Updating database'.format(filename=relative_path_name))
 
             else:
                 self.logger.info('Generating report for {}'.format(report_id))
@@ -611,10 +627,26 @@ class vulnWhispererQualys(vulnWhispererBase):
                     self.logger.info('New Report ID: {}'.format(generated_report_id))
 
                     vuln_ready = self.qualys_scan.process_data(path=self.write_path, file_id=str(generated_report_id))
+                    # Map and transform fields
+                    vuln_ready = self.qualys_scan.normalise(vuln_ready)
 
-                    vuln_ready['scan_name'] = scan_name
+                    # Set common fields
-                    vuln_ready['scan_reference'] = scan_reference
+                    vuln_ready['app_id'] = report_id
-                    vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
+                    vuln_ready['scan_id'] = scan_reference
+                    vuln_ready['scan_name'] = scan_name.encode('utf8')
+                    vuln_ready['scan_source'] = self.CONFIG_SECTION
+                    vuln_ready['scan_time'] = launched_date
+
+                    vuln_ready = self.common_normalise(vuln_ready)
+
+                    if output_format == 'json':
+                        vuln_ready.to_json(relative_path_name + '.tmp', orient='records', lines=True)
+                    elif output_format == 'csv':
+                        vuln_ready.to_csv(relative_path_name + '.tmp', index=False, header=True)
+                    os.rename(relative_path_name + '.tmp', relative_path_name)
+
+                    self.logger.info('{records} records written to {path} '.format(records=vuln_ready.shape[0],
+                                                                                   path=relative_path_name))
 
                     record_meta = (
                         scan_name,
@@ -629,21 +661,11 @@ class vulnWhispererQualys(vulnWhispererBase):
                         0,
                     )
                     self.record_insert(record_meta)
-
+                    self.logger.info('Scan {} ({}) written to database'.format(scan_name.encode('utf8'), report_id))
-                    if output_format == 'json':
-                        with open(relative_path_name, 'w') as f:
-                            f.write(vuln_ready.to_json(orient='records', lines=True))
-                            f.write('\n')
-
-                    elif output_format == 'csv':
-                       vuln_ready.to_csv(relative_path_name, index=False, header=True)  # add when timestamp occured
-
-                    self.logger.info('Report written to {}'.format(report_name))
 
                     if cleanup:
                         self.logger.info('Removing report {} from Qualys Database'.format(generated_report_id))
-                        cleaning_up = \
+                        cleaning_up = self.qualys_scan.qw.delete_report(generated_report_id)
-                            self.qualys_scan.qw.delete_report(generated_report_id)
                         os.remove(self.path_check(str(generated_report_id) + '.csv'))
                         self.logger.info('Deleted report from local disk: {}'.format(self.path_check(str(generated_report_id))))
                 else:
@@ -686,6 +708,7 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
                       'Hostname': 'hostname',
                       'Port': 'port',
                       'Port Protocol': 'protocol',
+                      'CVEs': 'cve',
                       'CVSS': 'cvss',
                       'Severity': 'severity',
                       'Solution Type': 'category',
@@ -694,7 +717,7 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
                       'Specific Result': 'plugin_output',
                       'NVT OID': 'nvt_oid',
                       'Task ID': 'task_id',
-                      'Task Name': 'task_name',
+                      'Task Name': 'scan_name',
                       'Timestamp': 'timestamp',
                       'Result ID': 'result_id',
                       'Impact': 'description',
@@ -762,7 +785,30 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
                     0,
                 )
                 self.record_insert(record_meta)
-                self.logger.info('File {filename} already exist! Updating database'.format(filename=relative_path_name))
+                self.logger.info('File {filename} already exists! Updating database'.format(filename=relative_path_name))
+
+
+            else:
+                vuln_ready = self.openvas_api.process_report(report_id=report_id)
+                # Map and transform fields
+                vuln_ready = self.openvas_api.normalise(vuln_ready)
+
+                # TODO move the following to the openvas_api.transform_values
+                vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
+                vuln_ready.port = vuln_ready.port.replace('', 0).astype(int)
+
+                # Set common fields
+                # vuln_ready['scan_name'] = scan_name.encode('utf8')
+                vuln_ready['scan_id'] = report_id
+                vuln_ready['scan_time'] = launched_date
+                vuln_ready['scan_source'] = self.CONFIG_SECTION
+
+                vuln_ready = self.common_normalise(vuln_ready)
+
+                vuln_ready.to_json(relative_path_name + '.tmp', orient='records', lines=True)
+                os.rename(relative_path_name + '.tmp', relative_path_name)
+                self.logger.info('{records} records written to {path} '.format(records=vuln_ready.shape[0],
+                                                                               path=relative_path_name))
 
                 record_meta = (
                     scan_name,
@@ -770,24 +816,14 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
                     launched_date,
                     report_name,
                     time.time(),
-                    file_length,
+                    vuln_ready.shape[0],
                     self.CONFIG_SECTION,
                     report_id,
                     1,
+                    0,
                 )
-
+                self.record_insert(record_meta)
-            else:
+                self.logger.info('Scan {} ({}) written to database'.format(scan_name.encode('utf8'), report_id))
-                vuln_ready = self.openvas_api.process_report(report_id=report_id)
-                vuln_ready['scan_name'] = scan_name
-                vuln_ready['scan_reference'] = report_id
-                vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
-                vuln_ready.port = vuln_ready.port.fillna(0).astype(int)
-                vuln_ready.fillna('', inplace=True)
-                if output_format == 'json':
-                    with open(relative_path_name, 'w') as f:
-                        f.write(vuln_ready.to_json(orient='records', lines=True))
-                        f.write('\n')
-                self.logger.info('Report written to {}'.format(report_name))
 
         return report
 
@@ -819,13 +855,6 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
 class vulnWhispererQualysVuln(vulnWhispererBase):
 
     CONFIG_SECTION = 'qualys_vuln'
-    COLUMN_MAPPING = {'cvss_base': 'cvss',
-                     'cvss3_base': 'cvss3',
-                     'cve_id': 'cve',
-                     'os': 'operating_system',
-                     'qid': 'plugin_id',
-                     'severity': 'risk',
-                     'title': 'plugin_name'}
 
     def __init__(
             self,
@@ -854,12 +883,11 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
                         scan_reference=None,
                         output_format='json',
                         cleanup=True):
-            launched_date
             if 'Z' in launched_date:
                 launched_date = self.qualys_scan.utils.iso_to_epoch(launched_date)
             report_name = 'qualys_vuln_' + report_id.replace('/','_') \
                           + '_{last_updated}'.format(last_updated=launched_date) \
-                          + '.json'
+                          + '.{extension}'.format(extension=output_format)
 
             relative_path_name = self.path_check(report_name).encode('utf8')
 
@@ -879,20 +907,36 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
                     0,
                 )
                 self.record_insert(record_meta)
-                self.logger.info('File {filename} already exist! Updating database'.format(filename=relative_path_name))
+                self.logger.info('File {filename} already exists! Updating database'.format(filename=relative_path_name))
 
             else:
                 try:
-                    self.logger.info('Processing report ID: {}'.format(report_id))
+                    self.logger.info('Processing {}: {}'.format(report_id, scan_name.encode('utf8')))
                     vuln_ready = self.qualys_scan.process_data(scan_id=report_id)
-                    vuln_ready['scan_name'] = scan_name
+                    # Map and transform fields
-                    vuln_ready['scan_reference'] = report_id
+                    vuln_ready = self.qualys_scan.normalise(vuln_ready)
-                    vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
+
+                    # Set common fields
+                    vuln_ready['scan_name'] = scan_name.encode('utf8')
+                    vuln_ready['scan_id'] = report_id
+                    vuln_ready['scan_time'] = launched_date
+                    vuln_ready['scan_source'] = self.CONFIG_SECTION
+
+                    vuln_ready = self.common_normalise(vuln_ready)
+
                 except Exception as e:
                     self.logger.error('Could not process {}: {}'.format(report_id, str(e)))
                     self.exit_code += 1
                     return self.exit_code
 
+                if output_format == 'json':
+                    vuln_ready.to_json(relative_path_name + '.tmp', orient='records', lines=True)
+                elif output_format == 'csv':
+                    vuln_ready.to_csv(relative_path_name + '.tmp', index=False, header=True)
+                os.rename(relative_path_name + '.tmp', relative_path_name)
+                self.logger.info('{records} records written to {path} '.format(records=vuln_ready.shape[0],
+                                                                               path=relative_path_name))
+
                 record_meta = (
                     scan_name,
                     scan_reference,
@@ -906,13 +950,8 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
                     0,
                 )
                 self.record_insert(record_meta)
+                self.logger.info('Scan {} ({}) written to database'.format(scan_name.encode('utf8'), report_id))
 
-                if output_format == 'json':
-                    with open(relative_path_name, 'w') as f:
-                        f.write(vuln_ready.to_json(orient='records', lines=True))
-                        f.write('\n')
-
-                self.logger.info('Report written to {}'.format(report_name))
             return self.exit_code
 
 
@@ -1258,9 +1297,6 @@ class vulnWhisperer(object):
 
         if self.profile == 'nessus':
             vw = vulnWhispererNessus(config=self.config,
-                                     username=self.username,
-                                     password=self.password,
-                                     verbose=self.verbose,
                                      profile=self.profile)
             self.exit_code += vw.whisper_nessus()
 
@@ -1274,9 +1310,6 @@ class vulnWhisperer(object):
 
         elif self.profile == 'tenable':
             vw = vulnWhispererNessus(config=self.config,
-                                     username=self.username,
-                                     password=self.password,
-                                     verbose=self.verbose,
                                      profile=self.profile)
             self.exit_code += vw.whisper_nessus()