Merge pull request #180 from pemontto/feature-filter-scans

This PR adds two CLI flags to filter which scans get imported/requested and one to list the scans:

    -f, --filter: allows supplying a regex pattern to match a scan name (this can also be specified in the config file)
    --days: the number of days to look back from the current date for scans (not supported on OpenVAS yet)
    --list: lists scans matching the filters and shows their imported/processed status

Other changes:

    combined all Logstash config into a single file
    create cvss and cvss_severity field which will always be populated from either cvss3 or cvss2
    renamed qualys_web -> qualys_was
    renamed qualys_vuln -> qualys_vm
    renamed plugin -> signature in field mappings
    added a helper script to pull Kibana API objects
    updated ES index template

.gitmodules

@@ -1,3 +1,4 @@
 [submodule "tests/data"]
 	path = tests/data
 	url = https://github.com/HASecuritySolutions/VulnWhisperer-tests.git
+	branch = master

.travis.yml

@@ -22,13 +22,13 @@ before_install:
 install:
     - pip install -r requirements.txt
     - pip install flake8  # pytest  # add another testing frameworks later
+    - python setup.py install
 before_script:
     # stop the build if there are Python syntax errors or undefined names
     - flake8 . --count --exclude=deps/qualysapi --select=E901,E999,F821,F822,F823 --show-source --statistics
     # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
     - flake8 . --count --exit-zero --exclude=deps/qualysapi --max-complexity=10 --max-line-length=127 --statistics
 script:
-    - python setup.py install
     - bash tests/test-vuln_whisperer.sh
     - bash tests/test-docker.sh
 notifications:

bin/vuln_whisperer

@@ -3,13 +3,14 @@
 __author__ = 'Austin Taylor'
 
 
-from vulnwhisp.vulnwhisp import vulnWhisperer
+import argparse
+import logging
+import os
+import sys
+
 from vulnwhisp.base.config import vwConfig
 from vulnwhisp.test.mock import mockAPI
-import os
-import argparse
-import sys
-import logging
+from vulnwhisp.vulnwhisp import vulnWhisperer
 
 
 def isFileValid(parser, arg):
@@ -27,16 +28,18 @@ def main():
                         help='Path of config file', type=lambda x: isFileValid(parser, x.strip()))
     parser.add_argument('-s', '--section', dest='section', required=False,
                         help='Section in config')
+    parser.add_argument('-f', '--filter', dest='scan_filter', required=False,
+                        help='Retrieve scans matching this regex pattern')
+    parser.add_argument('--days', dest='days', type=int, required=False,
+                        help='Retrieve scans from this many days ago to now')
+    parser.add_argument('-l', '--list', dest='list_scans', required=False, action="store_true",
+                        help='List available scans')
     parser.add_argument('--source', dest='source', required=False,
                         help='JIRA required only! Source scanner to report')
     parser.add_argument('-n', '--scanname', dest='scanname', required=False,
                         help='JIRA required only! Scan name from scan to report')
-    parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=True,
+    parser.add_argument('-v', '--verbose', dest='verbose', action='store_true',
                         help='Prints status out to screen (defaults to True)')
-    parser.add_argument('-u', '--username', dest='username', required=False, default=None,
-                        help='The NESSUS username', type=lambda x: x.strip())
-    parser.add_argument('-p', '--password', dest='password', required=False, default=None,
-                        help='The NESSUS password', type=lambda x: x.strip())
     parser.add_argument('-F', '--fancy', action='store_true',
                         help='Enable colourful logging output')
     parser.add_argument('-d', '--debug', action='store_true',
@@ -50,14 +53,14 @@ def main():
     # First setup logging
     logging.basicConfig(
         stream=sys.stdout,
-        #format only applies when not using -F flag for colouring
+        # format only applies when not using -F flag for colouring
         format='%(levelname)s:%(name)s:%(funcName)s:%(message)s',
-        level=logging.DEBUG if args.debug else logging.INFO
+        level=logging.DEBUG if args.debug else logging.INFO if args.verbose else logging.WARNING
     )
     logger = logging.getLogger()
     # we set up the logger to log as well to file
     fh = logging.FileHandler('vulnwhisperer.log')
-    fh.setLevel(logging.DEBUG if args.debug else logging.INFO)
+    fh.setLevel(logging.DEBUG if args.debug else logging.INFO if args.verbose else logging.WARNING)
     fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s - %(funcName)s:%(message)s", "%Y-%m-%d %H:%M:%S"))
     logger.addHandler(fh)
 
@@ -74,11 +77,13 @@ def main():
     try:
         if args.config and not args.section:
             # this remains a print since we are in the main binary
-            print('WARNING: {warning}'.format(warning='No section was specified, vulnwhisperer will scrape enabled modules from config file. \
-                            \nPlease specify a section using -s. \
-                                                    \nExample vuln_whisperer -c config.ini -s nessus'))
+            print(
+                "WARNING: No section was specified, vulnwhisperer will scrape enabled modules from config file. \
+                \nPlease specify a section using -s. \
+                \nExample vuln_whisperer -c config.ini -s nessus"
+            )
             logger.info('No section was specified, vulnwhisperer will scrape enabled modules from the config file.')
-            
+
             config = vwConfig(config_in=args.config)
             enabled_sections = config.get_sections_with_attribute('enabled')
 
@@ -86,30 +91,32 @@ def main():
                 vw = vulnWhisperer(config=args.config,
                                    profile=section,
                                    verbose=args.verbose,
-                                   username=args.username,
-                                   password=args.password,
+                                   debug=args.debug,
                                    source=args.source,
-                                   scanname=args.scanname)
+                                   scan_filter=args.scan_filter,
+                                   days=args.days,
+                                   scanname=args.scanname,
+                                   list_scans=args.list_scans)
                 exit_code += vw.whisper_vulnerabilities()
         else:
             logger.info('Running vulnwhisperer for section {}'.format(args.section))
             vw = vulnWhisperer(config=args.config,
                                profile=args.section,
                                verbose=args.verbose,
-                               username=args.username,
-                               password=args.password,
+                               debug=args.debug,
                                source=args.source,
-                               scanname=args.scanname)
+                               scan_filter=args.scan_filter,
+                               days=args.days,
+                               scanname=args.scanname,
+                               list_scans=args.list_scans)
             exit_code += vw.whisper_vulnerabilities()
 
         close_logging_handlers(logger)
         sys.exit(exit_code)
 
     except Exception as e:
-        if args.verbose:
-            # this will remain a print since we are in the main binary
-            logger.error('{}'.format(str(e)))
-            print('ERROR: {error}'.format(error=e))
+        logger.error('{}'.format(str(e)))
+        print('ERROR: {error}'.format(error=e))
         # TODO: fix this to NOT be exit 2 unless in error
         close_logging_handlers(logger)
         sys.exit(2)

configs/frameworks_example.ini

@@ -2,23 +2,29 @@
 enabled=true
 hostname=localhost
 port=8834
+access_key=
+secret_key=
 username=nessus_username
 password=nessus_password
 write_path=/opt/VulnWhisperer/data/nessus/
 db_path=/opt/VulnWhisperer/data/database
 trash=false
-verbose=true
+verbose=false
+scan_filter=
 
 [tenable]
 enabled=true
 hostname=cloud.tenable.com
 port=443
+access_key=
+secret_key=
 username=tenable.io_username
 password=tenable.io_password
 write_path=/opt/VulnWhisperer/data/tenable/
 db_path=/opt/VulnWhisperer/data/database
 trash=false
-verbose=true
+verbose=false
+scan_filter=
 
 [qualys_web]
 #Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
@@ -29,6 +35,7 @@ password = examplepass
 write_path=/opt/VulnWhisperer/data/qualys_web/
 db_path=/opt/VulnWhisperer/data/database
 verbose=true
+scan_filter=
 
 # Set the maximum number of retries each connection should attempt.
 #Note, this applies only to failed connections and timeouts, never to requests where the server returns a response.
@@ -44,7 +51,8 @@ username = exampleuser
 password = examplepass
 write_path=/opt/VulnWhisperer/data/qualys_vuln/
 db_path=/opt/VulnWhisperer/data/database
-verbose=true
+verbose=false
+scan_filter=
 
 [detectify]
 #Reference https://developer.detectify.com/
@@ -57,6 +65,7 @@ password = examplepass
 write_path =/opt/VulnWhisperer/data/detectify/
 db_path = /opt/VulnWhisperer/data/database
 verbose = true
+scan_filter=
 
 [openvas]
 enabled = false
@@ -66,7 +75,8 @@ username = exampleuser
 password = examplepass
 write_path=/opt/VulnWhisperer/data/openvas/
 db_path=/opt/VulnWhisperer/data/database
-verbose=true
+verbose=false
+scan_filter=
 
 [jira]
 enabled = false
@@ -84,7 +94,7 @@ dns_resolv = False
 #scan_name = Test Scan
 #jira_project = PROJECT
 ; if multiple components, separate by "," = None
-#components = 
+#components =
 ; minimum criticality to report (low, medium, high or critical) = None
 #min_critical_to_report = high
 

configs/test.ini

@@ -2,89 +2,99 @@
 enabled=true
 hostname=nessus
 port=443
+access_key=
+secret_key=
 username=nessus_username
 password=nessus_password
 write_path=/opt/VulnWhisperer/data/nessus/
 db_path=/opt/VulnWhisperer/data/database
 trash=false
-verbose=true
+verbose=false
+scan_filter=
 
 [tenable]
 enabled=true
 hostname=tenable
 port=443
+access_key=
+secret_key=
 username=tenable.io_username
 password=tenable.io_password
 write_path=/opt/VulnWhisperer/data/tenable/
 db_path=/opt/VulnWhisperer/data/database
 trash=false
-verbose=true
+verbose=false
+scan_filter=
 
-[qualys_web]
+[qualys_was]
 #Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
-enabled = false
-hostname = qualys_web
-username = exampleuser
-password = examplepass
-write_path=/opt/VulnWhisperer/data/qualys_web/
+enabled=true
+hostname=qualys_was
+username=exampleuser
+password=examplepass
+write_path=/opt/VulnWhisperer/data/qualys_was/
 db_path=/opt/VulnWhisperer/data/database
-verbose=true
+verbose=false
+scan_filter=
 
 # Set the maximum number of retries each connection should attempt.
 #Note, this applies only to failed connections and timeouts, never to requests where the server returns a response.
-max_retries = 10
+max_retries=10
 # Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID.
-template_id = 126024
+template_id=289109
 
-[qualys_vuln]
+[qualys_vm]
 #Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
-enabled = true
-hostname = qualys_vuln
-username = exampleuser
-password = examplepass
-write_path=/opt/VulnWhisperer/data/qualys_vuln/
+enabled=true
+hostname=qualys_vm
+username=exampleuser
+password=examplepass
+write_path=/opt/VulnWhisperer/data/qualys_vm/
 db_path=/opt/VulnWhisperer/data/database
-verbose=true
+verbose=false
+scan_filter=
 
 [detectify]
 #Reference https://developer.detectify.com/
-enabled = false
-hostname = detectify
+enabled=false
+hostname=detectify
 #username variable used as apiKey
-username = exampleuser
+username=exampleuser
 #password variable used as secretKey
-password = examplepass
+password=examplepass
 write_path =/opt/VulnWhisperer/data/detectify/
-db_path = /opt/VulnWhisperer/data/database
-verbose = true
+db_path=/opt/VulnWhisperer/data/database
+verbose=false
+scan_filter=
 
 [openvas]
-enabled = false
-hostname = openvas
-port = 4000
-username = exampleuser
-password = examplepass
+enabled=true
+hostname=openvas
+port=4000
+username=exampleuser
+password=examplepass
 write_path=/opt/VulnWhisperer/data/openvas/
 db_path=/opt/VulnWhisperer/data/database
-verbose=true
+verbose=false
+scan_filter=
 
 [jira]
-enabled = false
-hostname = jira-host
-username = username
-password = password
-write_path = /opt/VulnWhisperer/data/jira/
-db_path = /opt/VulnWhisperer/data/database
-verbose = true
-dns_resolv = False
+enabled=false
+hostname=jira-host
+username=username
+password=password
+write_path=/opt/VulnWhisperer/data/jira/
+db_path=/opt/VulnWhisperer/data/database
+verbose=false
+dns_resolv=False
 
 #Sample jira report scan, will automatically be created for existent scans
-#[jira.qualys_vuln.test_scan]
-#source = qualys_vuln
-#scan_name = Test Scan
-#jira_project = PROJECT
-; if multiple components, separate by "," = None
-#components = 
-; minimum criticality to report (low, medium, high or critical) = None
-#min_critical_to_report = high
+#[jira.qualys_vm.test_scan]
+#source=qualys_vm
+#scan_name=Test Scan
+#jira_project=PROJECT
+; if multiple components, separate by ","=None
+#components =
+; minimum criticality to report (low, medium, high or critical)=None
+#min_critical_to_report=high
 

docker-compose-test.yml

@@ -9,6 +9,7 @@ services:
       - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
       - xpack.security.enabled=false
       - cluster.routing.allocation.disk.threshold_enabled=false
+      - path.repo=/snapshots
     ulimits:
       memlock:
         soft: -1
@@ -80,12 +81,13 @@ services:
     entrypoint: [
                   "vuln_whisperer",
                   "-F",
+                  "-v",
                   "-c",
                   "/opt/VulnWhisperer/vulnwhisperer.ini",
                   "--mock",
                   "--mock_dir",
                   "/tests/data"
-    ]    
+    ]
     volumes:
       - ./data/vulnwhisperer/:/opt/VulnWhisperer/data
       # - ./resources/elk6/vulnwhisperer.ini:/opt/VulnWhisperer/vulnwhisperer.ini

docker-compose.v6.yml

@@ -8,6 +8,7 @@ services:
       - bootstrap.memory_lock=true
       - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
       - xpack.security.enabled=false
+      - path.repo=/snapshots
     ulimits:
       memlock:
         soft: -1
@@ -18,6 +19,7 @@ services:
     mem_limit: 8g
     volumes:
       - esdata1:/usr/share/elasticsearch/data
+      - ./data/es_snapshots:/snapshots
     ports:
       - 9200:9200
     #restart: always

resources/elk5-old_compatibility/docker-compose_ELK5_unsupported.yml

@@ -1,72 +0,0 @@
-version: '2'
-services:
-  vulnwhisp-es1:
-    image: docker.elastic.co/elasticsearch/elasticsearch:5.6.2
-    container_name: vulnwhisp-es1
-    environment:
-      - cluster.name=vulnwhisperer
-      - bootstrap.memory_lock=true
-      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
-    ulimits:
-      memlock:
-        soft: -1
-        hard: -1
-      nofile:
-        soft: 65536
-        hard: 65536
-    mem_limit: 8g
-    volumes:
-      - esdata1:/usr/share/elasticsearch/data
-    ports:
-      - 9200:9200
-    environment:
-      - xpack.security.enabled=false
-    #restart: always
-    networks:
-      esnet:
-        aliases:
-          - vulnwhisp-es1.local
-  vulnwhisp-ks1:
-    image: docker.elastic.co/kibana/kibana:5.6.2
-    environment:
-      SERVER_NAME: vulnwhisp-ks1
-      ELASTICSEARCH_URL: http://vulnwhisp-es1:9200
-    ports:
-      - 5601:5601
-    depends_on:
-      - vulnwhisp-es1
-    networks:
-      esnet:
-        aliases:
-          - vulnwhisp-ks1.local
-  vulnwhisp-ls1:
-    image: docker.elastic.co/logstash/logstash:5.6.2
-    container_name: vulnwhisp-ls1
-    volumes:
-      - ./docker/1000_nessus_process_file.conf:/usr/share/logstash/pipeline/1000_nessus_process_file.conf
-      - ./docker/2000_qualys_web_scans.conf:/usr/share/logstash/pipeline/2000_qualys_web_scans.conf
-      - ./docker/3000_openvas.conf:/usr/share/logstash/pipeline/3000_openvas.conf
-      - ./docker/4000_jira.conf:/usr/share/logstash/pipeline/4000_jira.conf
-      - ./docker/logstash.yml:/usr/share/logstash/config/logstash.yml
-      - ./data/:/opt/VulnWhisperer/data
-    environment:
-      - xpack.monitoring.enabled=false
-    depends_on:
-      - vulnwhisp-es1
-    networks:
-      esnet:
-        aliases:
-          - vulnwhisp-ls1.local
-  vulnwhisp-vulnwhisperer:
-    image: hasecuritysolutions/vulnwhisperer:latest
-    container_name: vulnwhisp-vulnwhisperer
-    volumes:
-      - ./data/:/opt/VulnWhisperer/data
-      - ./configs/frameworks_example.ini:/opt/VulnWhisperer/frameworks_example.ini
-    network_mode: host
-volumes:
-  esdata1:
-    driver: local
-
-networks:
-  esnet:

resources/elk5-old_compatibility/docker/1000_nessus_process_file.conf

@@ -1,220 +0,0 @@
-# Author: Austin Taylor and Justin Henderson
-# Email: email@austintaylor.io
-# Last Update: 12/20/2017
-# Version 0.3
-# Description: Take in nessus reports from vulnWhisperer and pumps into logstash
-
-
-input {
-  file {
-    path => "/opt/VulnWhisperer/data/nessus/**/*"
-    start_position => "beginning"
-    tags => "nessus"
-    type => "nessus"
-  }
-  file {
-    path => "/opt/VulnWhisperer/data/tenable/*.csv"
-    start_position => "beginning"
-    tags => "tenable"
-    type => "tenable"
-  }
-}
-
-filter {
-  if "nessus" in [tags] or "tenable" in [tags] {
-    # Drop the header column
-    if [message] =~ "^Plugin ID" { drop {} }
-
-    csv {
-      # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"]
-      columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss_vector", "system_type", "host_start", "host_end"]
-      separator => ","
-      source => "message"
-    }
-
-    ruby {
-        code => "if event.get('description')
-                    event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('synopsis')
-                    event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('solution')
-                    event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('see_also')
-                    event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('plugin_output')
-                    event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end"
-    }
-
-    #If using filebeats as your source, you will need to replace the "path" field to "source"
-    grok {
-      match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.csv$" }
-      tag_on_failure => []
-    }
-
-    date {
-      match => [ "last_updated", "UNIX" ]
-      target => "@timestamp"
-      remove_field => ["last_updated"]
-    }
-
-    if [risk] == "None" {
-      mutate { add_field => { "risk_number" => 0 }}
-    }
-    if [risk] == "Low" {
-      mutate { add_field => { "risk_number" => 1 }}
-    }
-    if [risk] == "Medium" {
-      mutate { add_field => { "risk_number" => 2 }}
-    }
-    if [risk] == "High" {
-      mutate { add_field => { "risk_number" => 3 }}
-    }
-    if [risk] == "Critical" {
-      mutate { add_field => { "risk_number" => 4 }}
-    }
-
-    if ![cve] or [cve] == "nan" {
-      mutate { remove_field => [ "cve" ] }
-    }
-    if ![cvss] or [cvss] == "nan" {
-      mutate { remove_field => [ "cvss" ] }
-    }
-    if ![cvss_base] or [cvss_base] == "nan" {
-      mutate { remove_field => [ "cvss_base" ] }
-    }
-    if ![cvss_temporal] or [cvss_temporal] == "nan" {
-      mutate { remove_field => [ "cvss_temporal" ] }
-    }
-    if ![cvss_temporal_vector] or [cvss_temporal_vector] == "nan" {
-      mutate { remove_field => [ "cvss_temporal_vector" ] }
-    }
-    if ![cvss_vector] or [cvss_vector] == "nan" {
-      mutate { remove_field => [ "cvss_vector" ] }
-    }
-    if ![cvss3_base] or [cvss3_base] == "nan" {
-      mutate { remove_field => [ "cvss3_base" ] }
-    }
-    if ![cvss3_temporal] or [cvss3_temporal] == "nan" {
-      mutate { remove_field => [ "cvss3_temporal" ] }
-    }
-    if ![cvss3_temporal_vector] or [cvss3_temporal_vector] == "nan" {
-      mutate { remove_field => [ "cvss3_temporal_vector" ] }
-    }
-    if ![description] or [description] == "nan" {
-      mutate { remove_field => [ "description" ] }
-    }
-    if ![mac_address] or [mac_address] == "nan" {
-      mutate { remove_field => [ "mac_address" ] }
-    }
-    if ![netbios] or [netbios] == "nan" {
-      mutate { remove_field => [ "netbios" ] }
-    }
-    if ![operating_system] or [operating_system] == "nan" {
-      mutate { remove_field => [ "operating_system" ] }
-    }
-    if ![plugin_output] or [plugin_output] == "nan" {
-      mutate { remove_field => [ "plugin_output" ] }
-    }
-    if ![see_also] or [see_also] == "nan" {
-      mutate { remove_field => [ "see_also" ] }
-    }
-    if ![synopsis] or [synopsis] == "nan" {
-      mutate { remove_field => [ "synopsis" ] }
-    }
-    if ![system_type] or [system_type] == "nan" {
-      mutate { remove_field => [ "system_type" ] }
-    }
-
-    mutate {
-      remove_field => [ "message" ]
-      add_field => { "risk_score" => "%{cvss}" }
-    }
-    mutate {
-      convert => { "risk_score" => "float" }
-    }
-    if [risk_score] == 0 {
-      mutate {
-        add_field => { "risk_score_name" => "info" }
-      }
-    }
-    if [risk_score] > 0 and [risk_score] < 3 {
-      mutate {
-        add_field => { "risk_score_name" => "low" }
-      }
-    }
-    if [risk_score] >= 3 and [risk_score] < 6 {
-      mutate {
-        add_field => { "risk_score_name" => "medium" }
-      }
-    }
-    if [risk_score] >=6 and [risk_score] < 9 {
-      mutate {
-        add_field => { "risk_score_name" => "high" }
-      }
-    }
-    if [risk_score] >= 9 {
-      mutate {
-        add_field => { "risk_score_name" => "critical" }
-      }
-    }
-
-    # Compensating controls - adjust risk_score
-    # Adobe and Java are not allowed to run in browser unless whitelisted
-    # Therefore, lower score by dividing by 3 (score is subjective to risk)
-
-    #Modify and uncomment when ready to use
-    #if [risk_score] != 0 {
-    #  if [plugin_name] =~ "Adobe" and [risk_score] > 6 or [plugin_name] =~ "Java" and [risk_score] > 6 {
-    #    ruby {
-    #      code => "event.set('risk_score', event.get('risk_score') / 3)"
-    #    }
-    #    mutate {
-    #      add_field => { "compensating_control" => "Adobe and Flash removed from browsers unless whitelisted site." }
-    #    }
-    #  }
-    #}
-
-    # Add tags for reporting based on assets or criticality
-
-    if [asset] == "dc01" or [asset] == "dc02" or [asset] == "pki01" or [asset] == "192.168.0.54" or [asset] =~ "^192\.168\.0\." or [asset] =~ "^42.42.42." {
-      mutate {
-        add_tag => [ "critical_asset" ]
-      }
-    }
-    #if [asset] =~ "^192\.168\.[45][0-9][0-9]\.1$" or [asset] =~ "^192.168\.[50]\.[0-9]{1,2}\.1$"{
-    #  mutate {
-    #    add_tag => [ "has_hipaa_data" ]
-    #  }
-    #}
-    #if [asset] =~ "^192\.168\.[45][0-9][0-9]\." {
-    #  mutate {
-    #    add_tag => [ "hipaa_asset" ]
-    #  }
-    #}
-    if [asset] =~ "^hr" {
-      mutate {
-        add_tag => [ "pci_asset" ]
-      }
-    }
-    #if [asset] =~ "^10\.0\.50\." {
-    #  mutate {
-    #    add_tag => [ "web_servers" ]
-    #  }
-    #}
-  }
-}
-
-output {
-  if "nessus" in [tags] or "tenable" in [tags] or [type] in [ "nessus", "tenable" ] {
-    # stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => [ "vulnwhisp-es1.local:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
-    }
-  }
-}

resources/elk5-old_compatibility/docker/2000_qualys_web_scans.conf

@@ -1,153 +0,0 @@
-# Author: Austin Taylor and Justin Henderson
-# Email: austin@hasecuritysolutions.com
-# Last Update: 12/30/2017
-# Version 0.3
-# Description: Take in qualys web scan reports from vulnWhisperer and pumps into logstash
-
-input {
-  file {
-    path => "/opt/VulnWhisperer/data/qualys/*.json"
-    type => json
-    codec => json
-    start_position => "beginning"
-    tags => [ "qualys" ]
-  }
-}
-
-filter {
-  if "qualys" in [tags] {
-    grok {
-      match => { "path" => [ "(?<tags>qualys_vuln)_scan_%{DATA}_%{INT:last_updated}.json$", "(?<tags>qualys_web)_%{INT:app_id}_%{INT:last_updated}.json$" ] }
-      tag_on_failure => []
-    }
-    
-    mutate {
-      replace => [ "message", "%{message}" ]
-      #gsub => [
-      #  "message", "\|\|\|", " ",
-      #  "message", "\t\t", " ",
-      #  "message", "    ", " ",
-      #  "message", "   ", " ",
-      #  "message", "  ", " ",
-      #  "message", "nan", " ",
-      #  "message",'\n',''
-      #]
-    }
-
-    if "qualys_web" in [tags] {
-    mutate {
-      add_field => { "asset" => "%{web_application_name}" }
-      add_field => { "risk_score" => "%{cvss}" }
-      }
-    } else if "qualys_vuln" in [tags] {
-      mutate {
-        add_field => { "asset" => "%{ip}" }
-        add_field => { "risk_score" => "%{cvss}" }
-      }
-    }
-
-    if [risk] == "1" {
-      mutate { add_field => { "risk_number" => 0 }}
-      mutate { replace => { "risk" => "info" }}
-    }
-    if [risk] == "2" {
-      mutate { add_field => { "risk_number" => 1 }}
-      mutate { replace => { "risk" => "low" }}
-    }
-    if [risk] == "3" {
-      mutate { add_field => { "risk_number" => 2 }}
-      mutate { replace => { "risk" => "medium" }}
-    }
-    if [risk] == "4" {
-      mutate { add_field => { "risk_number" => 3 }}
-      mutate { replace => { "risk" => "high" }}
-    }
-    if [risk] == "5" {
-      mutate { add_field => { "risk_number" => 4 }}
-      mutate { replace => { "risk" => "critical" }}
-    }
-
-    mutate {
-      remove_field => "message"
-    }
-
-    if [first_time_detected] {
-      date {
-        match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "first_time_detected"
-      }
-    }
-    if [first_time_tested] {
-      date {
-        match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "first_time_tested"
-      }
-    }
-    if [last_time_detected] {
-      date {
-        match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "last_time_detected"
-      }
-    }
-    if [last_time_tested] {
-      date {
-        match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "last_time_tested"
-      }
-    }
-    date {
-      match => [ "last_updated", "UNIX" ]
-      target => "@timestamp"
-      remove_field => "last_updated"
-    }
-    mutate {
-      convert => { "plugin_id" => "integer"}
-      convert => { "id" => "integer"}
-      convert => { "risk_number" => "integer"}
-      convert => { "risk_score" => "float"}
-      convert => { "total_times_detected" => "integer"}
-      convert => { "cvss_temporal" => "float"}
-      convert => { "cvss" => "float"}
-    }
-    if [risk_score] == 0 {
-      mutate {
-        add_field => { "risk_score_name" => "info" }
-      }
-    }
-    if [risk_score] > 0 and [risk_score] < 3 {
-      mutate {
-        add_field => { "risk_score_name" => "low" }
-      }
-    }
-    if [risk_score] >= 3 and [risk_score] < 6 {
-      mutate {
-        add_field => { "risk_score_name" => "medium" }
-      }
-    }
-    if [risk_score] >=6 and [risk_score] < 9 {
-      mutate {
-        add_field => { "risk_score_name" => "high" }
-      }
-    }
-    if [risk_score] >= 9 {
-      mutate {
-        add_field => { "risk_score_name" => "critical" }
-      }
-    }
-
-    if [asset] =~ "\.yourdomain\.(com|net)$" {
-      mutate {
-        add_tag => [ "critical_asset" ]
-      }
-    }
-  }
-}
-output {
-  if "qualys" in [tags] {
-    stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => [ "vulnwhisp-es1.local:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
-    }
-  }
-}

resources/elk5-old_compatibility/docker/3000_openvas.conf

@@ -1,146 +0,0 @@
-# Author: Austin Taylor and Justin Henderson
-# Email: austin@hasecuritysolutions.com
-# Last Update: 03/04/2018
-# Version 0.3
-# Description: Take in qualys web scan reports from vulnWhisperer and pumps into logstash
-
-input {
-  file {
-    path => "/opt/VulnWhisperer/data/openvas/*.json"
-    type => json
-    codec => json
-    start_position => "beginning"
-    tags => [ "openvas_scan", "openvas" ]
-  }
-}
-
-filter {
-  if "openvas_scan" in [tags] {
-    mutate {
-      replace => [ "message", "%{message}" ]
-      gsub => [
-        "message", "\|\|\|", " ",
-        "message", "\t\t", " ",
-        "message", "    ", " ",
-        "message", "   ", " ",
-        "message", "  ", " ",
-        "message", "nan", " ",
-        "message",'\n',''
-      ]
-    }
-
-
-    grok {
-        match => { "path" => "openvas_scan_%{DATA:scan_id}_%{INT:last_updated}.json$" }
-        tag_on_failure => []
-    }
-
-    mutate {
-      add_field => { "risk_score" => "%{cvss}" }
-    }
-
-    if [risk] == "1" {
-      mutate { add_field => { "risk_number" => 0 }}
-      mutate { replace => { "risk" => "info" }}
-    }
-    if [risk] == "2" {
-      mutate { add_field => { "risk_number" => 1 }}
-      mutate { replace => { "risk" => "low" }}
-    }
-    if [risk] == "3" {
-      mutate { add_field => { "risk_number" => 2 }}
-      mutate { replace => { "risk" => "medium" }}
-    }
-    if [risk] == "4" {
-      mutate { add_field => { "risk_number" => 3 }}
-      mutate { replace => { "risk" => "high" }}
-    }
-    if [risk] == "5" {
-      mutate { add_field => { "risk_number" => 4 }}
-      mutate { replace => { "risk" => "critical" }}
-    }
-
-    mutate {
-      remove_field => "message"
-    }
-
-    if [first_time_detected] {
-      date {
-        match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "first_time_detected"
-      }
-    }
-    if [first_time_tested] {
-      date {
-        match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "first_time_tested"
-      }
-    }
-    if [last_time_detected] {
-      date {
-        match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "last_time_detected"
-      }
-    }
-    if [last_time_tested] {
-      date {
-        match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "last_time_tested"
-      }
-    }
-    date {
-      match => [ "last_updated", "UNIX" ]
-      target => "@timestamp"
-      remove_field => "last_updated"
-    }
-    mutate {
-      convert => { "plugin_id" => "integer"}
-      convert => { "id" => "integer"}
-      convert => { "risk_number" => "integer"}
-      convert => { "risk_score" => "float"}
-      convert => { "total_times_detected" => "integer"}
-      convert => { "cvss_temporal" => "float"}
-      convert => { "cvss" => "float"}
-    }
-    if [risk_score] == 0 {
-      mutate {
-        add_field => { "risk_score_name" => "info" }
-      }
-    }
-    if [risk_score] > 0 and [risk_score] < 3 {
-      mutate {
-        add_field => { "risk_score_name" => "low" }
-      }
-    }
-    if [risk_score] >= 3 and [risk_score] < 6 {
-      mutate {
-        add_field => { "risk_score_name" => "medium" }
-      }
-    }
-    if [risk_score] >=6 and [risk_score] < 9 {
-      mutate {
-        add_field => { "risk_score_name" => "high" }
-      }
-    }
-    if [risk_score] >= 9 {
-      mutate {
-        add_field => { "risk_score_name" => "critical" }
-      }
-    }
-    # Add your critical assets by subnet or by hostname. Comment this field out if you don't want to tag any, but the asset panel will break.
-    if [asset] =~ "^10\.0\.100\." {
-      mutate {
-        add_tag => [ "critical_asset" ]
-      }
-    }
-  }
-}
-output {
-  if "openvas" in [tags] {
-    stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => [ "vulnwhisp-es1.local:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
-    }
-  }
-}

resources/elk5-old_compatibility/docker/4000_jira.conf

@@ -1,21 +0,0 @@
-# Description: Take in jira tickets from vulnWhisperer and pumps into logstash
-
-input {
-  file {
-    path => "/opt/Vulnwhisperer/jira/*.json"
-    type => json
-    codec => json
-    start_position => "beginning"
-    tags => [ "jira" ]
-  }
-}
-
-output {
-  if "jira" in [tags] {
-    stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => [ "vulnwhisp-es1.local:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
-    }
-  }
-}

resources/elk5-old_compatibility/docker/logstash.yml

@@ -1,5 +0,0 @@
-path.config: /usr/share/logstash/pipeline/
-xpack.monitoring.elasticsearch.password: changeme
-xpack.monitoring.elasticsearch.url: vulnwhisp-es1.local:9200
-xpack.monitoring.elasticsearch.username: elastic
-xpack.monitoring.enabled: false

resources/elk5-old_compatibility/elasticsearch/logstash-vulnwhisperer-template.json

@@ -1,122 +0,0 @@
-{
-  "order": 0,
-  "template": "logstash-vulnwhisperer-*",
-  "settings": {
-    "index": {
-      "routing": {
-        "allocation": {
-          "total_shards_per_node": "2"
-        }
-      },
-      "mapping": {
-        "total_fields": {
-          "limit": "3000"
-        }
-      },
-      "refresh_interval": "5s",
-      "number_of_shards": "1",
-      "number_of_replicas": "0"
-    }
-  },
-  "mappings": {
-    "_default_": {
-      "_all": {
-        "enabled": false
-      },
-      "dynamic_templates": [
-        {
-          "message_field": {
-            "path_match": "message",
-            "match_mapping_type": "string",
-            "mapping": {
-              "type": "text",
-              "norms": false
-            }
-          }
-        },
-        {
-          "string_fields": {
-            "match": "*",
-            "match_mapping_type": "string",
-            "mapping": {
-              "type": "text",
-              "norms": false,
-              "fields": {
-                "keyword": {
-                  "type": "keyword",
-                  "ignore_above": 256
-                }
-              }
-            }
-          }
-        }
-      ],
-      "properties": {
-        "plugin_id": {
-          "type": "float"
-        },
-        "last_updated": {
-          "type": "date"
-        },
-        "geoip": {
-          "dynamic": true,
-          "type": "object",
-          "properties": {
-            "ip": {
-              "type": "ip"
-            },
-            "latitude": {
-              "type": "float"
-            },
-            "location": {
-              "type": "geo_point"
-            },
-            "longitude": {
-              "type": "float"
-            }
-          }
-        },
-        "risk_score": {
-          "type": "float"
-        },
-        "source": {          
-          "type": "keyword"
-        },
-        "synopsis": {
-          "type": "keyword"
-        },
-        "see_also": {
-          "type": "keyword"
-        },
-        "@timestamp": {
-          "type": "date"
-        },
-        "cve": {
-          "type": "keyword"
-        },
-        "solution": {
-          "type": "keyword"
-        },
-        "port": {
-          "type": "integer"
-        },
-        "host": {
-          "type": "text"
-        },
-        "@version": {
-          "type": "keyword"
-        },
-        "risk": {
-          "type": "keyword"
-        },
-        "assign_ip": {
-          "type": "ip"
-        },
-        "cvss": {
-          "type": "float"
-        }
-      }
-    }
-  },
-  "aliases": {}
-}

resources/elk5-old_compatibility/filebeat/filebeat.yml

@@ -1,116 +0,0 @@
-###################### Filebeat Configuration Example #########################
-
-# This file is an example configuration file highlighting only the most common
-# options. The filebeat.full.yml file from the same directory contains all the
-# supported options with more comments. You can use it as a reference.
-#
-# You can find the full configuration reference here:
-# https://www.elastic.co/guide/en/beats/filebeat/index.html
-
-#=========================== Filebeat prospectors =============================
-
-filebeat.prospectors:
-
-# Each - is a prospector. Most options can be set at the prospector level, so
-# you can use different prospectors for various configurations.
-# Below are the prospector specific configurations.
-
-- input_type: log
-  # Paths that should be crawled and fetched. Glob based paths.
-  paths:
-    # Linux Example
-    #- /var/log/*.log
-
-    #Windows Example
-    - c:\nessus\My Scans\*
-
-  # Exclude lines. A list of regular expressions to match. It drops the lines that are
-  # matching any regular expression from the list.
-  #exclude_lines: ["^DBG"]
-
-  # Include lines. A list of regular expressions to match. It exports the lines that are
-  # matching any regular expression from the list.
-  #include_lines: ["^ERR", "^WARN"]
-
-  # Exclude files. A list of regular expressions to match. Filebeat drops the files that
-  # are matching any regular expression from the list. By default, no files are dropped.
-  #exclude_files: [".gz$"]
-
-  # Optional additional fields. These field can be freely picked
-  # to add additional information to the crawled log files for filtering
-  #fields:
-  #  level: debug
-  #  review: 1
-
-  ### Multiline options
-
-  # Mutiline can be used for log messages spanning multiple lines. This is common
-  # for Java Stack Traces or C-Line Continuation
-
-  # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [
-  #multiline.pattern: ^\[
-
-  # Defines if the pattern set under pattern should be negated or not. Default is false.
-  #multiline.negate: false
-
-  # Match can be set to "after" or "before". It is used to define if lines should be append to a pattern
-  # that was (not) matched before or after or as long as a pattern is not matched based on negate.
-  # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash
-  #multiline.match: after
-
-
-#================================ General =====================================
-
-# The name of the shipper that publishes the network data. It can be used to group
-# all the transactions sent by a single shipper in the web interface.
-#name:
-
-# The tags of the shipper are included in their own field with each
-# transaction published.
-#tags: ["service-X", "web-tier"]
-
-# Optional fields that you can specify to add additional information to the
-# output.
-#fields:
-#  env: staging
-
-#================================ Outputs =====================================
-
-# Configure what outputs to use when sending the data collected by the beat.
-# Multiple outputs may be used.
-
-#-------------------------- Elasticsearch output ------------------------------
-#output.elasticsearch:
-  # Array of hosts to connect to.
-#  hosts: ["logstash01:9200"]
-
-  # Optional protocol and basic auth credentials.
-  #protocol: "https"
-  #username: "elastic"
-  #password: "changeme"
-
-#----------------------------- Logstash output --------------------------------
-output.logstash:
-  # The Logstash hosts
-  hosts: ["logstashserver1:5044", "logstashserver2:5044", "logstashserver3:5044"]
-
-  # Optional SSL. By default is off.
-  # List of root certificates for HTTPS server verifications
-  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
-
-  # Certificate for SSL client authentication
-  #ssl.certificate: "/etc/pki/client/cert.pem"
-
-  # Client Certificate Key
-  #ssl.key: "/etc/pki/client/cert.key"
-
-#================================ Logging =====================================
-
-# Sets log level. The default log level is info.
-# Available log levels are: critical, error, warning, info, debug
-#logging.level: debug
-
-# At debug level, you can selectively enable logging only for some components.
-# To enable all selectors use ["*"]. Examples of other selectors are "beat",
-# "publish", "service".
-#logging.selectors: ["*"]

resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/1000_vulnWhispererBaseVisuals.json

@@ -1,450 +0,0 @@
-[
-  {
-    "_id": "80158c90-57c1-11e7-b484-a970fc9d150a",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - HIPAA TL",
-      "visState": "{\"type\":\"timelion\",\"title\":\"VulnWhisperer - HIPAA TL\",\"params\":{\"expression\":\".es(index=logstash-vulnwhisperer-*,q='risk_score:>9 AND tags:pci_asset').label(\\\"PCI Assets\\\"),.es(index=logstash-vulnwhisperer-*,q='risk_score:>9 AND tags:has_hipaa_data').label(\\\"Has HIPAA Data\\\"),.es(index=logstash-vulnwhisperer-*,q='risk_score:>9 AND tags:hipaa_asset').label(\\\"HIPAA Assets\\\")\",\"interval\":\"auto\"}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{}"
-      }
-    }
-  },
-  {
-    "_id": "479deab0-8a39-11e7-a58a-9bfcb3761a3d",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - TL - TaggedAssetsPluginNames",
-      "visState": "{\"title\":\"VulnWhisperer - TL - TaggedAssetsPluginNames\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*', q='tags:critical_asset OR tags:hipaa_asset OR tags:pci_asset', split=\\\"plugin_name.keyword:10\\\").bars(width=4).label(regex=\\\".*:(.+)>.*\\\",label=\\\"$1\\\")\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "84f5c370-8a38-11e7-a58a-9bfcb3761a3d",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - TL - CriticalAssetsPluginNames",
-      "visState": "{\"title\":\"VulnWhisperer - TL - CriticalAssetsPluginNames\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*', q='tags:critical_asset', split=\\\"plugin_name.keyword:10\\\").bars(width=4).label(regex=\\\".*:(.+)>.*\\\",label=\\\"$1\\\")\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "307cdae0-8a38-11e7-a58a-9bfcb3761a3d",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - TL - PluginNames",
-      "visState": "{\"title\":\"VulnWhisperer - TL - PluginNames\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*', split=\\\"plugin_name.keyword:25\\\").bars(width=4).label(regex=\\\".*:(.+)>.*\\\",label=\\\"$1\\\")\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "5093c620-44e9-11e7-8014-ede06a7e69f8",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Mitigation Readme",
-      "visState": "{\"title\":\"VulnWhisperer - Mitigation Readme\",\"type\":\"markdown\",\"params\":{\"markdown\":\"** Legend **\\n\\n* [Common Vulnerability Scoring System (CVSS)](https://nvd.nist.gov/vuln-metrics/cvss) is the NIST vulnerability scoring system\\n* Risk Number is residual risk score calculated from CVSS, which is adjusted to be specific to the netowrk owner, which accounts for services not in use such as Java and Flash\\n* Vulnerabilities by Tag are systems tagged with HIPAA and PCI identification.\\n\\n\\n** Workflow **\\n* Select 10.0 under Risk Number to identify Critical Vulnerabilities. \\n* For more information about a CVE, scroll down and click the CVE link.\\n* To filter by tags, use one of the following filters:\\n** tags:has_hipaa_data, tags:pci_asset, tags:hipaa_asset, tags:critical_asset**\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "7e7fbc90-3df2-11e7-a44e-c79ca8efb780",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer-PluginID",
-      "visState": "{\"title\":\"VulnWhisperer-PluginID\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"plugin_id\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "5a3c0340-3eb3-11e7-a192-93f36fbd9d05",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer-CVSSHeatmap",
-      "visState": "{\"title\":\"VulnWhisperer-CVSSHeatmap\",\"type\":\"heatmap\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"enableHover\":false,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":4,\"colorSchema\":\"Yellow to Red\",\"setColorRange\":false,\"colorsRange\":[],\"invertColors\":false,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":false,\"rotate\":0,\"color\":\"#555\"}}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"host.keyword\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"cvss.keyword\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"_term\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 3500\":\"rgb(255,255,204)\",\"3500 - 7000\":\"rgb(254,217,118)\",\"7000 - 10500\":\"rgb(253,141,60)\",\"10500 - 14000\":\"rgb(227,27,28)\"}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "1de9e550-3df1-11e7-a44e-c79ca8efb780",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer-Description",
-      "visState": "{\"title\":\"VulnWhisperer-Description\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"description.keyword\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Description\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "13c7d4e0-3df3-11e7-a44e-c79ca8efb780",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer-Solution",
-      "visState": "{\"title\":\"VulnWhisperer-Solution\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"solution.keyword\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Solution\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "297df800-3f7e-11e7-bd24-6903e3283192",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Plugin Name",
-      "visState": "{\"title\":\"VulnWhisperer - Plugin Name\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"plugin_name.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Plugin Name\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "de1a5f40-3f85-11e7-97f9-3777d794626d",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - ScanName",
-      "visState": "{\"title\":\"VulnWhisperer - ScanName\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"scan_name.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Scan Name\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "ecbb99c0-3f84-11e7-97f9-3777d794626d",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Total",
-      "visState": "{\"title\":\"VulnWhisperer - Total\",\"type\":\"metric\",\"params\":{\"handleNoResults\":true,\"fontSize\":60},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Total\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "471a3580-3f6b-11e7-88e7-df1abe6547fb",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Vulnerabilities by Tag",
-      "visState": "{\"title\":\"VulnWhisperer - Vulnerabilities by Tag\",\"type\":\"table\",\"params\":{\"perPage\":3,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"bucket\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"tags:has_hipaa_data\",\"analyze_wildcard\":true}}},\"label\":\"Systems with HIPAA data\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"tags:pci_asset\",\"analyze_wildcard\":true}}},\"label\":\"PCI Systems\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"tags:hipaa_asset\",\"analyze_wildcard\":true}}},\"label\":\"HIPAA Systems\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "35b6d320-3f7f-11e7-bd24-6903e3283192",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Residual Risk",
-      "visState": "{\"title\":\"VulnWhisperer - Residual Risk\",\"type\":\"table\",\"params\":{\"perPage\":15,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"risk_score\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Risk Number\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "a9225930-3df2-11e7-a44e-c79ca8efb780",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer-Risk",
-      "visState": "{\"title\":\"VulnWhisperer-Risk\",\"type\":\"table\",\"params\":{\"perPage\":4,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"risk\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Risk Severity\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "2f979030-44b9-11e7-a818-f5f80dfc3590",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - ScanBarChart",
-      "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customLabel\":\"Scan Name\",\"field\":\"plugin_name.keyword\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":10},\"schema\":\"segment\",\"type\":\"terms\"}],\"listeners\":{},\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"defaultYExtents\":false,\"legendPosition\":\"right\",\"mode\":\"stacked\",\"scale\":\"linear\",\"setYExtents\":false,\"times\":[]},\"title\":\"VulnWhisperer - ScanBarChart\",\"type\":\"histogram\"}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "a6508640-897a-11e7-bbc0-33592ce0be1e",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Critical Assets Aggregated",
-      "visState": "{\"title\":\"VulnWhisperer - Critical Assets Aggregated\",\"type\":\"heatmap\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"enableHover\":true,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":4,\"colorSchema\":\"Green to Red\",\"setColorRange\":true,\"colorsRange\":[{\"from\":0,\"to\":3},{\"from\":3,\"to\":7},{\"from\":7,\"to\":9},{\"from\":9,\"to\":11}],\"invertColors\":false,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":true,\"rotate\":0,\"color\":\"white\"}}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"risk_score\",\"customLabel\":\"Residual Risk Score\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Date\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"host\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Critical Asset IP\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"plugin_name.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"row\":true}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"colors\":{\"0 - 3\":\"#7EB26D\",\"3 - 7\":\"#EAB839\",\"7 - 9\":\"#EF843C\",\"8 - 10\":\"#BF1B00\",\"9 - 11\":\"#BF1B00\"},\"defaultColors\":{\"0 - 3\":\"rgb(0,104,55)\",\"3 - 7\":\"rgb(135,203,103)\",\"7 - 9\":\"rgb(255,255,190)\",\"9 - 11\":\"rgb(249,142,82)\"},\"legendOpen\":false}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":\"Critical Asset\",\"disabled\":false,\"index\":\"logstash-vulnwhisperer-*\",\"key\":\"tags\",\"negate\":false,\"type\":\"phrase\",\"value\":\"critical_asset\"},\"query\":{\"match\":{\"tags\":{\"query\":\"critical_asset\",\"type\":\"phrase\"}}}}]}"
-      }
-    }
-  },
-  {
-    "_id": "099a3820-3f68-11e7-a6bd-e764d950e506",
-    "_type": "visualization",
-    "_source": {
-      "title": "Timelion VulnWhisperer Example",
-      "visState": "{\"type\":\"timelion\",\"title\":\"Timelion VulnWhisperer Example\",\"params\":{\"expression\":\".es(index=logstash-vulnwhisperer-*,q=risk:high).label(\\\"Current High Risk\\\"),.es(index=logstash-vulnwhisperer-*,q=risk:high,offset=-1y).label(\\\"Last 1 Year High Risk\\\"),.es(index=logstash-vulnwhisperer-*,q=risk:medium).label(\\\"Current Medium Risk\\\"),.es(index=logstash-vulnwhisperer-*,q=risk:medium,offset=-1y).label(\\\"Last 1 Year Medium Risk\\\")\",\"interval\":\"auto\"}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{}"
-      }
-    }
-  },
-  {
-    "_id": "67d432e0-44ec-11e7-a05f-d9719b331a27",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - TL-Critical Risk",
-      "visState": "{\"title\":\"VulnWhisperer - TL-Critical Risk\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*',q='(risk_score:>=9 AND risk_score:<=10)').label(\\\"Original\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=9 AND risk_score:<=10)',offset=-1w).label(\\\"One week offset\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=9 AND risk_score:<=10)').subtract(.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=9 AND risk_score:<=10)',offset=-1w)).label(\\\"Difference\\\").lines(steps=3,fill=2,width=1)\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "a91b9fe0-44ec-11e7-a05f-d9719b331a27",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - TL-Medium Risk",
-      "visState": "{\"title\":\"VulnWhisperer - TL-Medium Risk\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*',q='(risk_score:>=4 AND risk_score:<7)').label(\\\"Original\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=4 AND risk_score:<7)',offset=-1w).label(\\\"One week offset\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=4 AND risk_score:<7)').subtract(.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=4 AND risk_score:<7)',offset=-1w)).label(\\\"Difference\\\").lines(steps=3,fill=2,width=1)\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "8d9592d0-44ec-11e7-a05f-d9719b331a27",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - TL-High Risk",
-      "visState": "{\"title\":\"VulnWhisperer - TL-High Risk\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*',q='(risk_score:>=7 AND risk_score:<9)').label(\\\"Original\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=7 AND risk_score:<9)',offset=-1w).label(\\\"One week offset\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=7 AND risk_score:<9)').subtract(.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=7 AND risk_score:<9)',offset=-1w)).label(\\\"Difference\\\").lines(steps=3,fill=2,width=1)\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "a2d66660-44ec-11e7-a05f-d9719b331a27",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - TL-Low Risk",
-      "visState": "{\"title\":\"VulnWhisperer - TL-Low Risk\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*',q='(risk_score:>0 AND risk_score:<4)').label(\\\"Original\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>0 AND risk_score:<4)',offset=-1w).label(\\\"One week offset\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>0 AND risk_score:<4)').subtract(.es(index='logstash-vulnwhisperer-*',q='(risk_score:>0 AND risk_score:<4)',offset=-1w)).label(\\\"Difference\\\").lines(steps=3,fill=2,width=1)\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "fb6eb020-49ab-11e7-8f8c-57ad64ec48a6",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Critical Risk Score for Tagged Assets",
-      "visState": "{\"title\":\"VulnWhisperer - Critical Risk Score for Tagged Assets\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index=logstash-vulnwhisperer-*,q='risk_score:>9 AND tags:hipaa_asset').label(\\\"HIPAA Assets\\\"),.es(index=logstash-vulnwhisperer-*,q='risk_score:>9 AND tags:pci_asset').label(\\\"PCI Systems\\\"),.es(index=logstash-vulnwhisperer-*,q='risk_score:>9 AND tags:has_hipaa_data').label(\\\"Has HIPAA Data\\\")\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "b2f2adb0-897f-11e7-a2d2-c57bca21b3aa",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer  - Risk: Total",
-      "visState": "{\"title\":\"VulnWhisperer  - Risk: Total\",\"type\":\"goal\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"gauge\":{\"autoExtend\":false,\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"gaugeColorMode\":\"Background\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Metric\",\"invertColors\":false,\"labels\":{\"color\":\"black\",\"show\":false},\"orientation\":\"vertical\",\"percentageMode\":false,\"scale\":{\"color\":\"#333\",\"labels\":false,\"show\":true,\"width\":2},\"style\":{\"bgColor\":true,\"bgFill\":\"white\",\"fontSize\":\"34\",\"labelColor\":false,\"subText\":\"Risk\"},\"type\":\"simple\",\"useRanges\":false,\"verticalSplit\":false},\"type\":\"gauge\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Total\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}},\"label\":\"Critical\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"colors\":{\"0 - 10000\":\"#64B0C8\"},\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "465c5820-8977-11e7-857e-e1d56b17746d",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Critical Assets",
-      "visState": "{\"title\":\"VulnWhisperer - Critical Assets\",\"type\":\"heatmap\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"enableHover\":true,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":4,\"colorSchema\":\"Green to Red\",\"setColorRange\":true,\"colorsRange\":[{\"from\":0,\"to\":3},{\"from\":3,\"to\":7},{\"from\":7,\"to\":9},{\"from\":9,\"to\":11}],\"invertColors\":false,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":false,\"rotate\":0,\"color\":\"white\"}}],\"type\":\"heatmap\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"risk_score\",\"customLabel\":\"Residual Risk Score\"}},{\"id\":\"2\",\"enabled\":false,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"risk_score\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"row\":true}},{\"id\":\"3\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Date\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"asset.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Critical Asset\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 3\":\"rgb(0,104,55)\",\"3 - 7\":\"rgb(135,203,103)\",\"7 - 9\":\"rgb(255,255,190)\",\"9 - 11\":\"rgb(249,142,82)\"},\"colors\":{\"8 - 10\":\"#BF1B00\",\"9 - 11\":\"#BF1B00\",\"7 - 9\":\"#EF843C\",\"3 - 7\":\"#EAB839\",\"0 - 3\":\"#7EB26D\"},\"legendOpen\":false}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[{\"meta\":{\"index\":\"logstash-vulnwhisperer-*\",\"negate\":false,\"disabled\":false,\"alias\":\"Critical Asset\",\"type\":\"phrase\",\"key\":\"tags\",\"value\":\"critical_asset\"},\"query\":{\"match\":{\"tags\":{\"query\":\"critical_asset\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}]}"
-      }
-    }
-  },
-  {
-    "_id": "852816e0-3eb1-11e7-90cb-918f9cb01e3d",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer-CVSS",
-      "visState": "{\"title\":\"VulnWhisperer-CVSS\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"cvss.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"CVSS Score\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"asset.keyword\",\"customLabel\":\"# of Assets\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "d048c220-80b3-11e7-8790-73b60225f736",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer  - Risk: High",
-      "visState": "{\"title\":\"VulnWhisperer  - Risk: High\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Background\",\"colorsRange\":[{\"from\":0,\"to\":1000}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"bgFill\":\"white\",\"bgColor\":true,\"labelColor\":false,\"subText\":\"\",\"fontSize\":\"34\"},\"extendRange\":true}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"High Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:high\"}}},\"label\":\"\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 10000\":\"#EF843C\",\"0 - 1000\":\"#E0752D\"}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "db55bce0-80b3-11e7-8790-73b60225f736",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer  - Risk: Critical",
-      "visState": "{\"title\":\"VulnWhisperer  - Risk: Critical\",\"type\":\"goal\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"gauge\":{\"autoExtend\":false,\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"gaugeColorMode\":\"Background\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Metric\",\"invertColors\":false,\"labels\":{\"color\":\"black\",\"show\":false},\"orientation\":\"vertical\",\"percentageMode\":false,\"scale\":{\"color\":\"#333\",\"labels\":false,\"show\":true,\"width\":2},\"style\":{\"bgColor\":true,\"bgFill\":\"white\",\"fontSize\":\"34\",\"labelColor\":false,\"subText\":\"Risk\"},\"type\":\"simple\",\"useRanges\":false,\"verticalSplit\":false},\"type\":\"gauge\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Critical Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:critical\"}}},\"label\":\"Critical\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"colors\":{\"0 - 10000\":\"#BF1B00\"},\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "56f0f5f0-3ebe-11e7-a192-93f36fbd9d05",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer-RiskOverTime",
-      "visState": "{\"title\":\"VulnWhisperer-RiskOverTime\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per 12 hours\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"orderBucketsBySum\":false,\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"mode\":\"normal\",\"show\":\"true\",\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}],\"type\":\"line\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:info\"}}},\"label\":\"Info\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:low\"}}},\"label\":\"Low\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:medium\"}}},\"label\":\"Medium\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:high\"}}},\"label\":\"High\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:critical\"}}},\"label\":\"Critical\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"colors\":{\"Critical\":\"#962D82\",\"High\":\"#BF1B00\",\"Low\":\"#629E51\",\"Medium\":\"#EAB839\",\"Info\":\"#65C5DB\"}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "c1361da0-80b3-11e7-8790-73b60225f736",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer  - Risk: Medium",
-      "visState": "{\"title\":\"VulnWhisperer  - Risk: Medium\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Background\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"bgFill\":\"white\",\"bgColor\":true,\"labelColor\":false,\"subText\":\"\",\"fontSize\":\"34\"},\"extendRange\":false},\"isDisplayWarning\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Medium Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:medium\"}}},\"label\":\"Medium Risk\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 10000\":\"#EAB839\"}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "e46ff7f0-897d-11e7-934b-67cec0a7da65",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer  - Risk: Low",
-      "visState": "{\"title\":\"VulnWhisperer  - Risk: Low\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Background\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"bgFill\":\"white\",\"bgColor\":true,\"labelColor\":false,\"subText\":\"\",\"fontSize\":\"34\"},\"extendRange\":false}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Low Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:low\"}}},\"label\":\"Low Risk\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 10000\":\"#629E51\"}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "995e2280-3df3-11e7-a44e-c79ca8efb780",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer-Asset",
-      "visState": "{\"title\":\"VulnWhisperer-Asset\",\"type\":\"table\",\"params\":{\"perPage\":15,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"asset.keyword\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Asset\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  }
-]

resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/1001_vulnWhisperer_ReportingMitigationDashboard.json

@@ -1,43 +0,0 @@
-[
-  {
-    "_id": "72051530-448e-11e7-a818-f5f80dfc3590",
-    "_type": "dashboard",
-    "_source": {
-      "title": "VulnWhisperer - Reporting",
-      "hits": 0,
-      "description": "",
-      "panelsJSON": "[{\"col\":1,\"id\":\"2f979030-44b9-11e7-a818-f5f80dfc3590\",\"panelIndex\":5,\"row\":12,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"8d9592d0-44ec-11e7-a05f-d9719b331a27\",\"panelIndex\":12,\"row\":8,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"67d432e0-44ec-11e7-a05f-d9719b331a27\",\"panelIndex\":14,\"row\":4,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":10,\"id\":\"297df800-3f7e-11e7-bd24-6903e3283192\",\"panelIndex\":15,\"row\":8,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"471a3580-3f6b-11e7-88e7-df1abe6547fb\",\"panelIndex\":20,\"row\":8,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":11,\"id\":\"995e2280-3df3-11e7-a44e-c79ca8efb780\",\"panelIndex\":22,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":9,\"id\":\"b2f2adb0-897f-11e7-a2d2-c57bca21b3aa\",\"panelIndex\":23,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"db55bce0-80b3-11e7-8790-73b60225f736\",\"panelIndex\":25,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"d048c220-80b3-11e7-8790-73b60225f736\",\"panelIndex\":26,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"e46ff7f0-897d-11e7-934b-67cec0a7da65\",\"panelIndex\":27,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":3,\"id\":\"c1361da0-80b3-11e7-8790-73b60225f736\",\"panelIndex\":28,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"479deab0-8a39-11e7-a58a-9bfcb3761a3d\",\"panelIndex\":29,\"row\":4,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"}]",
-      "optionsJSON": "{\"darkTheme\":false}",
-      "uiStateJSON": "{\"P-15\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-20\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-21\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-22\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-23\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-24\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-25\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-26\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-27\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-28\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-5\":{\"vis\":{\"legendOpen\":false}}}",
-      "version": 1,
-      "timeRestore": true,
-      "timeTo": "now",
-      "timeFrom": "now-1y",
-      "refreshInterval": {
-        "display": "Off",
-        "pause": false,
-        "value": 0
-      },
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"filter\":[{\"query\":{\"match_all\":{}}}],\"highlightAll\":true,\"version\":true}"
-      }
-    }
-  },
-  {
-    "_id": "AWCUqesWib22Ai8JwW3u",
-    "_type": "dashboard",
-    "_source": {
-      "title": "VulnWhisperer - Risk Mitigation",
-      "hits": 0,
-      "description": "",
-      "panelsJSON": "[{\"col\":11,\"id\":\"995e2280-3df3-11e7-a44e-c79ca8efb780\",\"panelIndex\":20,\"row\":8,\"size_x\":2,\"size_y\":6,\"type\":\"visualization\"},{\"col\":1,\"id\":\"852816e0-3eb1-11e7-90cb-918f9cb01e3d\",\"panelIndex\":21,\"row\":10,\"size_x\":3,\"size_y\":5,\"type\":\"visualization\"},{\"col\":4,\"id\":\"297df800-3f7e-11e7-bd24-6903e3283192\",\"panelIndex\":27,\"row\":8,\"size_x\":3,\"size_y\":5,\"type\":\"visualization\"},{\"col\":9,\"id\":\"35b6d320-3f7f-11e7-bd24-6903e3283192\",\"panelIndex\":28,\"row\":8,\"size_x\":2,\"size_y\":6,\"type\":\"visualization\"},{\"col\":11,\"id\":\"471a3580-3f6b-11e7-88e7-df1abe6547fb\",\"panelIndex\":30,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"de1a5f40-3f85-11e7-97f9-3777d794626d\",\"panelIndex\":31,\"row\":8,\"size_x\":2,\"size_y\":5,\"type\":\"visualization\"},{\"col\":10,\"id\":\"5093c620-44e9-11e7-8014-ede06a7e69f8\",\"panelIndex\":37,\"row\":4,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"columns\":[\"host\",\"risk\",\"risk_score\",\"cve\",\"plugin_name\",\"solution\",\"plugin_output\"],\"id\":\"54648700-3f74-11e7-852e-69207a3d0726\",\"panelIndex\":38,\"row\":15,\"size_x\":12,\"size_y\":6,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"fb6eb020-49ab-11e7-8f8c-57ad64ec48a6\",\"panelIndex\":39,\"row\":8,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":5,\"id\":\"465c5820-8977-11e7-857e-e1d56b17746d\",\"panelIndex\":40,\"row\":4,\"size_x\":5,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"56f0f5f0-3ebe-11e7-a192-93f36fbd9d05\",\"panelIndex\":46,\"row\":4,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"e46ff7f0-897d-11e7-934b-67cec0a7da65\",\"panelIndex\":47,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":3,\"id\":\"c1361da0-80b3-11e7-8790-73b60225f736\",\"panelIndex\":48,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"d048c220-80b3-11e7-8790-73b60225f736\",\"panelIndex\":49,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"db55bce0-80b3-11e7-8790-73b60225f736\",\"panelIndex\":50,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":9,\"id\":\"b2f2adb0-897f-11e7-a2d2-c57bca21b3aa\",\"panelIndex\":51,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"}]",
-      "optionsJSON": "{\"darkTheme\":false}",
-      "uiStateJSON": "{\"P-11\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-2\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-20\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-21\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}},\"P-27\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-28\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}},\"P-3\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"asc\"}}}},\"P-30\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-31\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-40\":{\"vis\":{\"defaultColors\":{\"0 - 3\":\"rgb(0,104,55)\",\"3 - 7\":\"rgb(135,203,103)\",\"7 - 9\":\"rgb(255,255,190)\",\"9 - 11\":\"rgb(249,142,82)\"}}},\"P-41\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-42\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-43\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"}}},\"P-44\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-45\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-46\":{\"vis\":{\"legendOpen\":true}},\"P-47\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-48\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-49\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-5\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-50\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-51\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-6\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-8\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}",
-      "version": 1,
-      "timeRestore": false,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"filter\":[{\"query\":{\"match_all\":{}}}],\"highlightAll\":true,\"version\":true}"
-      }
-    }
-  }
-]

resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/2000_vulnWhisperer_QualysVisuals (required with Dashboard).json

@@ -1,170 +0,0 @@
-[
-  {
-    "_id": "AWCUo-jRib22Ai8JwW1N",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer  - Risk: High Qualys Scoring",
-      "visState": "{\"title\":\"VulnWhisperer  - Risk: High Qualys Scoring\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Background\",\"colorsRange\":[{\"from\":0,\"to\":1000}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"bgFill\":\"white\",\"bgColor\":true,\"labelColor\":false,\"subText\":\"\",\"fontSize\":\"34\"},\"extendRange\":true}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"High Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:high\"}}},\"label\":\"\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 10000\":\"#EF843C\",\"0 - 1000\":\"#E0752D\"}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCUozGBib22Ai8JwW1B",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer  - Risk: Medium Qualys Scoring",
-      "visState": "{\"title\":\"VulnWhisperer  - Risk: Medium Qualys Scoring\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Background\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"bgFill\":\"white\",\"bgColor\":true,\"labelColor\":false,\"subText\":\"\",\"fontSize\":\"34\"},\"extendRange\":false}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Medium Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:medium\"}}},\"label\":\"Medium Risk\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 10000\":\"#EAB839\"}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCUpE3Kib22Ai8JwW1c",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer  - Risk: Critical Qualys Scoring",
-      "visState": "{\"title\":\"VulnWhisperer  - Risk: Critical Qualys Scoring\",\"type\":\"goal\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"gauge\":{\"autoExtend\":false,\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"gaugeColorMode\":\"Background\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Metric\",\"invertColors\":false,\"labels\":{\"color\":\"black\",\"show\":false},\"orientation\":\"vertical\",\"percentageMode\":false,\"scale\":{\"color\":\"#333\",\"labels\":false,\"show\":true,\"width\":2},\"style\":{\"bgColor\":true,\"bgFill\":\"white\",\"fontSize\":\"34\",\"labelColor\":false,\"subText\":\"Risk\"},\"type\":\"simple\",\"useRanges\":false,\"verticalSplit\":false},\"type\":\"gauge\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Critical Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:critical\"}}},\"label\":\"Critical\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"colors\":{\"0 - 10000\":\"#BF1B00\"},\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCUyeHGib22Ai8JwX62",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer-RiskOverTime Qualys Scoring",
-      "visState": "{\"title\":\"VulnWhisperer-RiskOverTime Qualys Scoring\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per 12 hours\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"orderBucketsBySum\":false,\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"mode\":\"normal\",\"show\":\"true\",\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}],\"type\":\"line\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:info\"}}},\"label\":\"Info\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:low\"}}},\"label\":\"Low\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:medium\"}}},\"label\":\"Medium\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:high\"}}},\"label\":\"High\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:critical\"}}},\"label\":\"Critical\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"colors\":{\"Critical\":\"#962D82\",\"High\":\"#BF1B00\",\"Low\":\"#629E51\",\"Medium\":\"#EAB839\",\"Info\":\"#65C5DB\"}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCUos-Fib22Ai8JwW0y",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer  - Risk: Low Qualys Scoring",
-      "visState": "{\"title\":\"VulnWhisperer  - Risk: Low Qualys Scoring\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Background\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"bgFill\":\"white\",\"bgColor\":true,\"labelColor\":false,\"subText\":\"\",\"fontSize\":\"34\"},\"extendRange\":false}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Low Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:low\"}}},\"label\":\"Low Risk\"}]}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 10000\":\"#629E51\"}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCg9Wsfib22Ai8Jww3v",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Qualys: Category Description",
-      "visState": "{\"title\":\"VulnWhisperer - Qualys: Category Description\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"category_description.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Category Description\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"match_all\":{}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCg88f1ib22Ai8Jww3C",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - QualysOS",
-      "visState": "{\"title\":\"VulnWhisperer - QualysOS\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"operating_system.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"match_all\":{}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCg9JUAib22Ai8Jww3Y",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - QualysOwner",
-      "visState": "{\"title\":\"VulnWhisperer - QualysOwner\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"owner.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"match_all\":{}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCg9tE6ib22Ai8Jww4R",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Qualys: Impact",
-      "visState": "{\"title\":\"VulnWhisperer - Qualys: Impact\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"impact.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Impact\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"match_all\":{}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCg9igvib22Ai8Jww36",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - Qualys: Level",
-      "visState": "{\"title\":\"VulnWhisperer - Qualys: Level\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"level.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Level\"}}],\"listeners\":{}}",
-      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"match_all\":{}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCUsp_3ib22Ai8JwW7R",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - TL-Critical Risk Qualys Scoring",
-      "visState": "{\"title\":\"VulnWhisperer - TL-Critical Risk Qualys Scoring\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*',q='(risk:critical)').label(\\\"Original\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk:critical)',offset=-1w).label(\\\"One week offset\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk:critical)').subtract(.es(index='logstash-vulnwhisperer-*',q='(risk:critical)',offset=-1w)).label(\\\"Difference\\\").lines(steps=3,fill=2,width=1)\",\"interval\":\"auto\",\"type\":\"timelion\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  },
-  {
-    "_id": "AWCUtHETib22Ai8JwW79",
-    "_type": "visualization",
-    "_source": {
-      "title": "VulnWhisperer - TL-High Risk Qualys Scoring",
-      "visState": "{\"title\":\"VulnWhisperer - TL-High Risk Qualys Scoring\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*',q='(risk:high)').label(\\\"Original\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk:high)',offset=-1w).label(\\\"One week offset\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk:high)').subtract(.es(index='logstash-vulnwhisperer-*',q='(risk:high)',offset=-1w)).label(\\\"Difference\\\").lines(steps=3,fill=2,width=1)\",\"interval\":\"auto\",\"type\":\"timelion\"},\"aggs\":[],\"listeners\":{}}",
-      "uiStateJSON": "{}",
-      "description": "",
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
-      }
-    }
-  }
-]

resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/2001_vulnWhisperer_ReportingMitigationDashboardQualysRisk.json

@@ -1,50 +0,0 @@
-[
-  {
-    "_id": "AWCUrIBqib22Ai8JwW43",
-    "_type": "dashboard",
-    "_source": {
-      "title": "VulnWhisperer - Reporting Qualys Scoring",
-      "hits": 0,
-      "description": "",
-      "panelsJSON": "[{\"col\":1,\"id\":\"2f979030-44b9-11e7-a818-f5f80dfc3590\",\"panelIndex\":5,\"row\":11,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":10,\"id\":\"297df800-3f7e-11e7-bd24-6903e3283192\",\"panelIndex\":15,\"row\":7,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"471a3580-3f6b-11e7-88e7-df1abe6547fb\",\"panelIndex\":20,\"row\":7,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":11,\"id\":\"995e2280-3df3-11e7-a44e-c79ca8efb780\",\"panelIndex\":22,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":9,\"id\":\"b2f2adb0-897f-11e7-a2d2-c57bca21b3aa\",\"panelIndex\":23,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"479deab0-8a39-11e7-a58a-9bfcb3761a3d\",\"panelIndex\":29,\"row\":4,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"size_x\":6,\"size_y\":3,\"panelIndex\":30,\"type\":\"visualization\",\"id\":\"AWCUtHETib22Ai8JwW79\",\"col\":1,\"row\":8},{\"size_x\":6,\"size_y\":3,\"panelIndex\":31,\"type\":\"visualization\",\"id\":\"AWCUsp_3ib22Ai8JwW7R\",\"col\":7,\"row\":4},{\"size_x\":2,\"size_y\":3,\"panelIndex\":33,\"type\":\"visualization\",\"id\":\"AWCUozGBib22Ai8JwW1B\",\"col\":3,\"row\":1},{\"size_x\":2,\"size_y\":3,\"panelIndex\":34,\"type\":\"visualization\",\"id\":\"AWCUo-jRib22Ai8JwW1N\",\"col\":5,\"row\":1},{\"size_x\":2,\"size_y\":3,\"panelIndex\":35,\"type\":\"visualization\",\"id\":\"AWCUpE3Kib22Ai8JwW1c\",\"col\":7,\"row\":1},{\"size_x\":2,\"size_y\":3,\"panelIndex\":36,\"type\":\"visualization\",\"id\":\"AWCUos-Fib22Ai8JwW0y\",\"col\":1,\"row\":1}]",
-      "optionsJSON": "{\"darkTheme\":false}",
-      "uiStateJSON": "{\"P-15\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-20\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-21\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-22\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-23\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-24\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-5\":{\"vis\":{\"legendOpen\":false}},\"P-33\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-34\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-35\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-27\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-28\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-26\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"}}},\"P-25\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-32\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-36\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}}}",
-      "version": 1,
-      "timeRestore": true,
-      "timeTo": "now",
-      "timeFrom": "now-30d",
-      "refreshInterval": {
-        "display": "Off",
-        "pause": false,
-        "value": 0
-      },
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"-vulnerability_category:\\\"INFORMATION_GATHERED\\\"\"}}}],\"highlightAll\":true,\"version\":true}"
-      }
-    }
-  },
-  {
-    "_id": "5dba30c0-3df3-11e7-a44e-c79ca8efb780",
-    "_type": "dashboard",
-    "_source": {
-      "title": "VulnWhisperer - Risk Mitigation Qualys Web Scoring",
-      "hits": 0,
-      "description": "",
-      "panelsJSON": "[{\"col\":11,\"id\":\"995e2280-3df3-11e7-a44e-c79ca8efb780\",\"panelIndex\":20,\"row\":8,\"size_x\":2,\"size_y\":7,\"type\":\"visualization\"},{\"col\":1,\"id\":\"852816e0-3eb1-11e7-90cb-918f9cb01e3d\",\"panelIndex\":21,\"row\":10,\"size_x\":3,\"size_y\":5,\"type\":\"visualization\"},{\"col\":4,\"id\":\"297df800-3f7e-11e7-bd24-6903e3283192\",\"panelIndex\":27,\"row\":8,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":9,\"id\":\"35b6d320-3f7f-11e7-bd24-6903e3283192\",\"panelIndex\":28,\"row\":8,\"size_x\":2,\"size_y\":7,\"type\":\"visualization\"},{\"col\":11,\"id\":\"471a3580-3f6b-11e7-88e7-df1abe6547fb\",\"panelIndex\":30,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"de1a5f40-3f85-11e7-97f9-3777d794626d\",\"panelIndex\":31,\"row\":8,\"size_x\":2,\"size_y\":4,\"type\":\"visualization\"},{\"col\":10,\"id\":\"5093c620-44e9-11e7-8014-ede06a7e69f8\",\"panelIndex\":37,\"row\":4,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"columns\":[\"host\",\"risk\",\"risk_score\",\"cve\",\"plugin_name\",\"solution\",\"plugin_output\"],\"id\":\"54648700-3f74-11e7-852e-69207a3d0726\",\"panelIndex\":38,\"row\":15,\"size_x\":12,\"size_y\":6,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"fb6eb020-49ab-11e7-8f8c-57ad64ec48a6\",\"panelIndex\":39,\"row\":8,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":5,\"id\":\"465c5820-8977-11e7-857e-e1d56b17746d\",\"panelIndex\":40,\"row\":4,\"size_x\":5,\"size_y\":4,\"type\":\"visualization\"},{\"col\":9,\"id\":\"b2f2adb0-897f-11e7-a2d2-c57bca21b3aa\",\"panelIndex\":45,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AWCUos-Fib22Ai8JwW0y\",\"panelIndex\":47,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":3,\"id\":\"AWCUozGBib22Ai8JwW1B\",\"panelIndex\":48,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"AWCUo-jRib22Ai8JwW1N\",\"panelIndex\":49,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"AWCUpE3Kib22Ai8JwW1c\",\"panelIndex\":50,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AWCUyeHGib22Ai8JwX62\",\"panelIndex\":51,\"row\":4,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":4,\"id\":\"AWCg88f1ib22Ai8Jww3C\",\"panelIndex\":52,\"row\":12,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"AWCg9JUAib22Ai8Jww3Y\",\"panelIndex\":53,\"row\":12,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"}]",
-      "optionsJSON": "{\"darkTheme\":false}",
-      "uiStateJSON": "{\"P-11\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-2\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-20\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-21\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}},\"P-27\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-28\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}},\"P-3\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"asc\"}}}},\"P-30\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-31\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-40\":{\"vis\":{\"defaultColors\":{\"0 - 3\":\"rgb(0,104,55)\",\"3 - 7\":\"rgb(135,203,103)\",\"7 - 9\":\"rgb(255,255,190)\",\"9 - 11\":\"rgb(249,142,82)\"}}},\"P-41\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-42\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-43\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"}}},\"P-44\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-45\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-47\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-48\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-49\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-5\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-50\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-6\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-8\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-52\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-53\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}",
-      "version": 1,
-      "timeRestore": true,
-      "timeTo": "now",
-      "timeFrom": "now-30d",
-      "refreshInterval": {
-        "display": "Off",
-        "pause": false,
-        "value": 0
-      },
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"-vulnerability_category:\\\"INFORMATION_GATHERED\\\"\"}}}],\"highlightAll\":true,\"version\":true}"
-      }
-    }
-  }
-]

resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/9000_vulnWhisperer_SavedSearch.json

@@ -1,28 +0,0 @@
-[
-  {
-    "_id": "54648700-3f74-11e7-852e-69207a3d0726",
-    "_type": "search",
-    "_source": {
-      "title": "VulnWhisperer - Saved Search",
-      "description": "",
-      "hits": 0,
-      "columns": [
-        "host",
-        "risk",
-        "risk_score",
-        "cve",
-        "plugin_name",
-        "solution",
-        "plugin_output"
-      ],
-      "sort": [
-        "@timestamp",
-        "desc"
-      ],
-      "version": 1,
-      "kibanaSavedObjectMeta": {
-        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}"
-      }
-    }
-  }
-]

resources/elk5-old_compatibility/logstash/0001_input_beats.conf

@@ -1,14 +0,0 @@
-input {
-  beats {
-    port => 5044
-    tags => "beats"
-  }
-}
-
-filter {
-  if [beat][hostname] == "filebeathost" {
-    mutate {
-      add_tag => ["nessus"]
-    }
-  }
-}

resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf

@@ -1,220 +0,0 @@
-# Author: Austin Taylor and Justin Henderson
-# Email: email@austintaylor.io
-# Last Update: 12/20/2017
-# Version 0.3
-# Description: Take in nessus reports from vulnWhisperer and pumps into logstash
-
-
-input {
-  file {
-    path => "/opt/VulnWhisperer/nessus/**/*"
-    start_position => "beginning"
-    tags => "nessus"
-    type => "nessus"
-  }
-  file {
-    path => "/opt/VulnWhisperer/tenable/*.csv"
-    start_position => "beginning"
-    tags => "tenable"
-    type => "tenable"
-  }
-}
-
-filter {
-  if "nessus" in [tags] or "tenable" in [tags] {
-    # Drop the header column
-    if [message] =~ "^Plugin ID" { drop {} }
-
-    csv {
-      # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"]
-      columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"]
-      separator => ","
-      source => "message"
-    }
-
-    ruby {
-        code => "if event.get('description')
-                    event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('synopsis')
-                    event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('solution')
-                    event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('see_also')
-                    event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('plugin_output')
-                    event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end"
-    }
-
-    #If using filebeats as your source, you will need to replace the "path" field to "source"
-    grok {
-      match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.csv$" }
-      tag_on_failure => []
-    }
-
-    date {
-      match => [ "last_updated", "UNIX" ]
-      target => "@timestamp"
-      remove_field => ["last_updated"]
-    }
-
-    if [risk] == "None" {
-      mutate { add_field => { "risk_number" => 0 }}
-    }
-    if [risk] == "Low" {
-      mutate { add_field => { "risk_number" => 1 }}
-    }
-    if [risk] == "Medium" {
-      mutate { add_field => { "risk_number" => 2 }}
-    }
-    if [risk] == "High" {
-      mutate { add_field => { "risk_number" => 3 }}
-    }
-    if [risk] == "Critical" {
-      mutate { add_field => { "risk_number" => 4 }}
-    }
-
-    if ![cve] or [cve] == "nan" {
-      mutate { remove_field => [ "cve" ] }
-    }
-    if ![cvss] or [cvss] == "nan" {
-      mutate { remove_field => [ "cvss" ] }
-    }
-    if ![cvss_base] or [cvss_base] == "nan" {
-      mutate { remove_field => [ "cvss_base" ] }
-    }
-    if ![cvss_temporal] or [cvss_temporal] == "nan" {
-      mutate { remove_field => [ "cvss_temporal" ] }
-    }
-    if ![cvss_temporal_vector] or [cvss_temporal_vector] == "nan" {
-      mutate { remove_field => [ "cvss_temporal_vector" ] }
-    }
-    if ![cvss_vector] or [cvss_vector] == "nan" {
-      mutate { remove_field => [ "cvss_vector" ] }
-    }
-    if ![cvss3_base] or [cvss3_base] == "nan" {
-      mutate { remove_field => [ "cvss3_base" ] }
-    }
-    if ![cvss3_temporal] or [cvss3_temporal] == "nan" {
-      mutate { remove_field => [ "cvss3_temporal" ] }
-    }
-    if ![cvss3_temporal_vector] or [cvss3_temporal_vector] == "nan" {
-      mutate { remove_field => [ "cvss3_temporal_vector" ] }
-    }
-    if ![description] or [description] == "nan" {
-      mutate { remove_field => [ "description" ] }
-    }
-    if ![mac_address] or [mac_address] == "nan" {
-      mutate { remove_field => [ "mac_address" ] }
-    }
-    if ![netbios] or [netbios] == "nan" {
-      mutate { remove_field => [ "netbios" ] }
-    }
-    if ![operating_system] or [operating_system] == "nan" {
-      mutate { remove_field => [ "operating_system" ] }
-    }
-    if ![plugin_output] or [plugin_output] == "nan" {
-      mutate { remove_field => [ "plugin_output" ] }
-    }
-    if ![see_also] or [see_also] == "nan" {
-      mutate { remove_field => [ "see_also" ] }
-    }
-    if ![synopsis] or [synopsis] == "nan" {
-      mutate { remove_field => [ "synopsis" ] }
-    }
-    if ![system_type] or [system_type] == "nan" {
-      mutate { remove_field => [ "system_type" ] }
-    }
-
-    mutate {
-      remove_field => [ "message" ]
-      add_field => { "risk_score" => "%{cvss}" }
-    }
-    mutate {
-      convert => { "risk_score" => "float" }
-    }
-    if [risk_score] == 0 {
-      mutate {
-        add_field => { "risk_score_name" => "info" }
-      }
-    }
-    if [risk_score] > 0 and [risk_score] < 3 {
-      mutate {
-        add_field => { "risk_score_name" => "low" }
-      }
-    }
-    if [risk_score] >= 3 and [risk_score] < 6 {
-      mutate {
-        add_field => { "risk_score_name" => "medium" }
-      }
-    }
-    if [risk_score] >=6 and [risk_score] < 9 {
-      mutate {
-        add_field => { "risk_score_name" => "high" }
-      }
-    }
-    if [risk_score] >= 9 {
-      mutate {
-        add_field => { "risk_score_name" => "critical" }
-      }
-    }
-
-    # Compensating controls - adjust risk_score
-    # Adobe and Java are not allowed to run in browser unless whitelisted
-    # Therefore, lower score by dividing by 3 (score is subjective to risk)
-
-    #Modify and uncomment when ready to use
-    #if [risk_score] != 0 {
-    #  if [plugin_name] =~ "Adobe" and [risk_score] > 6 or [plugin_name] =~ "Java" and [risk_score] > 6 {
-    #    ruby {
-    #      code => "event.set('risk_score', event.get('risk_score') / 3)"
-    #    }
-    #    mutate {
-    #      add_field => { "compensating_control" => "Adobe and Flash removed from browsers unless whitelisted site." }
-    #    }
-    #  }
-    #}
-
-    # Add tags for reporting based on assets or criticality
-
-    if [asset] == "dc01" or [asset] == "dc02" or [asset] == "pki01" or [asset] == "192.168.0.54" or [asset] =~ "^192\.168\.0\." or [asset] =~ "^42.42.42." {
-      mutate {
-        add_tag => [ "critical_asset" ]
-      }
-    }
-    #if [asset] =~ "^192\.168\.[45][0-9][0-9]\.1$" or [asset] =~ "^192.168\.[50]\.[0-9]{1,2}\.1$"{
-    #  mutate {
-    #    add_tag => [ "has_hipaa_data" ]
-    #  }
-    #}
-    #if [asset] =~ "^192\.168\.[45][0-9][0-9]\." {
-    #  mutate {
-    #    add_tag => [ "hipaa_asset" ]
-    #  }
-    #}
-    if [asset] =~ "^hr" {
-      mutate {
-        add_tag => [ "pci_asset" ]
-      }
-    }
-    #if [asset] =~ "^10\.0\.50\." {
-    #  mutate {
-    #    add_tag => [ "web_servers" ]
-    #  }
-    #}
-  }
-}
-
-output {
-  if "nessus" in [tags] or "tenable" in [tags] or [type] in [ "nessus", "tenable" ] {
-    # stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => [ "localhost:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
-    }
-  }
-}

resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf

@@ -1,153 +0,0 @@
-# Author: Austin Taylor and Justin Henderson
-# Email: austin@hasecuritysolutions.com
-# Last Update: 12/30/2017
-# Version 0.3
-# Description: Take in qualys web scan reports from vulnWhisperer and pumps into logstash
-
-input {
-  file {
-    path => [ "/opt/VulnWhisperer/data/qualys/*.json" , "/opt/VulnWhisperer/data/qualys_web/*.json", "/opt/VulnWhisperer/data/qualys_vuln/*.json" ]
-    type => json
-    codec => json
-    start_position => "beginning"
-    tags => [ "qualys" ]
-  }
-}
-
-filter {
-  if "qualys" in [tags] {
-    grok {
-      match => { "path" => [ "(?<tags>qualys_vuln)_scan_%{DATA}_%{INT:last_updated}.json$", "(?<tags>qualys_web)_%{INT:app_id}_%{INT:last_updated}.json$" ] }
-      tag_on_failure => []
-    }
-    
-    mutate {
-      replace => [ "message", "%{message}" ]
-      #gsub => [
-      #  "message", "\|\|\|", " ",
-      #  "message", "\t\t", " ",
-      #  "message", "    ", " ",
-      #  "message", "   ", " ",
-      #  "message", "  ", " ",
-      #  "message", "nan", " ",
-      #  "message",'\n',''
-      #]
-    }
-
-    if "qualys_web" in [tags] {
-    mutate {
-      add_field => { "asset" => "%{web_application_name}" }
-      add_field => { "risk_score" => "%{cvss}" }
-      }
-    } else if "qualys_vuln" in [tags] {
-      mutate {
-        add_field => { "asset" => "%{ip}" }
-        add_field => { "risk_score" => "%{cvss}" }
-      }
-    }
-
-    if [risk] == "1" {
-      mutate { add_field => { "risk_number" => 0 }}
-      mutate { replace => { "risk" => "info" }}
-    }
-    if [risk] == "2" {
-      mutate { add_field => { "risk_number" => 1 }}
-      mutate { replace => { "risk" => "low" }}
-    }
-    if [risk] == "3" {
-      mutate { add_field => { "risk_number" => 2 }}
-      mutate { replace => { "risk" => "medium" }}
-    }
-    if [risk] == "4" {
-      mutate { add_field => { "risk_number" => 3 }}
-      mutate { replace => { "risk" => "high" }}
-    }
-    if [risk] == "5" {
-      mutate { add_field => { "risk_number" => 4 }}
-      mutate { replace => { "risk" => "critical" }}
-    }
-
-    mutate {
-      remove_field => "message"
-    }
-
-    if [first_time_detected] {
-      date {
-        match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "first_time_detected"
-      }
-    }
-    if [first_time_tested] {
-      date {
-        match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "first_time_tested"
-      }
-    }
-    if [last_time_detected] {
-      date {
-        match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "last_time_detected"
-      }
-    }
-    if [last_time_tested] {
-      date {
-        match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "last_time_tested"
-      }
-    }
-    date {
-      match => [ "last_updated", "UNIX" ]
-      target => "@timestamp"
-      remove_field => "last_updated"
-    }
-    mutate {
-      convert => { "plugin_id" => "integer"}
-      convert => { "id" => "integer"}
-      convert => { "risk_number" => "integer"}
-      convert => { "risk_score" => "float"}
-      convert => { "total_times_detected" => "integer"}
-      convert => { "cvss_temporal" => "float"}
-      convert => { "cvss" => "float"}
-    }
-    if [risk_score] == 0 {
-      mutate {
-        add_field => { "risk_score_name" => "info" }
-      }
-    }
-    if [risk_score] > 0 and [risk_score] < 3 {
-      mutate {
-        add_field => { "risk_score_name" => "low" }
-      }
-    }
-    if [risk_score] >= 3 and [risk_score] < 6 {
-      mutate {
-        add_field => { "risk_score_name" => "medium" }
-      }
-    }
-    if [risk_score] >=6 and [risk_score] < 9 {
-      mutate {
-        add_field => { "risk_score_name" => "high" }
-      }
-    }
-    if [risk_score] >= 9 {
-      mutate {
-        add_field => { "risk_score_name" => "critical" }
-      }
-    }
-
-    if [asset] =~ "\.yourdomain\.(com|net)$" {
-      mutate {
-        add_tag => [ "critical_asset" ]
-      }
-    }
-  }
-}
-output {
-  if "qualys" in [tags] {
-    stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => [ "localhost:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
-    }
-  }
-}

resources/elk5-old_compatibility/logstash/3000_openvas.conf

@@ -1,146 +0,0 @@
-# Author: Austin Taylor and Justin Henderson
-# Email: austin@hasecuritysolutions.com
-# Last Update: 03/04/2018
-# Version 0.3
-# Description: Take in qualys web scan reports from vulnWhisperer and pumps into logstash
-
-input {
-  file {
-    path => "/opt/VulnWhisperer/openvas/*.json"
-    type => json
-    codec => json
-    start_position => "beginning"
-    tags => [ "openvas_scan", "openvas" ]
-  }
-}
-
-filter {
-  if "openvas_scan" in [tags] {
-    mutate {
-      replace => [ "message", "%{message}" ]
-      gsub => [
-        "message", "\|\|\|", " ",
-        "message", "\t\t", " ",
-        "message", "    ", " ",
-        "message", "   ", " ",
-        "message", "  ", " ",
-        "message", "nan", " ",
-        "message",'\n',''
-      ]
-    }
-
-
-    grok {
-        match => { "path" => "openvas_scan_%{DATA:scan_id}_%{INT:last_updated}.json$" }
-        tag_on_failure => []
-    }
-
-    mutate {
-      add_field => { "risk_score" => "%{cvss}" }
-    }
-
-    if [risk] == "1" {
-      mutate { add_field => { "risk_number" => 0 }}
-      mutate { replace => { "risk" => "info" }}
-    }
-    if [risk] == "2" {
-      mutate { add_field => { "risk_number" => 1 }}
-      mutate { replace => { "risk" => "low" }}
-    }
-    if [risk] == "3" {
-      mutate { add_field => { "risk_number" => 2 }}
-      mutate { replace => { "risk" => "medium" }}
-    }
-    if [risk] == "4" {
-      mutate { add_field => { "risk_number" => 3 }}
-      mutate { replace => { "risk" => "high" }}
-    }
-    if [risk] == "5" {
-      mutate { add_field => { "risk_number" => 4 }}
-      mutate { replace => { "risk" => "critical" }}
-    }
-
-    mutate {
-      remove_field => "message"
-    }
-
-    if [first_time_detected] {
-      date {
-        match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "first_time_detected"
-      }
-    }
-    if [first_time_tested] {
-      date {
-        match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "first_time_tested"
-      }
-    }
-    if [last_time_detected] {
-      date {
-        match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "last_time_detected"
-      }
-    }
-    if [last_time_tested] {
-      date {
-        match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "last_time_tested"
-      }
-    }
-    date {
-      match => [ "last_updated", "UNIX" ]
-      target => "@timestamp"
-      remove_field => "last_updated"
-    }
-    mutate {
-      convert => { "plugin_id" => "integer"}
-      convert => { "id" => "integer"}
-      convert => { "risk_number" => "integer"}
-      convert => { "risk_score" => "float"}
-      convert => { "total_times_detected" => "integer"}
-      convert => { "cvss_temporal" => "float"}
-      convert => { "cvss" => "float"}
-    }
-    if [risk_score] == 0 {
-      mutate {
-        add_field => { "risk_score_name" => "info" }
-      }
-    }
-    if [risk_score] > 0 and [risk_score] < 3 {
-      mutate {
-        add_field => { "risk_score_name" => "low" }
-      }
-    }
-    if [risk_score] >= 3 and [risk_score] < 6 {
-      mutate {
-        add_field => { "risk_score_name" => "medium" }
-      }
-    }
-    if [risk_score] >=6 and [risk_score] < 9 {
-      mutate {
-        add_field => { "risk_score_name" => "high" }
-      }
-    }
-    if [risk_score] >= 9 {
-      mutate {
-        add_field => { "risk_score_name" => "critical" }
-      }
-    }
-    # Add your critical assets by subnet or by hostname. Comment this field out if you don't want to tag any, but the asset panel will break.
-    if [asset] =~ "^10\.0\.100\." {
-      mutate {
-        add_tag => [ "critical_asset" ]
-      }
-    }
-  }
-}
-output {
-  if "openvas" in [tags] {
-    stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => [ "localhost:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
-    }
-  }
-}

resources/elk5-old_compatibility/logstash/4000_jira.conf

@@ -1,21 +0,0 @@
-# Description: Take in jira tickets from vulnWhisperer and pumps into logstash
-
-input {
-  file {
-    path => "/opt/VulnWhisperer/jira/*.json"
-    type => json
-    codec => json
-    start_position => "beginning"
-    tags => [ "jira" ]
-  }
-}
-
-output {
-  if "jira" in [tags] {
-    stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => [ "localhost:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
-    }
-  }
-}

resources/elk5-old_compatibility/logstash/9998_input_broker_rabbitmq.conf

@@ -1,13 +0,0 @@
-input {
-  rabbitmq {
-    key => "nessus"
-    queue => "nessus"
-    durable => true
-    exchange => "nessus"
-    user => "logstash"    
-    password => "yourpassword"
-    host => "buffer01"
-    port => 5672
-    tags => [ "queue_nessus", "rabbitmq" ]
-  }
-}

resources/elk5-old_compatibility/logstash/9998_output_broker_rabbitmq.conf

@@ -1,16 +0,0 @@
-output {
-  if "nessus" in [tags]{
-    rabbitmq {
-      key => "nessus"
-      exchange => "nessus"
-      exchange_type => "direct"
-      user => "logstash"
-      password => "yourbufferpassword"
-      host => "buffer01"
-      port => 5672
-      durable => true
-      persistent => true
-    }
-  }
-
-}

resources/elk6/get-kibana-objects.py

@@ -0,0 +1,15 @@
+import json
+import requests
+
+api_objects = []
+
+for object_type in ['dashboard', 'visualization', 'search', 'index-pattern', 'timelion-sheet']:
+    r = requests.get('http://localhost:5601/api/saved_objects/_find?per_page=500&type={}'.format(object_type)).json()
+    api_objects += r['saved_objects']
+    print object_type, len(r['saved_objects'])
+    print len(api_objects)
+
+for api_object in api_objects:
+    api_object.pop('updated_at', None)
+
+json.dump(sorted(api_objects, key=lambda x:x['id']), open('kibana_APIonly.json', 'w'), indent=2)

resources/elk6/logstash-vulnwhisperer-template.json

@@ -1,5 +1,6 @@
 {
   "index_patterns": "logstash-vulnwhisperer-*",
+  "version": 2019041701,
   "mappings": {
     "doc": {
       "properties": {
@@ -22,34 +23,34 @@
         "asset_uuid": {
           "type": "keyword"
         },
-        "assign_ip": {
-          "type": "ip"
-        },
         "category": {
           "type": "keyword"
         },
         "cve": {
           "type": "keyword"
         },
-        "cvss_base": {
-          "type": "float"
-        },
-        "cvss_temporal_vector": {
-          "type": "keyword"
-        },
-        "cvss_temporal": {
-          "type": "float"
-        },
-        "cvss_vector": {
-          "type": "keyword"
-        },
         "cvss": {
           "type": "float"
         },
+        "cvss_severity": {
+          "type": "keyword"
+        },
+        "cvss2_base": {
+          "type": "float"
+        },
+        "cvss2_severity": {
+          "type": "keyword"
+        },
+        "cvss2_temporal": {
+          "type": "float"
+        },
+        "cvss2_vector": {
+          "type": "keyword"
+        },
         "cvss3_base": {
           "type": "float"
         },
-        "cvss3_temporal_vector": {
+        "cvss3_severity": {
           "type": "keyword"
         },
         "cvss3_temporal": {
@@ -117,24 +118,14 @@
         "host_start": {
           "type": "date"
         },
-        "impact": {
-          "fields": {
-            "keyword": {
-              "ignore_above": 256,
-              "type": "keyword"
-            }
-          },
-          "norms": false,
-          "type": "text"
-        },
-        "ip_status": {
-          "type": "keyword"
-        },
         "ip": {
           "type": "ip"
         },
-        "last_updated": {
-          "type": "date"
+        "mac_address": {
+          "type": "keyword"
+        },
+        "netbios": {
+          "type": "keyword"
         },
         "operating_system": {
           "type": "keyword"
@@ -148,10 +139,10 @@
         "plugin_family": {
           "type": "keyword"
         },
-        "plugin_id": {
+        "signature_id": {
           "type": "keyword"
         },
-        "plugin_name": {
+        "signature": {
           "type": "keyword"
         },
         "plugin_output": {
@@ -170,18 +161,9 @@
         "protocol": {
           "type": "keyword"
         },
-        "results": {
-          "type": "text"
-        },
         "risk_number": {
           "type": "integer"
         },
-        "risk_score_name": {
-          "type": "keyword"
-        },
-        "risk_score": {
-          "type": "float"
-        },
         "risk": {
           "type": "keyword"
         },
@@ -189,43 +171,41 @@
           "type": "keyword"
         },
         "scan_name": {
+          "fields": {
+            "keyword": {
+              "ignore_above": 256,
+              "type": "keyword"
+            }
+          },
+          "norms": false,
+          "type": "text"
+        },
+        "scan_source": {
           "type": "keyword"
         },
-        "scan_reference": {
-          "type": "keyword"
-        },
-        "see_also": {
+        "severity": {
           "type": "keyword"
         },
         "solution": {
           "type": "keyword"
         },
-        "source": {
+        "ssl": {
           "type": "keyword"
         },
-        "ssl": {
+        "state": {
           "type": "keyword"
         },
         "synopsis": {
           "type": "keyword"
         },
-        "system_type": {
-          "type": "keyword"
-        },
         "tags": {
           "type": "keyword"
         },
-        "threat": {
-          "type": "text"
-        },
         "type": {
           "type": "keyword"
         },
         "vendor_reference": {
           "type": "keyword"
-        },
-        "vulnerability_state": {
-          "type": "keyword"
         }
       }
     }

resources/elk6/pipeline/1000_combined.conf

@@ -0,0 +1,110 @@
+input {
+  file {
+    codec => json
+    mode => "read"
+    path => ["/opt/VulnWhisperer/data/nessus/**/*.json", "/opt/VulnWhisperer/data/openvas/*.json", "/opt/VulnWhisperer/data/qualys_vm/*.json", "/opt/VulnWhisperer/data/qualys_was/*.json", "/opt/VulnWhisperer/data/tenable/*.json"]
+    start_position => "beginning"
+    file_completed_action => "delete"
+    file_chunk_size => 262144
+  }
+  file {
+    codec => json
+    mode => "read"
+    path => "/opt/VulnWhisperer/data/jira/*.json"
+    tags => [ "jira" ]
+    start_position => "beginning"
+    file_completed_action => "delete"
+  }
+}
+
+filter {
+  if [scan_source] in ["nessus", "tenable", "qualys_vm", "qualys_was", "openvas"] {
+
+    # Parse the date/time from scan_time
+    date {
+      match => [ "scan_time", "UNIX" ]
+      target => "@timestamp"
+      remove_field => ["scan_time"]
+    }
+
+    # Add scan_source to tags
+    mutate {
+      add_field => { "[tags]" => "%{scan_source}" }
+    }
+
+    # Create a unique document_id if _unique field exists
+    if [_unique] {
+      # Set document ID from _unique
+      mutate {
+        rename => { "_unique" => "[@metadata][id]" }
+      }
+    }
+
+    # Do we need this?
+    mutate {
+      convert => { "cvss" => "float"}
+      convert => { "cvss2" => "float"}
+      convert => { "cvss2_base" => "float"}
+      convert => { "cvss2_temporal" => "float"}
+      convert => { "cvss3" => "float"}
+      convert => { "cvss3_base" => "float"}
+      convert => { "cvss3_temporal" => "float"}
+      convert => { "risk_number" => "integer"}
+      convert => { "total_times_detected" => "integer"}
+    }
+  }
+
+  if [scan_source] == "qualys_was" {
+    if [first_time_detected] {
+      date {
+        match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "first_time_detected"
+      }
+    }
+    if [first_time_tested] {
+      date {
+        match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "first_time_tested"
+      }
+    }
+    if [last_time_detected] {
+      date {
+        match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "last_time_detected"
+      }
+    }
+    if [last_time_tested] {
+      date {
+        match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "last_time_tested"
+      }
+    }
+  }
+}
+
+output {
+  if [scan_source] in ["nessus", "tenable", "qualys_vm", "qualys_was", "openvas"] {
+    if [@metadata][id] {
+      elasticsearch {
+        hosts => [ "elasticsearch:9200" ]
+        index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+        document_id => "%{[@metadata][id]}"
+        manage_template => false
+      }
+    } else {
+      elasticsearch {
+        hosts => [ "elasticsearch:9200" ]
+        index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+        manage_template => false
+      }
+    }
+  }
+  # Should these go to the same index?
+  if "jira" in [tags] {
+    stdout { codec => rubydebug }
+    elasticsearch {
+      hosts => [ "elasticsearch:9200" ]
+      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+    }
+  }
+}

resources/elk6/pipeline/1000_nessus_process_file.conf

@@ -1,182 +0,0 @@
-# Author: Austin Taylor and Justin Henderson
-# Email: email@austintaylor.io
-# Last Update: 12/20/2017
-# Version 0.3
-# Description: Take in nessus reports from vulnWhisperer and pumps into logstash
-
-
-input {
-  file {
-    path => "/opt/VulnWhisperer/data/nessus/**/*"
-    mode => "read"
-    start_position => "beginning"
-    file_completed_action => "delete"
-    tags => "nessus"
-  }
-  file {
-    path => "/opt/VulnWhisperer/data/tenable/*.csv"
-    mode => "read"
-    start_position => "beginning"
-    file_completed_action => "delete"
-    tags => "tenable"
-  }
-}
-
-filter {
-  if "nessus" in [tags] or "tenable" in [tags] {
-    # Drop the header column
-    if [message] =~ "^Plugin ID" { drop {} }
-
-    csv {
-      # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"]
-      columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"]
-      separator => ","
-      source => "message"
-    }
-
-    ruby {
-        code => "if event.get('description')
-                    event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('synopsis')
-                    event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('solution')
-                    event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('see_also')
-                    event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end
-                 if event.get('plugin_output')
-                    event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
-                end"
-    }
-
-    #If using filebeats as your source, you will need to replace the "path" field to "source"
-    # Remove when scan name is included in event (current method is error prone)
-    grok {
-      match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.csv$" }
-      tag_on_failure => []
-    }
-
-    # TODO remove when @timestamp is included in event
-    date {
-      match => [ "last_updated", "UNIX" ]
-      target => "@timestamp"
-      remove_field => ["last_updated"]
-    }
-
-    if [risk] == "None" {
-      mutate { add_field => { "risk_number" => 0 }}
-    }
-    if [risk] == "Low" {
-      mutate { add_field => { "risk_number" => 1 }}
-    }
-    if [risk] == "Medium" {
-      mutate { add_field => { "risk_number" => 2 }}
-    }
-    if [risk] == "High" {
-      mutate { add_field => { "risk_number" => 3 }}
-    }
-    if [risk] == "Critical" {
-      mutate { add_field => { "risk_number" => 4 }}
-    }
-
-    if ![cve] or [cve] == "nan" {
-      mutate { remove_field => [ "cve" ] }
-    }
-    if ![cvss] or [cvss] == "nan" {
-      mutate { remove_field => [ "cvss" ] }
-    }
-    if ![cvss_base] or [cvss_base] == "nan" {
-      mutate { remove_field => [ "cvss_base" ] }
-    }
-    if ![cvss_temporal] or [cvss_temporal] == "nan" {
-      mutate { remove_field => [ "cvss_temporal" ] }
-    }
-    if ![cvss_temporal_vector] or [cvss_temporal_vector] == "nan" {
-      mutate { remove_field => [ "cvss_temporal_vector" ] }
-    }
-    if ![cvss_vector] or [cvss_vector] == "nan" {
-      mutate { remove_field => [ "cvss_vector" ] }
-    }
-    if ![cvss3_base] or [cvss3_base] == "nan" {
-      mutate { remove_field => [ "cvss3_base" ] }
-    }
-    if ![cvss3_temporal] or [cvss3_temporal] == "nan" {
-      mutate { remove_field => [ "cvss3_temporal" ] }
-    }
-    if ![cvss3_temporal_vector] or [cvss3_temporal_vector] == "nan" {
-      mutate { remove_field => [ "cvss3_temporal_vector" ] }
-    }
-    if ![description] or [description] == "nan" {
-      mutate { remove_field => [ "description" ] }
-    }
-    if ![mac_address] or [mac_address] == "nan" {
-      mutate { remove_field => [ "mac_address" ] }
-    }
-    if ![netbios] or [netbios] == "nan" {
-      mutate { remove_field => [ "netbios" ] }
-    }
-    if ![operating_system] or [operating_system] == "nan" {
-      mutate { remove_field => [ "operating_system" ] }
-    }
-    if ![plugin_output] or [plugin_output] == "nan" {
-      mutate { remove_field => [ "plugin_output" ] }
-    }
-    if ![see_also] or [see_also] == "nan" {
-      mutate { remove_field => [ "see_also" ] }
-    }
-    if ![synopsis] or [synopsis] == "nan" {
-      mutate { remove_field => [ "synopsis" ] }
-    }
-    if ![system_type] or [system_type] == "nan" {
-      mutate { remove_field => [ "system_type" ] }
-    }
-
-    mutate {
-      remove_field => [ "message" ]
-      add_field => { "risk_score" => "%{cvss}" }
-    }
-    mutate {
-      convert => { "risk_score" => "float" }
-    }
-    if [risk_score] == 0 {
-      mutate {
-        add_field => { "risk_score_name" => "info" }
-      }
-    }
-    if [risk_score] > 0 and [risk_score] < 3 {
-      mutate {
-        add_field => { "risk_score_name" => "low" }
-      }
-    }
-    if [risk_score] >= 3 and [risk_score] < 6 {
-      mutate {
-        add_field => { "risk_score_name" => "medium" }
-      }
-    }
-    if [risk_score] >=6 and [risk_score] < 9 {
-      mutate {
-        add_field => { "risk_score_name" => "high" }
-      }
-    }
-    if [risk_score] >= 9 {
-      mutate {
-        add_field => { "risk_score_name" => "critical" }
-      }
-    }
-  }
-}
-
-output {
-  if "nessus" in [tags] or "tenable" in [tags]{
-    stdout {
-      codec => dots
-    }
-    elasticsearch {
-      hosts => [ "elasticsearch:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
-    }
-  }
-}

resources/elk6/pipeline/2000_qualys_web_scans.conf

@@ -1,160 +0,0 @@
-# Author: Austin Taylor and Justin Henderson
-# Email: austin@hasecuritysolutions.com
-# Last Update: 12/30/2017
-# Version 0.3
-# Description: Take in qualys web scan reports from vulnWhisperer and pumps into logstash
-
-input {
-  file {
-    path => [ "/opt/VulnWhisperer/data/qualys/*.json" , "/opt/VulnWhisperer/data/qualys_web/*.json", "/opt/VulnWhisperer/data/qualys_vuln/*.json"] 
-    type => json
-    codec => json
-    start_position => "beginning"
-    tags => [ "qualys" ]
-    mode => "read"
-    start_position => "beginning"
-    file_completed_action => "delete"
-  }
-}
-
-filter {
-  if "qualys" in [tags] {
-    grok {
-      match => { "path" => [ "(?<tags>qualys_vuln)_scan_%{DATA}_%{INT:last_updated}.json$", "(?<tags>qualys_web)_%{INT:app_id}_%{INT:last_updated}.json$" ] }
-      tag_on_failure => []
-    }
-    
-    mutate {
-      replace => [ "message", "%{message}" ]
-      #gsub => [
-      #  "message", "\|\|\|", " ",
-      #  "message", "\t\t", " ",
-      #  "message", "    ", " ",
-      #  "message", "   ", " ",
-      #  "message", "  ", " ",
-      #  "message", "nan", " ",
-      #  "message",'\n',''
-      #]
-    }
-
-    if "qualys_web" in [tags] {
-    mutate {
-      add_field => { "asset" => "%{web_application_name}" }
-      add_field => { "risk_score" => "%{cvss}" }
-      }
-    } else if "qualys_vuln" in [tags] {
-      mutate {
-        add_field => { "asset" => "%{ip}" }
-        add_field => { "risk_score" => "%{cvss}" }
-      }
-    }
-
-    if [risk] == "1" {
-      mutate { add_field => { "risk_number" => 0 }}
-      mutate { replace => { "risk" => "info" }}
-    }
-    if [risk] == "2" {
-      mutate { add_field => { "risk_number" => 1 }}
-      mutate { replace => { "risk" => "low" }}
-    }
-    if [risk] == "3" {
-      mutate { add_field => { "risk_number" => 2 }}
-      mutate { replace => { "risk" => "medium" }}
-    }
-    if [risk] == "4" {
-      mutate { add_field => { "risk_number" => 3 }}
-      mutate { replace => { "risk" => "high" }}
-    }
-    if [risk] == "5" {
-      mutate { add_field => { "risk_number" => 4 }}
-      mutate { replace => { "risk" => "critical" }}
-    }
-
-    mutate {
-      remove_field => "message"
-    }
-
-    if [first_time_detected] {
-      date {
-        match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "first_time_detected"
-      }
-    }
-    if [first_time_tested] {
-      date {
-        match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "first_time_tested"
-      }
-    }
-    if [last_time_detected] {
-      date {
-        match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "last_time_detected"
-      }
-    }
-    if [last_time_tested] {
-      date {
-        match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "last_time_tested"
-      }
-    }
-
-    # TODO remove when @timestamp is included in event
-    date {
-      match => [ "last_updated", "UNIX" ]
-      target => "@timestamp"
-      remove_field => "last_updated"
-    }
-    mutate {
-      convert => { "plugin_id" => "integer"}
-      convert => { "id" => "integer"}
-      convert => { "risk_number" => "integer"}
-      convert => { "risk_score" => "float"}
-      convert => { "total_times_detected" => "integer"}
-      convert => { "cvss_temporal" => "float"}
-      convert => { "cvss" => "float"}
-    }
-    if [risk_score] == 0 {
-      mutate {
-        add_field => { "risk_score_name" => "info" }
-      }
-    }
-    if [risk_score] > 0 and [risk_score] < 3 {
-      mutate {
-        add_field => { "risk_score_name" => "low" }
-      }
-    }
-    if [risk_score] >= 3 and [risk_score] < 6 {
-      mutate {
-        add_field => { "risk_score_name" => "medium" }
-      }
-    }
-    if [risk_score] >=6 and [risk_score] < 9 {
-      mutate {
-        add_field => { "risk_score_name" => "high" }
-      }
-    }
-    if [risk_score] >= 9 {
-      mutate {
-        add_field => { "risk_score_name" => "critical" }
-      }
-    }
-
-    if [asset] =~ "\.yourdomain\.(com|net)$" {
-      mutate {
-        add_tag => [ "critical_asset" ]
-      }
-    }
-  }
-}
-output {
-  if "qualys" in [tags] {
-    stdout {
-      codec => dots
-    }
-    elasticsearch {
-      hosts => [ "elasticsearch:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
-    }
-  }
-}

resources/elk6/pipeline/3000_openvas.conf

@@ -1,154 +0,0 @@
-# Author: Austin Taylor and Justin Henderson
-# Email: austin@hasecuritysolutions.com
-# Last Update: 03/04/2018
-# Version 0.3
-# Description: Take in qualys web scan reports from vulnWhisperer and pumps into logstash
-
-input {
-  file {
-    path => "/opt/VulnWhisperer/data/openvas/*.json"
-    type => json
-    codec => json
-    start_position => "beginning"
-    tags => [ "openvas_scan", "openvas" ]
-    mode => "read"
-    start_position => "beginning"
-    file_completed_action => "delete"
-
-  }
-}
-
-filter {
-  if "openvas_scan" in [tags] {
-    mutate {
-      replace => [ "message", "%{message}" ]
-      gsub => [
-        "message", "\|\|\|", " ",
-        "message", "\t\t", " ",
-        "message", "    ", " ",
-        "message", "   ", " ",
-        "message", "  ", " ",
-        "message", "nan", " ",
-        "message",'\n',''
-      ]
-    }
-
-
-    grok {
-        match => { "path" => "openvas_scan_%{DATA:scan_id}_%{INT:last_updated}.json$" }
-        tag_on_failure => []
-    }
-
-    mutate {
-      add_field => { "risk_score" => "%{cvss}" }
-    }
-
-    if [risk] == "1" {
-      mutate { add_field => { "risk_number" => 0 }}
-      mutate { replace => { "risk" => "info" }}
-    }
-    if [risk] == "2" {
-      mutate { add_field => { "risk_number" => 1 }}
-      mutate { replace => { "risk" => "low" }}
-    }
-    if [risk] == "3" {
-      mutate { add_field => { "risk_number" => 2 }}
-      mutate { replace => { "risk" => "medium" }}
-    }
-    if [risk] == "4" {
-      mutate { add_field => { "risk_number" => 3 }}
-      mutate { replace => { "risk" => "high" }}
-    }
-    if [risk] == "5" {
-      mutate { add_field => { "risk_number" => 4 }}
-      mutate { replace => { "risk" => "critical" }}
-    }
-
-    mutate {
-      remove_field => "message"
-    }
-
-    if [first_time_detected] {
-      date {
-        match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "first_time_detected"
-      }
-    }
-    if [first_time_tested] {
-      date {
-        match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "first_time_tested"
-      }
-    }
-    if [last_time_detected] {
-      date {
-        match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "last_time_detected"
-      }
-    }
-    if [last_time_tested] {
-      date {
-        match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
-        target => "last_time_tested"
-      }
-    }
-
-    # TODO remove when @timestamp is included in event
-    date {
-      match => [ "last_updated", "UNIX" ]
-      target => "@timestamp"
-      remove_field => "last_updated"
-    }
-    mutate {
-      convert => { "plugin_id" => "integer"}
-      convert => { "id" => "integer"}
-      convert => { "risk_number" => "integer"}
-      convert => { "risk_score" => "float"}
-      convert => { "total_times_detected" => "integer"}
-      convert => { "cvss_temporal" => "float"}
-      convert => { "cvss" => "float"}
-    }
-    if [risk_score] == 0 {
-      mutate {
-        add_field => { "risk_score_name" => "info" }
-      }
-    }
-    if [risk_score] > 0 and [risk_score] < 3 {
-      mutate {
-        add_field => { "risk_score_name" => "low" }
-      }
-    }
-    if [risk_score] >= 3 and [risk_score] < 6 {
-      mutate {
-        add_field => { "risk_score_name" => "medium" }
-      }
-    }
-    if [risk_score] >=6 and [risk_score] < 9 {
-      mutate {
-        add_field => { "risk_score_name" => "high" }
-      }
-    }
-    if [risk_score] >= 9 {
-      mutate {
-        add_field => { "risk_score_name" => "critical" }
-      }
-    }
-    # Add your critical assets by subnet or by hostname. Comment this field out if you don't want to tag any, but the asset panel will break.
-    if [asset] =~ "^10\.0\.100\." {
-      mutate {
-        add_tag => [ "critical_asset" ]
-      }
-    }
-  }
-}
-output {
-  if "openvas" in [tags] {
-    stdout {
-      codec => dots
-    }
-    elasticsearch {
-      hosts => [ "elasticsearch:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
-    }
-  }
-}

resources/elk6/pipeline/4000_jira.conf

@@ -1,25 +0,0 @@
-# Description: Take in jira tickets from vulnWhisperer and pumps into logstash
-
-input {
-  file {
-    path => "/opt/VulnWhisperer/data/jira/*.json"
-    type => json
-    codec => json
-    start_position => "beginning"
-    mode => "read"
-    start_position => "beginning"
-    file_completed_action => "delete"
-    
-    tags => [ "jira" ]
-  }
-}
-
-output {
-  if "jira" in [tags] {
-    stdout { codec => rubydebug }
-    elasticsearch {
-      hosts => [ "elasticsearch:9200" ]
-      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
-    }
-  }
-}

tests/test-docker.sh

@@ -29,49 +29,49 @@ done
 green "✅ Elasticsearch status is green..."
 
 count=0
-until [[ $(curl -s "$logstash_url/_node/stats" | jq '.events.out') -ge 1236 ]]; do
-    yellow "Waiting for Logstash load to finish...  $(curl -s "$logstash_url/_node/stats" | jq '.events.out') of 1236 (attempt $count of 60)"
+until [[ $(curl -s "$logstash_url/_node/stats" | jq '.events.out') -ge 1617 ]]; do
+    yellow "Waiting for Logstash load to finish...  $(curl -s "$logstash_url/_node/stats" | jq '.events.out') of 1617 (attempt $count of 60)"
     ((count++)) && ((count==60)) && break
     sleep 5
 done
 
-if [[ count -le 60 && $(curl -s "$logstash_url/_node/stats" | jq '.events.out') -ge 1236 ]]; then
-    green "✅ Logstash load finished..."
+if [[ count -le 60 && $(curl -s "$logstash_url/_node/stats" | jq '.events.out') -ge 1617 ]]; then
+    green "✅ Logstash load finished $(curl -s "$logstash_url/_node/stats" | jq '.events.out') logs processed..."
 else
-    red "❌ Logstash load didn't complete... $(curl -s "$logstash_url/_node/stats" | jq '.events.out')"
+    red "❌ Logstash load didn't complete  $(curl -s "$logstash_url/_node/stats" | jq '.events.out') logs processed... $(curl -s "$logstash_url/_node/stats" | jq '.events.out')"
 fi
 
 
 count=0
-until [[ $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq '.count') -ge 1232 ]] ; do
-    yellow "Waiting for Elasticsearch index to sync... $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq '.count') of 1232 logs loaded (attempt $count of 150)"
+until [[ $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_count" | jq '.count') -ge 1617 ]] ; do
+    yellow "Waiting for Elasticsearch index to sync... $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_count" | jq '.count') of 1617 logs loaded (attempt $count of 150)"
     ((count++)) && ((count==150)) && break
     sleep 2
 done
-if [[ count -le 50 && $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq '.count') -ge 1232 ]]; then
-    green "✅ logstash-vulnwhisperer-2019.03 document count >= 1232"
+if [[ count -le 50 && $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_count" | jq '.count') -ge 1617 ]]; then
+    green "✅ logstash-vulnwhisperer-* document count $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_count" | jq '.count') >= 1617"
 else
-    red "❌ TIMED OUT waiting for logstash-vulnwhisperer-2019.03 document count: $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq) != 1232"
+    red "❌ TIMED OUT waiting for logstash-vulnwhisperer-* document count: $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_count" | jq) != 1617"
 fi
 
-# if [[ $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq '.count') == 1232 ]]; then
-#     green "✅ Passed: logstash-vulnwhisperer-2019.03 document count == 1232"
+# if [[ $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_count" | jq '.count') == 1232 ]]; then
+#     green "✅ Passed: logstash-vulnwhisperer-* document count == 1232"
 # else
-#     red "❌ Failed: logstash-vulnwhisperer-2019.03 document count == 1232 was: $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count") instead"
+#     red "❌ Failed: logstash-vulnwhisperer-* document count == 1232 was: $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_count") instead"
 #     ((return_code = return_code + 1))
 # fi
 
-# Test Nessus plugin_name:Backported Security Patch Detection (FTP)
-nessus_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_search?q=plugin_name:%22Backported%20Security%20Patch%20Detection%20(FTP)%22%20AND%20asset:176.28.50.164%20AND%20tags:nessus" | jq '.hits.hits[]._source')
-if echo $nessus_doc | jq '.risk' | grep -q "None"; then
-    green "✅ Passed: Nessus risk == None"
+# Test Nessus signature:Backported Security Patch Detection (FTP)
+nessus_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_search?q=signature:%22Backported%20Security%20Patch%20Detection%20(FTP)%22%20AND%20asset:176.28.50.164%20AND%20tags:nessus" | jq '.hits.hits[]._source')
+if echo $nessus_doc | jq '.risk' | grep -q "none"; then
+    green "✅ Passed: Nessus risk == none"
 else
-    red "❌ Failed: Nessus risk == None was: $(echo $nessus_doc | jq '.risk') instead"
+    red "❌ Failed: Nessus risk == none was: $(echo $nessus_doc | jq '.risk') instead"
     ((return_code = return_code + 1))
 fi
 
-# Test Tenable plugin_name:Backported Security Patch Detection (FTP)
-tenable_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_search?q=plugin_name:%22Backported%20Security%20Patch%20Detection%20(FTP)%22%20AND%20asset:176.28.50.164%20AND%20tags:tenable" | jq '.hits.hits[]._source')
+# Test Tenable signature:Backported Security Patch Detection (FTP)
+tenable_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_search?q=signature:%22Backported%20Security%20Patch%20Detection%20(FTP)%22%20AND%20asset:176.28.50.164%20AND%20tags:tenable" | jq '.hits.hits[]._source')
 # Test asset
 if echo $tenable_doc | jq .asset | grep -q '176.28.50.164'; then
     green "✅ Passed: Tenable asset == 176.28.50.164"
@@ -88,21 +88,21 @@ else
     ((return_code = return_code + 1))
 fi
 
-# Test Qualys plugin_name:OpenSSL Multiple Remote Security Vulnerabilities
-qualys_vuln_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_search?q=tags:qualys_vuln%20AND%20ip:%22176.28.50.164%22%20AND%20plugin_name:%22OpenSSL%20Multiple%20Remote%20Security%20Vulnerabilities%22%20AND%20port:465" | jq '.hits.hits[]._source')
+# Test Qualys signature:OpenSSL Multiple Remote Security Vulnerabilities
+qualys_vm_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_search?q=tags:qualys_vm%20AND%20ip:%22176.28.50.164%22%20AND%20signature:%22OpenSSL%20Multiple%20Remote%20Security%20Vulnerabilities%22%20AND%20port:465" | jq '.hits.hits[]._source')
 # Test @timestamp
-if echo $qualys_vuln_doc | jq '.["@timestamp"]' | grep -q '2019-03-30T10:17:41.000Z'; then
+if echo $qualys_vm_doc | jq '.["@timestamp"]' | grep -q '2019-03-30T10:17:41.000Z'; then
     green "✅ Passed: Qualys VM @timestamp == 2019-03-30T10:17:41.000Z"
 else
-    red "❌ Failed: Qualys VM @timestamp == 2019-03-30T10:17:41.000Z was: $(echo $qualys_vuln_doc | jq '.["@timestamp"]') instead"
+    red "❌ Failed: Qualys VM @timestamp == 2019-03-30T10:17:41.000Z was: $(echo $qualys_vm_doc | jq '.["@timestamp"]') instead"
     ((return_code = return_code + 1))
 fi
 
 # Test @XXXX
-if echo $qualys_vuln_doc | jq '.cvss' | grep -q '6.8'; then
-    green "✅ Passed: Qualys VM cvss == 6.8"
+if echo $qualys_vm_doc | jq '.cvss' | grep -q '5.6'; then
+    green "✅ Passed: Qualys VM cvss == 5.6"
 else
-    red "❌ Failed: Qualys VM cvss == 6.8 was: $(echo $qualys_vuln_doc | jq '.cvss') instead"
+    red "❌ Failed: Qualys VM cvss == 5.6 was: $(echo $qualys_vm_doc | jq '.cvss') instead"
     ((return_code = return_code + 1))
 fi
 

tests/test-vuln_whisperer.sh

@@ -59,8 +59,8 @@ yellow "\n*********************************************"
 yellow "*           Test two failed scans           *"
 yellow "*********************************************"
 rm -rf /opt/VulnWhisperer/*
-yellow "Removing ${TEST_PATH}/qualys_vuln/scan_1553941061.87241"
-mv "${TEST_PATH}/qualys_vuln/scan_1553941061.87241"{,.bak}
+yellow "Removing ${TEST_PATH}/qualys_vm/scan_1553941061.87241"
+mv "${TEST_PATH}/qualys_vm/scan_1553941061.87241"{,.bak}
 if vuln_whisperer -F -c configs/test.ini --mock --mock_dir "${TEST_PATH}"; [[ $? -eq 2 ]]; then
     green "\n✅ Passed: Test two failed scans"
 else
@@ -79,11 +79,11 @@ else
     ((return_code = return_code + 1))
 fi
 
-yellow "*********************************************"
+yellow "\n*********************************************"
 yellow "* Test only Qualys VM with one failed scan  *"
 yellow "*********************************************"
 rm -rf /opt/VulnWhisperer/*
-if vuln_whisperer -F -c configs/test.ini -s qualys_vuln --mock --mock_dir "${TEST_PATH}"; [[ $? -eq 1 ]]; then
+if vuln_whisperer -F -c configs/test.ini -s qualys_vm --mock --mock_dir "${TEST_PATH}"; [[ $? -eq 1 ]]; then
     green "\n✅ Passed: Test only Qualys VM with one failed scan"
 else
     red "\n❌ Failed: Test only Qualys VM with one failed scan"
@@ -91,7 +91,7 @@ else
 fi
 
 # Restore the removed files
-mv "${TEST_PATH}/qualys_vuln/scan_1553941061.87241.bak" "${TEST_PATH}/qualys_vuln/scan_1553941061.87241"
+mv "${TEST_PATH}/qualys_vm/scan_1553941061.87241.bak" "${TEST_PATH}/qualys_vm/scan_1553941061.87241"
 mv "${TEST_PATH}/nessus/GET_scans_exports_164_download.bak" "${TEST_PATH}/nessus/GET_scans_exports_164_download"
 
 exit $return_code

vulnwhisp/base/config.py

@@ -31,7 +31,7 @@ class vwConfig(object):
         for section in self.config.sections():
             try:
                 if self.get(section, attribute) in check:
-                    sections.append(section)	
+                    sections.append(section)
             except:
                 self.logger.warn("Section {} has no option '{}'".format(section, attribute))
         return sections
@@ -45,7 +45,7 @@ class vwConfig(object):
         return True
 
     def update_jira_profiles(self, profiles):
-        # create JIRA profiles in the ini config file 
+        # create JIRA profiles in the ini config file
         self.logger.debug('Updating Jira profiles: {}'.format(str(profiles)))
 
         for profile in profiles:
@@ -67,7 +67,7 @@ class vwConfig(object):
                 self.config.set(section_name, 'min_critical_to_report', 'high')
                 self.config.set(section_name, '; automatically report, boolean value ')
                 self.config.set(section_name, 'autoreport', 'false')
-        
+
         # TODO: try/catch this
         # writing changes back to file
         with open(self.config_in, 'w') as configfile:

vulnwhisp/frameworks/nessus.py

@@ -2,7 +2,7 @@ import json
 import logging
 import sys
 import time
-from datetime import datetime
+from datetime import datetime, timedelta
 
 import pytz
 import requests
@@ -17,24 +17,46 @@ class NessusAPI(object):
     SCANS = '/scans'
     SCAN_ID = SCANS + '/{scan_id}'
     HOST_VULN = SCAN_ID + '/hosts/{host_id}'
-    PLUGINS = HOST_VULN + '/plugins/{plugin_id}'
+    PLUGINS = HOST_VULN + '/plugins/{signature_id}'
     EXPORT = SCAN_ID + '/export'
     EXPORT_TOKEN_DOWNLOAD = '/scans/exports/{token_id}/download'
     EXPORT_FILE_DOWNLOAD = EXPORT + '/{file_id}/download'
     EXPORT_STATUS = EXPORT + '/{file_id}/status'
     EXPORT_HISTORY = EXPORT + '?history_id={history_id}'
+    # All column mappings should be lowercase
+    COLUMN_MAPPING = {
+        'cvss base score': 'cvss2_base',
+        'cvss temporal score': 'cvss2_temporal',
+        'cvss temporal vector': 'cvss2_temporal_vector',
+        'cvss vector': 'cvss2_vector',
+        'cvss3 base score': 'cvss3_base',
+        'cvss3 temporal score': 'cvss3_temporal',
+        'cvss3 temporal vector': 'cvss3_temporal_vector',
+        'fqdn': 'dns',
+        'host': 'asset',
+        'ip address': 'ip',
+        'name': 'signature',
+        'os': 'operating_system',
+        'plugin id': 'signature_id',
+        'see also': 'exploitability',
+        'system type': 'category',
+        'vulnerability state': 'state'
+    }
 
-    def __init__(self, hostname=None, port=None, username=None, password=None, verbose=True):
+    def __init__(self, hostname=None, port=None, username=None, password=None, verbose=True, profile=None, access_key=None, secret_key=None):
         self.logger = logging.getLogger('NessusAPI')
-        if verbose:
-            self.logger.setLevel(logging.DEBUG)
-        if username is None or password is None:
-            raise Exception('ERROR: Missing username or password.')
+        self.logger.setLevel(logging.DEBUG if verbose else logging.INFO)
+        if not all((username, password)) and not all((access_key, secret_key)):
+            raise Exception('ERROR: Missing username, password or API keys.')
 
         self.user = username
         self.password = password
+        self.api_keys = False
+        self.access_key = access_key
+        self.secret_key = secret_key
         self.base = 'https://{hostname}:{port}'.format(hostname=hostname, port=port)
         self.verbose = verbose
+        self.profile = profile
 
         self.session = requests.Session()
         self.session.verify = False
@@ -52,9 +74,12 @@ class NessusAPI(object):
             'X-Cookie': None
         }
 
-        self.login()
-        self.scans = self.get_scans()
-        self.scan_ids = self.get_scan_ids()
+        if all((self.access_key, self.secret_key)):
+            self.logger.debug('Using {} API keys'.format(self.profile))
+            self.api_keys = True
+            self.session.headers['X-ApiKeys'] = 'accessKey={}; secretKey={}'.format(self.access_key, self.secret_key)
+        else:
+            self.login()
 
     def login(self):
         auth = '{"username":"%s", "password":"%s"}' % (self.user, self.password)
@@ -64,22 +89,24 @@ class NessusAPI(object):
         else:
             raise Exception('[FAIL] Could not login to Nessus')
 
-    def request(self, url, data=None, headers=None, method='POST', download=False, json_output=False):
+    def request(self, url, data=None, headers=None, method='POST', download=False, json_output=False, params=None):
         timeout = 0
         success = False
-        
+
         method = method.lower()
         url = self.base + url
         self.logger.debug('Requesting to url {}'.format(url))
 
         while (timeout <= 10) and (not success):
-            response = getattr(self.session, method)(url, data=data)
+            response = getattr(self.session, method)(url, data=data, params=params)
             if response.status_code == 401:
                 if url == self.base + self.SESSION:
                     break
                 try:
-                    self.login()
                     timeout += 1
+                    if self.api_keys:
+                        continue
+                    self.login()
                     self.logger.info('Token refreshed')
                 except Exception as e:
                     self.logger.error('Could not refresh token\nReason: {}'.format(str(e)))
@@ -100,12 +127,16 @@ class NessusAPI(object):
             return response_data
         return response
 
-    def get_scans(self):
-        scans = self.request(self.SCANS, method='GET', json_output=True)
+    def get_scans(self, days=None):
+        parameters = {}
+        if days != None:
+            parameters = {
+                "last_modification_date": (datetime.now() - timedelta(days=days)).strftime("%s")
+            }
+        scans = self.request(self.SCANS, method="GET", params=parameters, json_output=True)
         return scans
 
-    def get_scan_ids(self):
-        scans = self.scans
+    def get_scan_ids(self, scans):
         scan_ids = [scan_id['id'] for scan_id in scans['scans']] if scans['scans'] else []
         self.logger.debug('Found {} scan_ids'.format(len(scan_ids)))
         return scan_ids
@@ -114,10 +145,8 @@ class NessusAPI(object):
         data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json_output=True)
         return data['history']
 
-    def download_scan(self, scan_id=None, history=None, export_format="", profile=""):
+    def download_scan(self, scan_id=None, history=None, export_format=''):
         running = True
-        counter = 0
-
         data = {'format': export_format}
         if not history:
             query = self.EXPORT.format(scan_id=scan_id)
@@ -127,23 +156,17 @@ class NessusAPI(object):
         req = self.request(query, data=json.dumps(data), method='POST', json_output=True)
         try:
             file_id = req['file']
-            token_id = req['token'] if 'token' in req else req['temp_token']
+            if self.profile == 'nessus':
+                token_id = req['token'] if 'token' in req else req['temp_token']
         except Exception as e:
             self.logger.error('{}'.format(str(e)))
-        self.logger.info('Download for file id {}'.format(str(file_id)))
+        self.logger.info('Downloading file id {}'.format(str(file_id)))
         while running:
             time.sleep(2)
-            counter += 2
             report_status = self.request(self.EXPORT_STATUS.format(scan_id=scan_id, file_id=file_id), method='GET',
                                          json_output=True)
             running = report_status['status'] != 'ready'
-            sys.stdout.write(".")
-            sys.stdout.flush()
-            # FIXME: why? can this be removed in favour of a counter?
-            if counter % 60 == 0:
-                self.logger.info("Completed: {}".format(counter))
-        self.logger.info("Done: {}".format(counter))
-        if profile == 'tenable':
+        if self.profile == 'tenable' or self.api_keys:
             content = self.request(self.EXPORT_FILE_DOWNLOAD.format(scan_id=scan_id, file_id=file_id), method='GET', download=True)
         else:
             content = self.request(self.EXPORT_TOKEN_DOWNLOAD.format(token_id=token_id), method='GET', download=True)
@@ -169,3 +192,43 @@ class NessusAPI(object):
                     'Pacific Standard Time': 'US/Pacific',
                     'None': 'US/Central'}
         return time_map.get(tz, None)
+
+    def normalise(self, df):
+        self.logger.debug('Normalising data')
+        df = self.map_fields(df)
+        df = self.transform_values(df)
+        return df
+
+    def map_fields(self, df):
+        self.logger.debug('Mapping fields')
+        # Any specific mappings here
+        if self.profile == 'tenable':
+            # Prefer CVSS Base Score over CVSS for tenable
+            self.logger.debug('Dropping redundant tenable fields')
+            df.drop('CVSS', axis=1, inplace=True, errors='ignore')
+
+        # Lowercase and map fields from COLUMN_MAPPING
+        df.columns = [x.lower() for x in df.columns]
+        df.rename(columns=self.COLUMN_MAPPING, inplace=True)
+        df.columns = [x.replace(' ', '_') for x in df.columns]
+
+        return df
+
+    def transform_values(self, df):
+        self.logger.debug('Transforming values')
+
+        df.fillna('', inplace=True)
+
+        if self.profile == 'nessus':
+            # Set IP from asset field
+            df["ip"] = df.loc[df["asset"].str.match("^\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}$"), "asset"]
+
+        # upper/lowercase fields
+        self.logger.debug('Changing case of fields')
+        df['cve'] = df['cve'].str.upper()
+        df['protocol'] = df['protocol'].str.lower()
+        df['risk'] = df['risk'].str.lower()
+
+        df.fillna('', inplace=True)
+
+        return df

vulnwhisp/frameworks/openvas.py

@@ -13,6 +13,20 @@ from bs4 import BeautifulSoup
 
 class OpenVAS_API(object):
     OMP = '/omp'
+    COLUMN_MAPPING = {
+                      'affected software/os': 'affected_software',
+                      'cves': 'cve',
+                      'impact': 'description',
+                      'nvt name': 'signature',
+                      'nvt oid': 'signature_id',
+                      'other references': 'exploitability',
+                      'port protocol': 'protocol',
+                      'severity': 'risk',
+                      'solution type': 'category',
+                      'task name': 'scan_name',
+                      'specific result': 'plugin_output',
+                      'summary': 'synopsis',
+                      }
 
     def __init__(self,
                  hostname=None,
@@ -110,6 +124,7 @@ class OpenVAS_API(object):
         ]
         token = requests.post(self.base + self.OMP, data=data, verify=False)
         return token
+
     def get_report_formats(self):
         params = (
             ('cmd', 'get_report_formats'),
@@ -190,3 +205,25 @@ class OpenVAS_API(object):
         self.processed_reports += 1
         merged_df = pd.merge(report_df, self.openvas_reports, on='report_ids').reset_index().drop('index', axis=1)
         return merged_df
+
+    def normalise(self, df):
+        self.logger.debug('Normalising data')
+        df = self.map_fields(df)
+        df = self.transform_values(df)
+        return df
+
+    def map_fields(self, df):
+        self.logger.debug('Mapping fields')
+        # Lowercase and map fields from COLUMN_MAPPING
+        df.columns = [x.lower() for x in df.columns]
+        df.rename(columns=self.COLUMN_MAPPING, inplace=True)
+        df.columns = [x.replace(' ', '_') for x in df.columns]
+        return df
+
+    def transform_values(self, df):
+        self.logger.debug('Transforming values')
+        df['port'].fillna(0).astype(int)
+        df['risk'] = df['risk'].str.lower()
+        df['asset'] = df['ip']
+        df.fillna('', inplace=True)
+        return df

vulnwhisp/frameworks/qualys_vm.py

@@ -5,6 +5,7 @@ __author__ = 'Nathan Young'
 import logging
 import sys
 import xml.etree.ElementTree as ET
+from datetime import datetime, timedelta
 
 import dateutil.parser as dp
 import pandas as pd
@@ -18,7 +19,7 @@ class qualysWhisperAPI(object):
         self.logger = logging.getLogger('qualysWhisperAPI')
         self.config = config
         try:
-            self.qgc = qualysapi.connect(config, 'qualys_vuln')
+            self.qgc = qualysapi.connect(config, 'qualys_vm')
             # Fail early if we can't make a request or auth is incorrect
             self.qgc.request('about.php')
             self.logger.info('Connected to Qualys at {}'.format(self.qgc.server))
@@ -28,7 +29,9 @@ class qualysWhisperAPI(object):
 
     def scan_xml_parser(self, xml):
         all_records = []
-        root = ET.XML(xml.encode("utf-8"))
+        root = ET.XML(xml.encode('utf-8'))
+        if len(root.find('.//SCAN_LIST')) == 0:
+            return pd.DataFrame(columns=['id', 'status'])
         for child in root.find('.//SCAN_LIST'):
             all_records.append({
                 'name': child.find('TITLE').text,
@@ -40,12 +43,17 @@ class qualysWhisperAPI(object):
             })
         return pd.DataFrame(all_records)
 
-    def get_all_scans(self):
+    def get_all_scans(self, days=None):
+        if days == None:
+            self.launched_date = '0001-01-01'
+        else:
+            self.launched_date = (datetime.now() - timedelta(days=days)).strftime('%Y-%m-%d')
         parameters = {
             'action': 'list',
             'echo_request': 0,
             'show_op': 0,
-            'launched_after_datetime': '0001-01-01'
+            'state': 'Finished',
+            'launched_after_datetime': self.launched_date
         }
         scans_xml = self.qgc.request(self.SCANS, parameters)
         return self.scan_xml_parser(scans_xml)
@@ -78,6 +86,17 @@ class qualysUtils:
 
 class qualysVulnScan:
 
+    COLUMN_MAPPING = {
+        'cve_id': 'cve',
+        'impact': 'synopsis',
+        'ip_status': 'state',
+        'os': 'operating_system',
+        'qid': 'signature_id',
+        'results': 'plugin_output',
+        'threat': 'description',
+        'title': 'signature'
+    }
+
     def __init__(
         self,
         config=None,
@@ -122,3 +141,51 @@ class qualysVulnScan:
             return scan_report
 
         return scan_report
+
+    def normalise(self, df):
+        self.logger.debug('Normalising data')
+        df = self.map_fields(df)
+        df = self.transform_values(df)
+        return df
+
+    def map_fields(self, df):
+        self.logger.info('Mapping fields')
+
+        # Lowercase and map fields from COLUMN_MAPPING
+        df.columns = [x.lower() for x in df.columns]
+        df.rename(columns=self.COLUMN_MAPPING, inplace=True)
+        df.columns = [x.replace(' ', '_') for x in df.columns]
+
+        return df
+
+    def transform_values(self, df):
+        self.logger.info('Transforming values')
+
+        df.fillna('', inplace=True)
+
+        # upper/lowercase fields
+        self.logger.info('Changing case of fields')
+        df['cve'] = df['cve'].str.upper()
+        df['protocol'] = df['protocol'].str.lower()
+
+        # Contruct the CVSS vector
+        self.logger.info('Extracting CVSS components')
+        df['cvss2_vector'] = df['cvss_base'].str.extract('\((.*)\)', expand=False)
+        df['cvss2_base'] = df['cvss_base'].str.extract('^(\d+(?:\.\d+)?)', expand=False)
+        df['cvss2_temporal_vector'] = df['cvss_temporal'].str.extract('\((.*)\)', expand=False)
+        df['cvss2_temporal'] = df['cvss_temporal'].str.extract('^(\d+(?:\.\d+)?)', expand=False)
+        df.drop('cvss_base', axis=1, inplace=True, errors='ignore')
+        df.drop('cvss_temporal', axis=1, inplace=True, errors='ignore')
+
+        # Set asset to ip
+        df['asset'] = df['ip']
+
+        # Set dns to fqdn if missing
+        df.loc[df['dns'] == '', 'dns'] = df['fqdn']
+
+        # Convert Qualys severity to standardised risk number
+        df['risk_number'] = df['severity'].astype(int)-1
+
+        df.fillna('', inplace=True)
+
+        return df

vulnwhisp/frameworks/qualys_was.py

@@ -2,18 +2,20 @@
 # -*- coding: utf-8 -*-
 __author__ = 'Austin Taylor'
 
-from lxml import objectify
-from lxml.builder import E
+import csv
+import logging
+import os
+import sys
 import xml.etree.ElementTree as ET
+from datetime import datetime, timedelta
+
+import dateutil.parser as dp
 import pandas as pd
 import qualysapi
 import qualysapi.config as qcconf
 import requests
-import sys
-import os
-import csv
-import logging
-import dateutil.parser as dp
+from lxml import objectify
+from lxml.builder import E
 
 
 class qualysWhisperAPI(object):
@@ -37,7 +39,7 @@ class qualysWhisperAPI(object):
         self.logger = logging.getLogger('qualysWhisperAPI')
         self.config = config
         try:
-            self.qgc = qualysapi.connect(config, 'qualys_web')
+            self.qgc = qualysapi.connect(config, 'qualys_was')
             self.logger.info('Connected to Qualys at {}'.format(self.qgc.server))
         except Exception as e:
             self.logger.error('Could not connect to Qualys: {}'.format(str(e)))
@@ -45,7 +47,7 @@ class qualysWhisperAPI(object):
             #"content-type": "text/xml"}
             "Accept" : "application/json",
             "Content-Type": "application/json"}
-        self.config_parse = qcconf.QualysConnectConfig(config, 'qualys_web')
+        self.config_parse = qcconf.QualysConnectConfig(config, 'qualys_was')
         try:
             self.template_id = self.config_parse.get_template_id()
         except:
@@ -59,10 +61,12 @@ class qualysWhisperAPI(object):
         """
         Checks number of scans, used to control the api limits
         """
-        parameters = (
-            E.ServiceRequest(
+        parameters = E.ServiceRequest(
                 E.filters(
-                    E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status))))
+                    E.Criteria({"field": "status", "operator": "EQUALS"}, status),
+                    E.Criteria({"field": "launchedDate", "operator": "GREATER"}, self.launched_date)
+                )
+            )
         xml_output = self.qgc.request(self.COUNT_WASSCAN, parameters)
         root = objectify.fromstring(xml_output.encode('utf-8'))
         return root.count.text
@@ -70,8 +74,8 @@ class qualysWhisperAPI(object):
     def generate_scan_result_XML(self, limit=1000, offset=1, status='FINISHED'):
         report_xml = E.ServiceRequest(
             E.filters(
-                E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status
-                           ),
+                E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status),
+                E.Criteria({"field": "launchedDate", "operator": "GREATER"}, self.launched_date)
             ),
             E.preferences(
                 E.startFromOffset(str(offset)),
@@ -103,13 +107,17 @@ class qualysWhisperAPI(object):
                 all_records.append(record)
         return pd.DataFrame(all_records)
 
-    def get_all_scans(self, limit=1000, offset=1, status='FINISHED'):
+
+    def get_all_scans(self, limit=1000, offset=1, status='FINISHED', days=None):
+        if days == None:
+            self.launched_date = '0001-01-01'
+        else:
+            self.launched_date = (datetime.now() - timedelta(days=days)).strftime('%Y-%m-%d')
         qualys_api_limit = limit
         dataframes = []
         _records = []
         try:
             total = int(self.get_was_scan_count(status=status))
-            self.logger.error('Already have WAS scan count')
             self.logger.info('Retrieving information for {} scans'.format(total))
             for i in range(0, total):
                 if i % limit == 0:
@@ -120,6 +128,8 @@ class qualysWhisperAPI(object):
                     _records.append(scan_info)
             self.logger.debug('Converting XML to DataFrame')
             dataframes = [self.xml_parser(xml) for xml in _records]
+            if not dataframes:
+                return pd.DataFrame(columns=['id'])
         except Exception as e:
             self.logger.error("Couldn't process all scans: {}".format(e))
 
@@ -282,13 +292,26 @@ class qualysUtils:
     def iso_to_epoch(self, dt):
         return dp.parse(dt).strftime('%s')
 
-    def cleanser(self, _data):
-        repls = (('\n', '|||'), ('\r', '|||'), (',', ';'), ('\t', '|||'))
-        if _data:
-            _data = reduce(lambda a, kv: a.replace(*kv), repls, str(_data))
-        return _data
-
 class qualysScanReport:
+
+    COLUMN_MAPPING = {
+        'CVSS Base': 'cvss2_base',
+        'CVSS Temporal': 'cvss2_temporal',
+        'DescriptionCatSev': 'category_description',
+        'DescriptionSeverity': 'synopsis',
+        'Evidence #1': 'evidence',
+        'Payload #1': 'payload',
+        'QID': 'signature_id',
+        'Request Headers #1': 'request_headers',
+        'Request Method #1': 'request_method',
+        'Request URL #1': 'request_url',
+        'Response #1': 'plugin_output',
+        'Title': 'signature',
+        'Url': 'uri',
+        'URL': 'url',
+        'Vulnerability Category': 'type',
+    }
+
     # URL Vulnerability Information
     WEB_SCAN_VULN_BLOCK = list(qualysReportFields.VULN_BLOCK)
     WEB_SCAN_VULN_BLOCK.insert(WEB_SCAN_VULN_BLOCK.index('QID'), 'Detection ID')
@@ -297,6 +320,16 @@ class qualysScanReport:
     WEB_SCAN_VULN_HEADER[WEB_SCAN_VULN_BLOCK.index(qualysReportFields.CATEGORIES[0])] = \
         'Vulnerability Category'
 
+    # Add an alternative vulnerability header
+    WEB_SCAN_VULN_BLOCK_ALT = WEB_SCAN_VULN_BLOCK[:]
+    WEB_SCAN_VULN_BLOCK_ALT.insert(WEB_SCAN_VULN_BLOCK_ALT.index('First Time Detected'), 'Detection Date')
+    remove_fields = ['Last Time Tested', 'Times Detected', 'First Time Detected', 'Last Time Detected']
+    WEB_SCAN_VULN_BLOCK_ALT = [x for x in WEB_SCAN_VULN_BLOCK_ALT if x not in remove_fields]
+
+    WEB_SCAN_VULN_HEADER_ALT = WEB_SCAN_VULN_BLOCK_ALT[:]
+    WEB_SCAN_VULN_HEADER_ALT[WEB_SCAN_VULN_BLOCK_ALT.index(qualysReportFields.CATEGORIES[0])] = \
+        'Vulnerability Category'
+
     WEB_SCAN_SENSITIVE_HEADER = list(WEB_SCAN_VULN_HEADER)
     WEB_SCAN_SENSITIVE_HEADER.insert(WEB_SCAN_SENSITIVE_HEADER.index('Url'
                                                                      ), 'Content')
@@ -358,6 +391,17 @@ class qualysScanReport:
                                                                                            self.WEB_SCAN_INFO_BLOCK],
                                                                                        pop_last=True),
                                                                columns=self.WEB_SCAN_VULN_HEADER)
+            if len(dict_tracker['WEB_SCAN_VULN_BLOCK']) == 0:
+                # Try alternative headers
+                dict_tracker["WEB_SCAN_VULN_BLOCK"] = pd.DataFrame(
+                    self.utils.grab_section(
+                        report,
+                        self.WEB_SCAN_VULN_BLOCK_ALT,
+                        end=[self.WEB_SCAN_SENSITIVE_BLOCK, self.WEB_SCAN_INFO_BLOCK],
+                        pop_last=True,
+                    ),
+                    columns=self.WEB_SCAN_VULN_HEADER_ALT,
+                )
             dict_tracker['WEB_SCAN_SENSITIVE_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
                                                                                             self.WEB_SCAN_SENSITIVE_BLOCK,
                                                                                             end=[
@@ -423,12 +467,9 @@ class qualysScanReport:
                               'Request Headers #1', 'Response #1', 'Evidence #1',
                               'Description', 'Impact', 'Solution', 'Url', 'Content']
 
-        for col in columns_to_cleanse:
-            merged_df[col] = merged_df[col].apply(self.utils.cleanser)
-
         merged_df = merged_df.drop(['QID_y', 'QID_x'], axis=1)
         merged_df = merged_df.rename(columns={'Id': 'QID'})
-        
+
         merged_df = merged_df.assign(**df_dict['SCAN_META'].to_dict(orient='records')[0])
 
         merged_df = pd.merge(merged_df, df_dict['CATEGORY_HEADER'], how='left', left_on=['Category', 'Severity Level'],
@@ -444,7 +485,7 @@ class qualysScanReport:
         return merged_df
 
     def download_file(self, path='', file_id=None):
-        report = self.qw.download_report(file_id)
+        report = self.qw.download_report(file_id).encode('utf-8')
         filename = path + str(file_id) + '.csv'
         file_out = open(filename, 'w')
         for line in report.splitlines():
@@ -463,3 +504,40 @@ class qualysScanReport:
         merged_data.sort_index(axis=1, inplace=True)
 
         return merged_data
+
+    def normalise(self, df):
+        self.logger.debug('Normalising data')
+        df = self.map_fields(df)
+        df = self.transform_values(df)
+        return df
+
+    def map_fields(self, df):
+        self.logger.debug('Mapping fields')
+
+        df.rename(columns=self.COLUMN_MAPPING, inplace=True)
+
+        # Lowercase and map fields from COLUMN_MAPPING
+        df.columns = [x.lower() for x in df.columns]
+        df.columns = [x.replace(' ', '_') for x in df.columns]
+
+        return df
+
+    def transform_values(self, df):
+        self.logger.debug('Transforming values')
+        df.fillna('', inplace=True)
+
+        self.logger.info('Changing case of fields')
+        df['cwe'] = df['cwe'].str.upper()
+
+        # Convert Qualys severity to standardised risk number
+        df['risk_number'] =  df['severity'].astype(int)-1
+
+        # Extract dns field from URL
+        df['dns'] = df['url'].str.extract('https?://([^/]+)', expand=False)
+        df['dns'] = df.loc[df['uri'] != '','uri'].str.extract('https?://([^/]+)', expand=False)
+
+        # Set asset to web_application_name
+        df['asset'] = df['web_application_name']
+
+        df.fillna('', inplace=True)
+        return df

vulnwhisp/reporting/jira_api.py

@@ -40,12 +40,12 @@ class JiraAPI(object):
             self.close_obsolete_tickets()
         # deletes the tag "server_decommission" from those tickets closed <=3 months ago
         self.decommission_cleanup()
-        
+
         self.jira_still_vulnerable_comment = '''This ticket has been reopened due to the vulnerability not having been fixed (if multiple assets are affected, all need to be fixed; if the server is down, lastest known vulnerability might be the one reported).
         - In the case of the team accepting the risk and wanting to close the ticket, please add the label "*risk_accepted*" to the ticket before closing it.
         - If server has been decommissioned, please add the label "*server_decommission*" to the ticket before closing it.
         - If when checking the vulnerability it looks like a false positive, _+please elaborate in a comment+_ and add the label "*false_positive*" before closing it; we will review it and report it to the vendor.
-        
+
         If you have further doubts, please contact the Security Team.'''
 
     def create_ticket(self, title, desc, project="IS", components=[], tags=[], attachment_contents = []):
@@ -67,31 +67,31 @@ class JiraAPI(object):
             if not exists:
                 self.logger.error("Error creating Ticket: component {} not found".format(component))
                 return 0
-                    
+
         new_issue = self.jira.create_issue(project=project,
                                            summary=title,
                                            description=desc,
                                            issuetype={'name': 'Bug'},
                                            labels=labels,
                                            components=components_ticket)
-        
+
         self.logger.info("Ticket {} created successfully".format(new_issue))
-        
+
         if attachment_contents:
             self.add_content_as_attachment(new_issue, attachment_contents)
-        
+
         return new_issue
-    
+
     #Basic JIRA Metrics
     def metrics_open_tickets(self, project=None):
-        jql = "labels= vulnerability_management and resolution = Unresolved" 
+        jql = "labels= vulnerability_management and resolution = Unresolved"
         if project:
             jql += " and (project='{}')".format(project)
-        self.logger.debug('Executing: {}'.format(jql)) 
+        self.logger.debug('Executing: {}'.format(jql))
         return len(self.jira.search_issues(jql, maxResults=0))
 
     def metrics_closed_tickets(self, project=None):
-        jql = "labels= vulnerability_management and NOT resolution = Unresolved AND created >=startOfMonth(-{})".format(self.max_time_tracking) 
+        jql = "labels= vulnerability_management and NOT resolution = Unresolved AND created >=startOfMonth(-{})".format(self.max_time_tracking)
         if project:
             jql += " and (project='{}')".format(project)
         return len(self.jira.search_issues(jql, maxResults=0))
@@ -105,10 +105,10 @@ class JiraAPI(object):
             # if it has, they will be replaced by "_"
             if " " in  vuln['scan_name']:
                 vuln['scan_name'] = "_".join(vuln['scan_name'].split(" "))
-            
+
             # we exclude from the vulnerabilities to report those assets that already exist with *risk_accepted*/*server_decommission*
             vuln = self.exclude_accepted_assets(vuln)
-            
+
             # make sure after exclusion of risk_accepted assets there are still assets
             if vuln['ips']:
                 exists = False
@@ -140,18 +140,18 @@ class JiraAPI(object):
                 self.create_ticket(title=vuln['title'], desc=tpl, project=project, components=components, tags=[vuln['source'], vuln['scan_name'], 'vulnerability', vuln['risk']], attachment_contents = attachment_contents)
             else:
                 self.logger.info("Ignoring vulnerability as all assets are already reported in a risk_accepted ticket")
-        
+
         self.close_fixed_tickets(vulnerabilities)
         # we reinitialize so the next sync redoes the query with their specific variables
         self.all_tickets = []
         self.excluded_tickets = []
         return True
-    
+
     def exclude_accepted_assets(self, vuln):
         # we want to check JIRA tickets with risk_accepted/server_decommission or false_positive labels sharing the same source
         # will exclude tickets older than 12 months, old tickets will get closed for higiene and recreated if still vulnerable
-        labels = [vuln['source'], vuln['scan_name'], 'vulnerability_management', 'vulnerability'] 
-        
+        labels = [vuln['source'], vuln['scan_name'], 'vulnerability_management', 'vulnerability']
+
         if not self.excluded_tickets:
             jql = "{} AND labels in (risk_accepted,server_decommission, false_positive) AND NOT labels=advisory AND created >=startOfMonth(-{})".format(" AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
             self.excluded_tickets = self.jira.search_issues(jql, maxResults=0)
@@ -169,14 +169,14 @@ class JiraAPI(object):
                     #checking_assets is a list, we add to our full list for later delete all assets
                     assets_to_exclude+=checking_assets
                     tickets_excluded_assets.append(checking_ticketid)
-       
+
         if assets_to_exclude:
             assets_to_remove = []
             self.logger.warn("Vulnerable Assets seen on an already existing risk_accepted Jira ticket: {}".format(', '.join(tickets_excluded_assets)))
             self.logger.debug("Original assets: {}".format(vuln['ips']))
-            #assets in vulnerability have the structure "ip - hostname - port", so we need to match by partial 
+            #assets in vulnerability have the structure "ip - hostname - port", so we need to match by partial
             for exclusion in assets_to_exclude:
-                # for efficiency, we walk the backwards the array of ips from the scanners, as we will be popping out the matches 
+                # for efficiency, we walk the backwards the array of ips from the scanners, as we will be popping out the matches
                 # and we don't want it to affect the rest of the processing (otherwise, it would miss the asset right after the removed one)
                 for index in range(len(vuln['ips']))[::-1]:
                     if exclusion == vuln['ips'][index].split(" - ")[0]:
@@ -194,28 +194,28 @@ class JiraAPI(object):
         # we need to return if the vulnerability has already been reported and the ID of the ticket for further processing
         #function returns array [duplicated(bool), update(bool), ticketid, ticket_assets]
         title = vuln['title']
-        labels = [vuln['source'], vuln['scan_name'], 'vulnerability_management', 'vulnerability'] 
+        labels = [vuln['source'], vuln['scan_name'], 'vulnerability_management', 'vulnerability']
         #list(set()) to remove duplicates
         assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", ",".join(vuln['ips']))))
-        
+
         if not self.all_tickets:
             self.logger.info("Retrieving all JIRA tickets with the following tags {}".format(labels))
             # we want to check all JIRA tickets, to include tickets moved to other queues
             # will exclude tickets older than 12 months, old tickets will get closed for higiene and recreated if still vulnerable
             jql = "{} AND NOT labels=advisory AND created >=startOfMonth(-{})".format(" AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
-            
+
             self.all_tickets = self.jira.search_issues(jql, maxResults=0)
-        
+
         #WARNING: function IGNORES DUPLICATES, after finding a "duplicate" will just return it exists
         #it wont iterate over the rest of tickets looking for other possible duplicates/similar issues
         self.logger.info("Comparing Vulnerabilities to created tickets")
         for index in range(len(self.all_tickets)):
             checking_ticketid, checking_title, checking_assets = self.ticket_get_unique_fields(self.all_tickets[index])
             # added "not risk_accepted", as if it is risk_accepted, we will create a new ticket excluding the accepted assets
-            if title.encode('ascii') == checking_title.encode('ascii') and not self.is_risk_accepted(self.jira.issue(checking_ticketid)): 
+            if title.encode('ascii') == checking_title.encode('ascii') and not self.is_risk_accepted(self.jira.issue(checking_ticketid)):
                 difference = list(set(assets).symmetric_difference(checking_assets))
                 #to check intersection - set(assets) & set(checking_assets)
-                if difference: 
+                if difference:
                     self.logger.info("Asset mismatch, ticket to update. Ticket ID: {}".format(checking_ticketid))
                     return False, True, checking_ticketid, checking_assets #this will automatically validate
                 else:
@@ -230,12 +230,12 @@ class JiraAPI(object):
         try:
             affected_assets_section = ticket.raw.get('fields', {}).get('description').encode("ascii").split("{panel:title=Affected Assets}")[1].split("{panel}")[0]
             assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", affected_assets_section)))
-        
+
         except Exception as e:
             self.logger.error("Ticket IPs regex failed. Ticket ID: {}. Reason: {}".format(ticketid, e))
             assets = []
-        
-        try:     
+
+        try:
             if not assets:
                 #check if attachment, if so, get assets from attachment
                 affected_assets_section = self.check_ips_attachment(ticket)
@@ -243,7 +243,7 @@ class JiraAPI(object):
                     assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", affected_assets_section)))
         except Exception as e:
             self.logger.error("Ticket IPs Attachment regex failed. Ticket ID: {}. Reason: {}".format(ticketid, e))
-                
+
         return ticketid, title, assets
 
     def check_ips_attachment(self, ticket):
@@ -260,11 +260,11 @@ class JiraAPI(object):
                     if item.get('filename') == self.attachment_filename:
                         if not latest:
                             latest = item.get('created')
-                            attachment_id = item.get('id') 
+                            attachment_id = item.get('id')
                         else:
                             if latest < item.get('created'):
-                                latest = item.get('created')         
-                                attachment_id = item.get('id') 
+                                latest = item.get('created')
+                                attachment_id = item.get('id')
             affected_assets_section = self.jira.attachment(attachment_id).get()
 
         except Exception as e:
@@ -300,7 +300,7 @@ class JiraAPI(object):
         return True
 
     def get_ticket_reported_assets(self, ticket):
-        #[METRICS] return a list with all the affected assets for that vulnerability (including already resolved ones) 
+        #[METRICS] return a list with all the affected assets for that vulnerability (including already resolved ones)
         return list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b",str(self.jira.issue(ticket).raw))))
 
     def get_resolution_time(self, ticket):
@@ -311,7 +311,7 @@ class JiraAPI(object):
             #dates follow format '2018-11-06T10:36:13.849+0100'
             created = [int(x) for x in ticket_data['created'].split('.')[0].replace('T', '-').replace(':','-').split('-')]
             resolved =[int(x) for x in ticket_data['resolutiondate'].split('.')[0].replace('T', '-').replace(':','-').split('-')]
-            
+
             start = datetime(created[0],created[1],created[2],created[3],created[4],created[5])
             end = datetime(resolved[0],resolved[1],resolved[2],resolved[3],resolved[4],resolved[5])
             return (end-start).days
@@ -323,7 +323,7 @@ class JiraAPI(object):
     def ticket_update_assets(self, vuln, ticketid, ticket_assets):
         # correct description will always be in the vulnerability to report, only needed to update description to new one
         self.logger.info("Ticket {} exists, UPDATE requested".format(ticketid))
-        
+
         #for now, if a vulnerability has been accepted ('accepted_risk'), ticket is completely ignored and not updated (no new assets)
 
         #TODO when vulnerability accepted, create a new ticket with only the non-accepted vulnerable assets
@@ -335,12 +335,12 @@ class JiraAPI(object):
             if self.is_risk_accepted(ticket_obj):
                 return 0
             self.reopen_ticket(ticketid=ticketid, comment=self.jira_still_vulnerable_comment)
-        
+
         #First will do the comparison of assets
         ticket_obj.update()
         assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", ",".join(vuln['ips']))))
         difference = list(set(assets).symmetric_difference(ticket_assets))
-        
+
         comment = ''
         added = ''
         removed = ''
@@ -356,13 +356,13 @@ class JiraAPI(object):
                 removed += '* {}\n'.format(asset)
 
         comment = added + removed
-        
+
         #then will check if assets are too many that need to be added as an attachment
         attachment_contents = []
         if len(vuln['ips']) > self.max_ips_ticket:
             attachment_contents = vuln['ips']
             vuln['ips'] = ["Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(assets = len(attachment_contents))]
-        
+
         #fill the ticket description template
         try:
             tpl = template(self.template_path, vuln)
@@ -376,7 +376,7 @@ class JiraAPI(object):
             if attachment_contents:
                 self.clean_old_attachments(ticket_obj)
                 self.add_content_as_attachment(ticket_obj, attachment_contents)
-                
+
             ticket_obj.update(description=tpl, comment=comment, fields={"labels":ticket_obj.fields.labels})
             self.logger.info("Ticket {} updated successfully".format(ticketid))
             self.add_label(ticketid, 'updated')
@@ -386,24 +386,24 @@ class JiraAPI(object):
 
     def add_label(self, ticketid, label):
         ticket_obj = self.jira.issue(ticketid)
-        
+
         if label not in [x.encode('utf8') for x in ticket_obj.fields.labels]:
             ticket_obj.fields.labels.append(label)
-        
+
             try:
                 ticket_obj.update(fields={"labels":ticket_obj.fields.labels})
                 self.logger.info("Added label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
             except:
                 self.logger.error("Error while trying to add label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
-        
+
         return 0
 
     def remove_label(self, ticketid, label):
         ticket_obj = self.jira.issue(ticketid)
-        
+
         if label in [x.encode('utf8') for x in ticket_obj.fields.labels]:
             ticket_obj.fields.labels.remove(label)
-        
+
             try:
                 ticket_obj.update(fields={"labels":ticket_obj.fields.labels})
                 self.logger.info("Removed label {label} from ticket {ticket}".format(label=label, ticket=ticketid))
@@ -411,7 +411,7 @@ class JiraAPI(object):
                 self.logger.error("Error while trying to remove label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
         else:
             self.logger.error("Error: label {label} not in ticket {ticket}".format(label=label, ticket=ticketid))
-        
+
         return 0
 
     def close_fixed_tickets(self, vulnerabilities):
@@ -431,7 +431,7 @@ class JiraAPI(object):
                 self.logger.info("Ticket {} is still vulnerable".format(ticket))
                 continue
             self.logger.info("Ticket {} is no longer vulnerable".format(ticket))
-            self.close_ticket(ticket, self.JIRA_RESOLUTION_FIXED, comment) 
+            self.close_ticket(ticket, self.JIRA_RESOLUTION_FIXED, comment)
         return 0
 
 
@@ -484,7 +484,7 @@ class JiraAPI(object):
         self.logger.debug("Ticket {} exists, REOPEN requested".format(ticketid))
         # this will reopen a ticket by ticketid
         ticket_obj = self.jira.issue(ticketid)
-        
+
         if self.is_ticket_resolved(ticket_obj):
             if (not self.is_risk_accepted(ticket_obj) or ignore_labels):
                 try:
@@ -516,21 +516,21 @@ class JiraAPI(object):
                 # continue with ticket data so that a new ticket is created in place of the "lost" one
                 self.logger.error("error closing ticket {}: {}".format(ticketid, e))
                 return 0
-                
+
         return 0
 
     def close_obsolete_tickets(self):
-        # Close tickets older than 12 months, vulnerabilities not solved will get created a new ticket 
+        # Close tickets older than 12 months, vulnerabilities not solved will get created a new ticket
         self.logger.info("Closing obsolete tickets older than {} months".format(self.max_time_tracking))
         jql = "labels=vulnerability_management AND created <startOfMonth(-{}) and resolution=Unresolved".format(self.max_time_tracking)
         tickets_to_close = self.jira.search_issues(jql, maxResults=0)
-        
+
         comment = '''This ticket is being closed for hygiene, as it is more than {} months old.
         If the vulnerability still exists, a new ticket will be opened.'''.format(self.max_time_tracking)
-        
+
         for ticket in tickets_to_close:
                 self.close_ticket(ticket, self.JIRA_RESOLUTION_OBSOLETE, comment)
-        
+
         return 0
 
     def project_exists(self, project):
@@ -547,7 +547,7 @@ class JiraAPI(object):
         '''
         #check if file already exists
         check_date = str(date.today())
-        fname = '{}jira_{}.json'.format(path, check_date) 
+        fname = '{}jira_{}.json'.format(path, check_date)
         if os.path.isfile(fname):
             self.logger.info("File {} already exists, skipping ticket download".format(fname))
             return True
@@ -555,23 +555,23 @@ class JiraAPI(object):
             self.logger.info("Saving locally tickets from the last {} months".format(self.max_time_tracking))
             jql = "labels=vulnerability_management AND created >=startOfMonth(-{})".format(self.max_time_tracking)
             tickets_data = self.jira.search_issues(jql, maxResults=0)
-            
+
             #end of line needed, as writelines() doesn't add it automatically, otherwise one big line
             to_save = [json.dumps(ticket.raw.get('fields'))+"\n" for ticket in tickets_data]
             with open(fname, 'w') as outfile:
                 outfile.writelines(to_save)
                 self.logger.info("Tickets saved succesfully.")
-            
+
             return True
 
         except Exception as e:
             self.logger.error("Tickets could not be saved locally: {}.".format(e))
-        
-        return False 
+
+        return False
 
     def decommission_cleanup(self):
         '''
-        deletes the server_decomission tag from those tickets that have been 
+        deletes the server_decomission tag from those tickets that have been
         closed already for more than x months (default is 3 months) in order to clean solved issues
         for statistics purposes
         '''
@@ -579,15 +579,15 @@ class JiraAPI(object):
 
         jql = "labels=vulnerability_management AND labels=server_decommission and resolutiondate <=startOfMonth(-{})".format(self.max_decommission_time)
         decommissioned_tickets = self.jira.search_issues(jql, maxResults=0)
-        
+
         comment = '''This ticket is having deleted the *server_decommission* tag, as it is more than {} months old and is expected to already have been decommissioned.
         If that is not the case and the vulnerability still exists, the vulnerability will be opened again.'''.format(self.max_decommission_time)
-        
+
         for ticket in decommissioned_tickets:
-            #we open first the ticket, as we want to make sure the process is not blocked due to 
+            #we open first the ticket, as we want to make sure the process is not blocked due to
             #an unexisting jira workflow or unallowed edit from closed tickets
             self.reopen_ticket(ticketid=ticket, ignore_labels=True)
             self.remove_label(ticket, 'server_decommission')
             self.close_ticket(ticket, self.JIRA_RESOLUTION_FIXED, comment)
-        
+
         return 0

vulnwhisp/test/mock.py

@@ -1,5 +1,6 @@
-import os
 import logging
+import os
+
 import httpretty
 
 
@@ -16,61 +17,112 @@ class mockAPI(object):
             self.logger.setLevel(logging.DEBUG)
 
         self.logger.info('mockAPI initialised, API requests will be mocked')
-        self.logger.debug('Test path resolved as {}'.format(self.mock_dir))
+        self.logger.info('Test path resolved as {}'.format(self.mock_dir))
 
     def get_directories(self, path):
         dir, subdirs, files = next(os.walk(path))
-        return subdirs
+        return sorted(subdirs)
 
     def get_files(self, path):
         dir, subdirs, files = next(os.walk(path))
-        return files
-
-    def qualys_vuln_callback(self, request, uri, response_headers):
-        self.logger.debug('Simulating response for {} ({})'.format(uri, request.body))
-        if 'list' in request.parsed_body['action']:
-            return [200,
-                    response_headers,
-                    open('{}/{}'.format(self.qualys_vuln_path, 'scans')).read()]
-        elif 'fetch' in request.parsed_body['action']:
-            try:
-                response_body = open('{}/{}'.format(
-                                      self.qualys_vuln_path, 
-                                      request.parsed_body['scan_ref'][0].replace('/', '_'))
-                                    ).read()
-            except:
-                # Can't find the file, just send an empty response
-                response_body = ''
-        return [200, response_headers, response_body] 
+        return sorted(files)
 
     def create_nessus_resource(self, framework):
         for filename in self.get_files('{}/{}'.format(self.mock_dir, framework)):
             method, resource = filename.split('_', 1)
             resource = resource.replace('_', '/')
-            self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, method, resource))
+            self.logger.info('Adding mocked {} endpoint {} {}'.format(framework, method, resource))
             httpretty.register_uri(
                 getattr(httpretty, method), 'https://{}:443/{}'.format(framework, resource),
                 body=open('{}/{}/{}'.format(self.mock_dir, framework, filename)).read()
             )
 
-    def create_qualys_vuln_resource(self, framework):
+    def qualys_vm_callback(self, request, uri, response_headers):
+        self.logger.info('Simulating response for {} ({})'.format(uri, request.body))
+        if 'list' in request.parsed_body['action']:
+            return [200,
+                    response_headers,
+                    open(self.qualys_vm_path + '/scans').read()]
+        elif 'fetch' in request.parsed_body['action']:
+            try:
+                response_body = open('{}/{}'.format(
+                                      self.qualys_vm_path,
+                                      request.parsed_body['scan_ref'][0].replace('/', '_'))
+                                    ).read()
+            except:
+                # Can't find the file, just send an empty response
+                response_body = ''
+        return [200, response_headers, response_body]
+
+    def create_qualys_vm_resource(self, framework):
         # Create health check endpoint
-        self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, 'GET', 'msp/about.php'))
+        self.logger.info('Adding mocked {} endpoint GET msp/about.php'.format(framework))
         httpretty.register_uri(
                 httpretty.GET,
-                'https://{}:443/{}'.format(framework, 'msp/about.php'),
+                'https://{}:443/msp/about.php'.format(framework),
                 body='')
-        
-        self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, 'POST', 'api/2.0/fo/scan'))
+
+        self.logger.info('Adding mocked {} endpoint {} {}'.format(framework, 'POST', 'api/2.0/fo/scan'))
         httpretty.register_uri(
-            httpretty.POST, 'https://{}:443/{}'.format(framework, 'api/2.0/fo/scan/'),
-            body=self.qualys_vuln_callback)
+            httpretty.POST, 'https://{}:443/api/2.0/fo/scan/'.format(framework),
+            body=self.qualys_vm_callback)
+
+    def qualys_was_callback(self, request, uri, response_headers):
+        self.logger.info('Simulating response for {} ({})'.format(uri, request.body))
+        report_id = request.parsed_body.split('<WasScan><id>')[1].split('<')[0]
+        response_body = open('{}/create_{}'.format(self.qualys_was_path, report_id)).read()
+        return [200, response_headers, response_body]
+
+    def create_qualys_was_resource(self, framework):
+        for filename in self.get_files('{}/{}'.format(self.mock_dir, framework)):
+            if filename.startswith('POST') or filename.startswith('GET'):
+                method, resource = filename.split('_', 1)
+                resource = resource.replace('_', '/')
+                self.logger.info('Adding mocked {} endpoint {} {}'.format(framework, method, resource))
+                httpretty.register_uri(
+                    getattr(httpretty, method), 'https://{}:443/{}'.format(framework, resource),
+                    body=open('{}/{}/{}'.format(self.mock_dir, framework, filename)).read()
+                )
+
+        self.logger.info('Adding mocked {} endpoint {} {}'.format(framework, 'POST', 'qps/rest/3.0/create/was/report'))
+        httpretty.register_uri(
+            httpretty.POST, 'https://{}:443/qps/rest/3.0/create/was/report'.format(framework),
+            body=self.qualys_was_callback)
+
+    def openvas_callback(self, request, uri, response_headers):
+        self.logger.info('Simulating response for {} ({})'.format(uri, request.body))
+        if request.querystring['cmd'][0] in ['get_reports', 'get_report_formats']:
+            response_body = open('{}/{}'.format(self.openvas_path, request.querystring['cmd'][0])).read()
+
+        if request.querystring['cmd'][0] == 'get_report':
+                response_body = open('{}/report_{}'.format(self.openvas_path, request.querystring['report_id'][0])).read()
+
+        return [200, response_headers, response_body]
+
+    def create_openvas_resource(self, framework):
+        # Create login endpoint
+        httpretty.register_uri(
+            httpretty.POST, 'https://{}:4000/omp'.format(framework),
+            body=open('{}/{}/{}'.format(self.mock_dir, framework, 'login')).read()
+        )
+
+        # Create GET requests endpoint
+        httpretty.register_uri(
+            httpretty.GET, 'https://{}:4000/omp'.format(framework),
+            body=self.openvas_callback
+        )
 
     def mock_endpoints(self):
         for framework in self.get_directories(self.mock_dir):
             if framework in ['nessus', 'tenable']:
                 self.create_nessus_resource(framework)
-            elif framework == 'qualys_vuln':
-                self.qualys_vuln_path = self.mock_dir + '/' + framework
-                self.create_qualys_vuln_resource(framework)
-        httpretty.enable()
+            elif framework == 'qualys_vm':
+                self.qualys_vm_path = self.mock_dir + '/' + framework
+                self.create_qualys_vm_resource(framework)
+            elif framework == 'qualys_was':
+                self.qualys_was_path = self.mock_dir + '/' + framework
+                self.create_qualys_was_resource(framework)
+            elif framework == 'openvas':
+                self.openvas_path = self.mock_dir + '/' + framework
+                self.create_openvas_resource(framework)
+        httpretty.enable()