cherry pick #0227636

docker-compose-test.yml

@@ -0,0 +1,94 @@
+version: '2'
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.6.0
+    container_name: elasticsearch
+    environment:
+      - cluster.name=vulnwhisperer
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
+      - xpack.security.enabled=false
+      - cluster.routing.allocation.disk.threshold_enabled=false
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    mem_limit: 8g
+    volumes:
+      - ./docker_data/esdata1:/usr/share/elasticsearch/data
+    ports:
+      - 9200:9200
+    #restart: always
+    networks:
+      esnet:
+        aliases:
+          - elasticsearch.local
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:6.6.0
+    container_name: kibana
+    environment:
+      SERVER_NAME: kibana
+      ELASTICSEARCH_URL: http://elasticsearch:9200
+    ports:
+      - 5601:5601
+    depends_on:
+      - elasticsearch
+    # volumes:
+    #   - ./kibana-data:
+    networks:
+      esnet:
+        aliases:
+          - kibana.local
+
+  kibana-config:
+    image: alpine
+    container_name: kibana-config
+    volumes:
+      - ./resources/elk6/init_kibana.sh:/opt/init_kibana.sh
+      - ./resources/elk6/kibana_APIonly.json:/opt/kibana_APIonly.json
+      - ./docker_data/kibana_optimize:/usr/share/kibana/optimize
+    command: sh -c "apk add --no-cache curl bash && chmod +x /opt/init_kibana.sh && chmod +r /opt/kibana_APIonly.json && cd /opt/ && /bin/bash /opt/init_kibana.sh" # /opt/kibana_APIonly.json"
+    networks:
+      esnet:
+        aliases:
+          - kibana-config.local
+
+  logstash:
+    image: docker.elastic.co/logstash/logstash:6.6.0
+    container_name: logstash
+    volumes:
+      - ./resources/elk6/pipeline/:/usr/share/logstash/pipeline
+      - ./docker_data/data/:/opt/VulnWhisperer/data
+      - ./resources/elk6/logstash.yml:/usr/share/logstash/config/logstash.yml
+    environment:
+      - xpack.monitoring.enabled=false
+    depends_on:
+      - elasticsearch
+    networks:
+      esnet:
+        aliases:
+          - logstash.local
+  vulnwhisperer:
+    image: vulnwhisperer-1.8
+    container_name: vulnwhisperer
+    entrypoint: [
+                  "vuln_whisperer",
+                  "-c",
+                  "/opt/VulnWhisperer/vulnwhisperer.ini",
+                  "--mock",
+                  "--mock_dir",
+                  "/tests/data"
+    ]    
+    volumes:
+      # - /opt/VulnWhisperer/data/:/opt/VulnWhisperer/data
+      - ./docker_data/data/:/opt/VulnWhisperer/data
+      - ./configs/test.ini:/opt/VulnWhisperer/vulnwhisperer.ini
+      - ./tests/data/:/tests/data
+    network_mode: host
+
+networks:
+  esnet:

docker-compose.v6.yml

@@ -56,7 +56,7 @@ services:
     container_name: logstash
     volumes:
       - ./resources/elk6/pipeline/:/usr/share/logstash/pipeline
-      - ./data/:/opt/vulnwhisperer/data
+      - ./data/:/opt/VulnWhisperer/data
       #- ./resources/elk6/logstash.yml:/usr/share/logstash/config/logstash.yml
     environment:
       - xpack.monitoring.enabled=false
@@ -72,12 +72,12 @@ services:
     entrypoint: [
                   "vuln_whisperer",
                   "-c",
-                  "/opt/vulnwhisperer/vulnwhisperer.ini"
+                  "/opt/VulnWhisperer/vulnwhisperer.ini"
     ]    
     volumes:
-      - /opt/vulnwhisperer/data/:/opt/vulnwhisperer/data
+      - /opt/VulnWhisperer/data/:/opt/VulnWhisperer/data
-      - ./data/:/opt/vulnwhisperer/data
+      - ./data/:/opt/VulnWhisperer/data
-      - ./resources/elk6/vulnwhisperer.ini:/opt/vulnwhisperer/vulnwhisperer.ini
+      - ./resources/elk6/vulnwhisperer.ini:/opt/VulnWhisperer/vulnwhisperer.ini
     network_mode: host
 volumes:
   esdata1:

resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf

@@ -6,7 +6,7 @@
 
 input {
   file {
-    path => [ "/opt/vulnwhisperer/data/qualys/*.json" , "/opt/vulnwhisperer/data/qualys_web/*.json", "/opt/vulnwhisperer/data/qualys_vuln/*.json" ]
+    path => [ "/opt/VulnWhisperer/data/qualys/*.json" , "/opt/VulnWhisperer/data/qualys_web/*.json", "/opt/VulnWhisperer/data/qualys_vuln/*.json" ]
     type => json
     codec => json
     start_position => "beginning"

resources/elk5-old_compatibility/logstash/3000_openvas.conf

@@ -6,7 +6,7 @@
 
 input {
   file {
-    path => "/opt/vulnwhisperer/openvas/*.json"
+    path => "/opt/VulnWhisperer/openvas/*.json"
     type => json
     codec => json
     start_position => "beginning"

resources/elk5-old_compatibility/logstash/4000_jira.conf

@@ -2,7 +2,7 @@
 
 input {
   file {
-    path => "/opt/vulnwhisperer/jira/*.json"
+    path => "/opt/VulnWhisperer/jira/*.json"
     type => json
     codec => json
     start_position => "beginning"

resources/elk6/pipeline/1000_nessus_process_file.conf

@@ -26,11 +26,13 @@ filter {
   if "nessus" in [tags] or "tenable" in [tags] {
 
     #If using filebeats as your source, you will need to replace the "path" field to "source"
+    # Remove when scan name is included in event (current method is error prone)
     grok {
       match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.(csv|json)$" }
       tag_on_failure => []
     }
 
+    # TODO remove when @timestamp is included in event
     date {
       match => [ "last_updated", "UNIX" ]
       target => "@timestamp"
@@ -142,6 +144,9 @@ filter {
 
 output {
   if "nessus" in [tags] or "tenable" in [tags]{
+    stdout {
+      codec => dots
+    }
     elasticsearch {
       hosts => [ "elasticsearch:9200" ]
       index => "logstash-vulnwhisperer-%{+YYYY.MM}"

resources/elk6/pipeline/2000_qualys_web_scans.conf

@@ -6,7 +6,7 @@
 
 input {
   file {
-    path => [ "/opt/vulnwhisperer/data/qualys/*.json" , "/opt/vulnwhisperer/data/qualys_web/*.json", "/opt/vulnwhisperer/data/qualys_vuln/*.json"] 
+    path => [ "/opt/VulnWhisperer/data/qualys/*.json" , "/opt/VulnWhisperer/data/qualys_web/*.json", "/opt/VulnWhisperer/data/qualys_vuln/*.json"] 
     type => json
     codec => json
     start_position => "beginning"
@@ -98,6 +98,8 @@ filter {
         target => "last_time_tested"
       }
     }
+
+    # TODO remove when @timestamp is included in event
     date {
       match => [ "last_updated", "UNIX" ]
       target => "@timestamp"
@@ -147,6 +149,9 @@ filter {
 }
 output {
   if "qualys" in [tags] {
+    stdout {
+      codec => dots
+    }
     elasticsearch {
       hosts => [ "elasticsearch:9200" ]
       index => "logstash-vulnwhisperer-%{+YYYY.MM}"

resources/elk6/pipeline/3000_openvas.conf

@@ -6,7 +6,7 @@
 
 input {
   file {
-    path => "/opt/vulnwhisperer/data/openvas/*.json"
+    path => "/opt/VulnWhisperer/data/openvas/*.json"
     type => json
     codec => json
     start_position => "beginning"
@@ -92,6 +92,8 @@ filter {
         target => "last_time_tested"
       }
     }
+
+    # TODO remove when @timestamp is included in event
     date {
       match => [ "last_updated", "UNIX" ]
       target => "@timestamp"
@@ -141,6 +143,9 @@ filter {
 }
 output {
   if "openvas" in [tags] {
+    stdout {
+      codec => dots
+    }
     elasticsearch {
       hosts => [ "elasticsearch:9200" ]
       index => "logstash-vulnwhisperer-%{+YYYY.MM}"

resources/elk6/pipeline/4000_jira.conf

@@ -2,7 +2,7 @@
 
 input {
   file {
-    path => "/opt/vulnwhisperer/data/jira/*.json"
+    path => "/opt/VulnWhisperer/data/jira/*.json"
     type => json
     codec => json
     start_position => "beginning"

resources/elk6/vulnwhisperer.ini

@@ -4,8 +4,8 @@ hostname=localhost
 port=8834
 username=nessus_username
 password=nessus_password
-write_path=/opt/vulnwhisperer/data/nessus/
+write_path=/opt/VulnWhisperer/data/nessus/
-db_path=/opt/vulnwhisperer/database
+db_path=/opt/VulnWhisperer/database
 trash=false
 verbose=true
 
@@ -15,7 +15,7 @@ hostname=cloud.tenable.com
 port=443
 username=tenable.io_username
 password=tenable.io_password
-write_path=/opt/vulnwhisperer/data/tenable/
+write_path=/opt/VulnWhisperer/data/tenable/
 db_path=/opt/VulnWhisperer/data/database
 trash=false
 verbose=true
@@ -26,8 +26,8 @@ enabled = true
 hostname = qualysapi.qg2.apps.qualys.com
 username = exampleuser
 password = examplepass
-write_path=/opt/vulnwhisperer/data/qualys/
+write_path=/opt/VulnWhisperer/data/qualys/
-db_path=/opt/vulnwhisperer/data/database
+db_path=/opt/VulnWhisperer/data/database
 verbose=true
 
 # Set the maximum number of retries each connection should attempt.
@@ -42,8 +42,8 @@ enabled = true
 hostname = qualysapi.qg2.apps.qualys.com
 username = exampleuser
 password = examplepass
-write_path=/opt/vulnwhisperer/data/qualys/
+write_path=/opt/VulnWhisperer/data/qualys/
-db_path=/opt/vulnwhisperer/data/database
+db_path=/opt/VulnWhisperer/data/database
 verbose=true
 
 # Set the maximum number of retries each connection should attempt.
@@ -60,8 +60,8 @@ hostname = api.detectify.com
 username = exampleuser
 #password variable used as secretKey
 password = examplepass
-write_path =/opt/vulnwhisperer/data/detectify/
+write_path =/opt/VulnWhisperer/data/detectify/
-db_path = /opt/vulnwhisperer/data/database
+db_path = /opt/VulnWhisperer/data/database
 verbose = true
 
 [openvas]
@@ -70,8 +70,8 @@ hostname = localhost
 port = 4000
 username = exampleuser
 password = examplepass
-write_path=/opt/vulnwhisperer/data/openvas/
+write_path=/opt/VulnWhisperer/data/openvas/
-db_path=/opt/vulnwhisperer/data/database
+db_path=/opt/VulnWhisperer/data/database
 verbose=true
 
 #[proxy]
@@ -92,8 +92,8 @@ verbose=true
 hostname = jira-host
 username = username
 password = password
-write_path = /opt/vulnwhisperer/data/jira/
+write_path = /opt/VulnWhisperer/data/jira/
-db_path = /opt/vulnwhisperer/data/database
+db_path = /opt/VulnWhisperer/data/database
 verbose = true
 dns_resolv = False