unify case among config

2019-04-12 17:54:17 +10:00
parent b35da1c79e
commit 0227636c4c
11 changed files with 137 additions and 28 deletions
--- a/docker-compose-test.yml
+++ b/docker-compose-test.yml
@ -0,0 +1,94 @@
+version: '2'
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.6.0
+    container_name: elasticsearch
+    environment:
+      - cluster.name=vulnwhisperer
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
+      - xpack.security.enabled=false
+      - cluster.routing.allocation.disk.threshold_enabled=false
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    mem_limit: 8g
+    volumes:
+      - ./docker_data/esdata1:/usr/share/elasticsearch/data
+    ports:
+      - 9200:9200
+    #restart: always
+    networks:
+      esnet:
+        aliases:
+          - elasticsearch.local
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:6.6.0
+    container_name: kibana
+    environment:
+      SERVER_NAME: kibana
+      ELASTICSEARCH_URL: http://elasticsearch:9200
+    ports:
+      - 5601:5601
+    depends_on:
+      - elasticsearch
+    # volumes:
+    #   - ./kibana-data:
+    networks:
+      esnet:
+        aliases:
+          - kibana.local
+
+  kibana-config:
+    image: alpine
+    container_name: kibana-config
+    volumes:
+      - ./resources/elk6/init_kibana.sh:/opt/init_kibana.sh
+      - ./resources/elk6/kibana_APIonly.json:/opt/kibana_APIonly.json
+      - ./docker_data/kibana_optimize:/usr/share/kibana/optimize
+    command: sh -c "apk add --no-cache curl bash && chmod +x /opt/init_kibana.sh && chmod +r /opt/kibana_APIonly.json && cd /opt/ && /bin/bash /opt/init_kibana.sh" # /opt/kibana_APIonly.json"
+    networks:
+      esnet:
+        aliases:
+          - kibana-config.local
+
+  logstash:
+    image: docker.elastic.co/logstash/logstash:6.6.0
+    container_name: logstash
+    volumes:
+      - ./resources/elk6/pipeline/:/usr/share/logstash/pipeline
+      - ./docker_data/data/:/opt/VulnWhisperer/data
+      - ./resources/elk6/logstash.yml:/usr/share/logstash/config/logstash.yml
+    environment:
+      - xpack.monitoring.enabled=false
+    depends_on:
+      - elasticsearch
+    networks:
+      esnet:
+        aliases:
+          - logstash.local
+  vulnwhisperer:
+    image: vulnwhisperer-1.8
+    container_name: vulnwhisperer
+    entrypoint: [
+                  "vuln_whisperer",
+                  "-c",
+                  "/opt/VulnWhisperer/vulnwhisperer.ini",
+                  "--mock",
+                  "--mock_dir",
+                  "/tests/data"
+    ]    
+    volumes:
+      # - /opt/VulnWhisperer/data/:/opt/VulnWhisperer/data
+      - ./docker_data/data/:/opt/VulnWhisperer/data
+      - ./configs/test.ini:/opt/VulnWhisperer/vulnwhisperer.ini
+      - ./tests/data/:/tests/data
+    network_mode: host
+
+networks:
+  esnet:
--- a/docker-compose.v6.yml
+++ b/docker-compose.v6.yml
@ -56,7 +56,7 @@ services:
    container_name: logstash
    volumes:
      - ./resources/elk6/pipeline/:/usr/share/logstash/pipeline
-      - ./data/:/opt/vulnwhisperer/data
+      - ./data/:/opt/VulnWhisperer/data
      #- ./resources/elk6/logstash.yml:/usr/share/logstash/config/logstash.yml
    environment:
      - xpack.monitoring.enabled=false
@ -72,12 +72,12 @@ services:
    entrypoint: [
                  "vuln_whisperer",
                  "-c",
-                  "/opt/vulnwhisperer/vulnwhisperer.ini"
+                  "/opt/VulnWhisperer/vulnwhisperer.ini"
    ]    
    volumes:
-      - /opt/vulnwhisperer/data/:/opt/vulnwhisperer/data
-      - ./data/:/opt/vulnwhisperer/data
-      - ./resources/elk6/vulnwhisperer.ini:/opt/vulnwhisperer/vulnwhisperer.ini
+      - /opt/VulnWhisperer/data/:/opt/VulnWhisperer/data
+      - ./data/:/opt/VulnWhisperer/data
+      - ./resources/elk6/vulnwhisperer.ini:/opt/VulnWhisperer/vulnwhisperer.ini
    network_mode: host
 volumes:
  esdata1:
--- a/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf
+++ b/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf
@ -7,13 +7,13 @@

 input {
  file {
-    path => "/opt/vulnwhisperer/nessus/**/*"
+    path => "/opt/VulnWhisperer/nessus/**/*"
    start_position => "beginning"
    tags => "nessus"
    type => "nessus"
  }
  file {
-    path => "/opt/vulnwhisperer/tenable/*.csv"
+    path => "/opt/VulnWhisperer/tenable/*.csv"
    start_position => "beginning"
    tags => "tenable"
    type => "tenable"
--- a/resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf
+++ b/resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf
@ -6,7 +6,7 @@

 input {
  file {
-    path => [ "/opt/vulnwhisperer/data/qualys/*.json" , "/opt/vulnwhisperer/data/qualys_web/*.json", "/opt/vulnwhisperer/data/qualys_vuln/*.json" ]
+    path => [ "/opt/VulnWhisperer/data/qualys/*.json" , "/opt/VulnWhisperer/data/qualys_web/*.json", "/opt/VulnWhisperer/data/qualys_vuln/*.json" ]
    type => json
    codec => json
    start_position => "beginning"
--- a/resources/elk5-old_compatibility/logstash/3000_openvas.conf
+++ b/resources/elk5-old_compatibility/logstash/3000_openvas.conf
@ -6,7 +6,7 @@

 input {
  file {
-    path => "/opt/vulnwhisperer/openvas/*.json"
+    path => "/opt/VulnWhisperer/openvas/*.json"
    type => json
    codec => json
    start_position => "beginning"
--- a/resources/elk5-old_compatibility/logstash/4000_jira.conf
+++ b/resources/elk5-old_compatibility/logstash/4000_jira.conf
@ -2,7 +2,7 @@

 input {
  file {
-    path => "/opt/vulnwhisperer/jira/*.json"
+    path => "/opt/VulnWhisperer/jira/*.json"
    type => json
    codec => json
    start_position => "beginning"
--- a/resources/elk6/pipeline/1000_nessus_process_file.conf
+++ b/resources/elk6/pipeline/1000_nessus_process_file.conf
@ -7,14 +7,14 @@

 input {
  file {
-    path => "/opt/vulnwhisperer/data/nessus/**/*"
+    path => "/opt/VulnWhisperer/data/nessus/**/*"
    mode => "read"
    start_position => "beginning"
    file_completed_action => "delete"
    tags => "nessus"
  }
  file {
-    path => "/opt/vulnwhisperer/data/tenable/*.csv"
+    path => "/opt/VulnWhisperer/data/tenable/*.csv"
    mode => "read"
    start_position => "beginning"
    file_completed_action => "delete"
@ -53,11 +53,13 @@ filter {
    }

    #If using filebeats as your source, you will need to replace the "path" field to "source"
+    # Remove when scan name is included in event (current method is error prone)
    grok {
      match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.csv$" }
      tag_on_failure => []
    }

+    # TODO remove when @timestamp is included in event
    date {
      match => [ "last_updated", "UNIX" ]
      target => "@timestamp"
@ -169,6 +171,9 @@ filter {

 output {
  if "nessus" in [tags] or "tenable" in [tags]{
+    stdout {
+      codec => dots
+    }
    elasticsearch {
      hosts => [ "elasticsearch:9200" ]
      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
--- a/resources/elk6/pipeline/2000_qualys_web_scans.conf
+++ b/resources/elk6/pipeline/2000_qualys_web_scans.conf
@ -6,7 +6,7 @@

 input {
  file {
-    path => [ "/opt/vulnwhisperer/data/qualys/*.json" , "/opt/vulnwhisperer/data/qualys_web/*.json", "/opt/vulnwhisperer/data/qualys_vuln/*.json"] 
+    path => [ "/opt/VulnWhisperer/data/qualys/*.json" , "/opt/VulnWhisperer/data/qualys_web/*.json", "/opt/VulnWhisperer/data/qualys_vuln/*.json"] 
    type => json
    codec => json
    start_position => "beginning"
@ -98,6 +98,8 @@ filter {
        target => "last_time_tested"
      }
    }
+
+    # TODO remove when @timestamp is included in event
    date {
      match => [ "last_updated", "UNIX" ]
      target => "@timestamp"
@ -147,6 +149,9 @@ filter {
 }
 output {
  if "qualys" in [tags] {
+    stdout {
+      codec => dots
+    }
    elasticsearch {
      hosts => [ "elasticsearch:9200" ]
      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
--- a/resources/elk6/pipeline/3000_openvas.conf
+++ b/resources/elk6/pipeline/3000_openvas.conf
@ -6,7 +6,7 @@

 input {
  file {
-    path => "/opt/vulnwhisperer/data/openvas/*.json"
+    path => "/opt/VulnWhisperer/data/openvas/*.json"
    type => json
    codec => json
    start_position => "beginning"
@ -92,6 +92,8 @@ filter {
        target => "last_time_tested"
      }
    }
+
+    # TODO remove when @timestamp is included in event
    date {
      match => [ "last_updated", "UNIX" ]
      target => "@timestamp"
@ -141,6 +143,9 @@ filter {
 }
 output {
  if "openvas" in [tags] {
+    stdout {
+      codec => dots
+    }
    elasticsearch {
      hosts => [ "elasticsearch:9200" ]
      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
--- a/resources/elk6/pipeline/4000_jira.conf
+++ b/resources/elk6/pipeline/4000_jira.conf
@ -2,7 +2,7 @@

 input {
  file {
-    path => "/opt/vulnwhisperer/data/jira/*.json"
+    path => "/opt/VulnWhisperer/data/jira/*.json"
    type => json
    codec => json
    start_position => "beginning"
--- a/resources/elk6/vulnwhisperer.ini
+++ b/resources/elk6/vulnwhisperer.ini
@ -4,8 +4,8 @@ hostname=localhost
 port=8834
 username=nessus_username
 password=nessus_password
-write_path=/opt/vulnwhisperer/data/nessus/
-db_path=/opt/vulnwhisperer/database
+write_path=/opt/VulnWhisperer/data/nessus/
+db_path=/opt/VulnWhisperer/database
 trash=false
 verbose=true

@ -15,7 +15,7 @@ hostname=cloud.tenable.com
 port=443
 username=tenable.io_username
 password=tenable.io_password
-write_path=/opt/vulnwhisperer/data/tenable/
+write_path=/opt/VulnWhisperer/data/tenable/
 db_path=/opt/VulnWhisperer/data/database
 trash=false
 verbose=true
@ -26,8 +26,8 @@ enabled = true
 hostname = qualysapi.qg2.apps.qualys.com
 username = exampleuser
 password = examplepass
-write_path=/opt/vulnwhisperer/data/qualys/
-db_path=/opt/vulnwhisperer/data/database
+write_path=/opt/VulnWhisperer/data/qualys/
+db_path=/opt/VulnWhisperer/data/database
 verbose=true

 # Set the maximum number of retries each connection should attempt.
@ -42,8 +42,8 @@ enabled = true
 hostname = qualysapi.qg2.apps.qualys.com
 username = exampleuser
 password = examplepass
-write_path=/opt/vulnwhisperer/data/qualys/
-db_path=/opt/vulnwhisperer/data/database
+write_path=/opt/VulnWhisperer/data/qualys/
+db_path=/opt/VulnWhisperer/data/database
 verbose=true

 # Set the maximum number of retries each connection should attempt.
@ -60,8 +60,8 @@ hostname = api.detectify.com
 username = exampleuser
 #password variable used as secretKey
 password = examplepass
-write_path =/opt/vulnwhisperer/data/detectify/
-db_path = /opt/vulnwhisperer/data/database
+write_path =/opt/VulnWhisperer/data/detectify/
+db_path = /opt/VulnWhisperer/data/database
 verbose = true

 [openvas]
@ -70,8 +70,8 @@ hostname = localhost
 port = 4000
 username = exampleuser
 password = examplepass
-write_path=/opt/vulnwhisperer/data/openvas/
-db_path=/opt/vulnwhisperer/data/database
+write_path=/opt/VulnWhisperer/data/openvas/
+db_path=/opt/VulnWhisperer/data/database
 verbose=true

 #[proxy]
@ -92,8 +92,8 @@ verbose=true
 hostname = jira-host
 username = username
 password = password
-write_path = /opt/vulnwhisperer/data/jira/
-db_path = /opt/vulnwhisperer/data/database
+write_path = /opt/VulnWhisperer/data/jira/
+db_path = /opt/VulnWhisperer/data/database
 verbose = true
 dns_resolv = False