diff --git a/docker-compose-test.yml b/docker-compose-test.yml new file mode 100644 index 0000000..89e3ee4 --- /dev/null +++ b/docker-compose-test.yml @@ -0,0 +1,94 @@ +version: '2' +services: + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:6.6.0 + container_name: elasticsearch + environment: + - cluster.name=vulnwhisperer + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms1g -Xmx1g" + - xpack.security.enabled=false + - cluster.routing.allocation.disk.threshold_enabled=false + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + mem_limit: 8g + volumes: + - ./docker_data/esdata1:/usr/share/elasticsearch/data + ports: + - 9200:9200 + #restart: always + networks: + esnet: + aliases: + - elasticsearch.local + + kibana: + image: docker.elastic.co/kibana/kibana:6.6.0 + container_name: kibana + environment: + SERVER_NAME: kibana + ELASTICSEARCH_URL: http://elasticsearch:9200 + ports: + - 5601:5601 + depends_on: + - elasticsearch + # volumes: + # - ./kibana-data: + networks: + esnet: + aliases: + - kibana.local + + kibana-config: + image: alpine + container_name: kibana-config + volumes: + - ./resources/elk6/init_kibana.sh:/opt/init_kibana.sh + - ./resources/elk6/kibana_APIonly.json:/opt/kibana_APIonly.json + - ./docker_data/kibana_optimize:/usr/share/kibana/optimize + command: sh -c "apk add --no-cache curl bash && chmod +x /opt/init_kibana.sh && chmod +r /opt/kibana_APIonly.json && cd /opt/ && /bin/bash /opt/init_kibana.sh" # /opt/kibana_APIonly.json" + networks: + esnet: + aliases: + - kibana-config.local + + logstash: + image: docker.elastic.co/logstash/logstash:6.6.0 + container_name: logstash + volumes: + - ./resources/elk6/pipeline/:/usr/share/logstash/pipeline + - ./docker_data/data/:/opt/VulnWhisperer/data + - ./resources/elk6/logstash.yml:/usr/share/logstash/config/logstash.yml + environment: + - xpack.monitoring.enabled=false + depends_on: + - elasticsearch + networks: + esnet: + aliases: + - logstash.local + vulnwhisperer: + image: vulnwhisperer-1.8 + container_name: vulnwhisperer + entrypoint: [ + "vuln_whisperer", + "-c", + "/opt/VulnWhisperer/vulnwhisperer.ini", + "--mock", + "--mock_dir", + "/tests/data" + ] + volumes: + # - /opt/VulnWhisperer/data/:/opt/VulnWhisperer/data + - ./docker_data/data/:/opt/VulnWhisperer/data + - ./configs/test.ini:/opt/VulnWhisperer/vulnwhisperer.ini + - ./tests/data/:/tests/data + network_mode: host + +networks: + esnet: diff --git a/docker-compose.v6.yml b/docker-compose.v6.yml index b5a833e..f53aa0c 100644 --- a/docker-compose.v6.yml +++ b/docker-compose.v6.yml @@ -56,7 +56,7 @@ services: container_name: logstash volumes: - ./resources/elk6/pipeline/:/usr/share/logstash/pipeline - - ./data/:/opt/vulnwhisperer/data + - ./data/:/opt/VulnWhisperer/data #- ./resources/elk6/logstash.yml:/usr/share/logstash/config/logstash.yml environment: - xpack.monitoring.enabled=false @@ -72,12 +72,12 @@ services: entrypoint: [ "vuln_whisperer", "-c", - "/opt/vulnwhisperer/vulnwhisperer.ini" + "/opt/VulnWhisperer/vulnwhisperer.ini" ] volumes: - - /opt/vulnwhisperer/data/:/opt/vulnwhisperer/data - - ./data/:/opt/vulnwhisperer/data - - ./resources/elk6/vulnwhisperer.ini:/opt/vulnwhisperer/vulnwhisperer.ini + - /opt/VulnWhisperer/data/:/opt/VulnWhisperer/data + - ./data/:/opt/VulnWhisperer/data + - ./resources/elk6/vulnwhisperer.ini:/opt/VulnWhisperer/vulnwhisperer.ini network_mode: host volumes: esdata1: diff --git a/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf b/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf index d8d4f92..306bd3d 100644 --- a/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf +++ b/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf @@ -7,13 +7,13 @@ input { file { - path => "/opt/vulnwhisperer/nessus/**/*" + path => "/opt/VulnWhisperer/nessus/**/*" start_position => "beginning" tags => "nessus" type => "nessus" } file { - path => "/opt/vulnwhisperer/tenable/*.csv" + path => "/opt/VulnWhisperer/tenable/*.csv" start_position => "beginning" tags => "tenable" type => "tenable" diff --git a/resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf b/resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf index 504de84..fe98ef8 100644 --- a/resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf +++ b/resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf @@ -6,7 +6,7 @@ input { file { - path => [ "/opt/vulnwhisperer/data/qualys/*.json" , "/opt/vulnwhisperer/data/qualys_web/*.json", "/opt/vulnwhisperer/data/qualys_vuln/*.json" ] + path => [ "/opt/VulnWhisperer/data/qualys/*.json" , "/opt/VulnWhisperer/data/qualys_web/*.json", "/opt/VulnWhisperer/data/qualys_vuln/*.json" ] type => json codec => json start_position => "beginning" diff --git a/resources/elk5-old_compatibility/logstash/3000_openvas.conf b/resources/elk5-old_compatibility/logstash/3000_openvas.conf index f560731..32e889c 100644 --- a/resources/elk5-old_compatibility/logstash/3000_openvas.conf +++ b/resources/elk5-old_compatibility/logstash/3000_openvas.conf @@ -6,7 +6,7 @@ input { file { - path => "/opt/vulnwhisperer/openvas/*.json" + path => "/opt/VulnWhisperer/openvas/*.json" type => json codec => json start_position => "beginning" diff --git a/resources/elk5-old_compatibility/logstash/4000_jira.conf b/resources/elk5-old_compatibility/logstash/4000_jira.conf index e4106c7..03a0b04 100644 --- a/resources/elk5-old_compatibility/logstash/4000_jira.conf +++ b/resources/elk5-old_compatibility/logstash/4000_jira.conf @@ -2,7 +2,7 @@ input { file { - path => "/opt/vulnwhisperer/jira/*.json" + path => "/opt/VulnWhisperer/jira/*.json" type => json codec => json start_position => "beginning" diff --git a/resources/elk6/pipeline/1000_nessus_process_file.conf b/resources/elk6/pipeline/1000_nessus_process_file.conf index 0c64047..0ea00c1 100644 --- a/resources/elk6/pipeline/1000_nessus_process_file.conf +++ b/resources/elk6/pipeline/1000_nessus_process_file.conf @@ -7,14 +7,14 @@ input { file { - path => "/opt/vulnwhisperer/data/nessus/**/*" + path => "/opt/VulnWhisperer/data/nessus/**/*" mode => "read" start_position => "beginning" file_completed_action => "delete" tags => "nessus" } file { - path => "/opt/vulnwhisperer/data/tenable/*.csv" + path => "/opt/VulnWhisperer/data/tenable/*.csv" mode => "read" start_position => "beginning" file_completed_action => "delete" @@ -53,11 +53,13 @@ filter { } #If using filebeats as your source, you will need to replace the "path" field to "source" + # Remove when scan name is included in event (current method is error prone) grok { match => { "path" => "(?[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.csv$" } tag_on_failure => [] } + # TODO remove when @timestamp is included in event date { match => [ "last_updated", "UNIX" ] target => "@timestamp" @@ -169,6 +171,9 @@ filter { output { if "nessus" in [tags] or "tenable" in [tags]{ + stdout { + codec => dots + } elasticsearch { hosts => [ "elasticsearch:9200" ] index => "logstash-vulnwhisperer-%{+YYYY.MM}" diff --git a/resources/elk6/pipeline/2000_qualys_web_scans.conf b/resources/elk6/pipeline/2000_qualys_web_scans.conf index fbf83ee..7c207c7 100644 --- a/resources/elk6/pipeline/2000_qualys_web_scans.conf +++ b/resources/elk6/pipeline/2000_qualys_web_scans.conf @@ -6,7 +6,7 @@ input { file { - path => [ "/opt/vulnwhisperer/data/qualys/*.json" , "/opt/vulnwhisperer/data/qualys_web/*.json", "/opt/vulnwhisperer/data/qualys_vuln/*.json"] + path => [ "/opt/VulnWhisperer/data/qualys/*.json" , "/opt/VulnWhisperer/data/qualys_web/*.json", "/opt/VulnWhisperer/data/qualys_vuln/*.json"] type => json codec => json start_position => "beginning" @@ -98,6 +98,8 @@ filter { target => "last_time_tested" } } + + # TODO remove when @timestamp is included in event date { match => [ "last_updated", "UNIX" ] target => "@timestamp" @@ -147,6 +149,9 @@ filter { } output { if "qualys" in [tags] { + stdout { + codec => dots + } elasticsearch { hosts => [ "elasticsearch:9200" ] index => "logstash-vulnwhisperer-%{+YYYY.MM}" diff --git a/resources/elk6/pipeline/3000_openvas.conf b/resources/elk6/pipeline/3000_openvas.conf index 5fcc56c..4a96ca3 100644 --- a/resources/elk6/pipeline/3000_openvas.conf +++ b/resources/elk6/pipeline/3000_openvas.conf @@ -6,7 +6,7 @@ input { file { - path => "/opt/vulnwhisperer/data/openvas/*.json" + path => "/opt/VulnWhisperer/data/openvas/*.json" type => json codec => json start_position => "beginning" @@ -92,6 +92,8 @@ filter { target => "last_time_tested" } } + + # TODO remove when @timestamp is included in event date { match => [ "last_updated", "UNIX" ] target => "@timestamp" @@ -141,6 +143,9 @@ filter { } output { if "openvas" in [tags] { + stdout { + codec => dots + } elasticsearch { hosts => [ "elasticsearch:9200" ] index => "logstash-vulnwhisperer-%{+YYYY.MM}" diff --git a/resources/elk6/pipeline/4000_jira.conf b/resources/elk6/pipeline/4000_jira.conf index 83030cc..47d978c 100644 --- a/resources/elk6/pipeline/4000_jira.conf +++ b/resources/elk6/pipeline/4000_jira.conf @@ -2,7 +2,7 @@ input { file { - path => "/opt/vulnwhisperer/data/jira/*.json" + path => "/opt/VulnWhisperer/data/jira/*.json" type => json codec => json start_position => "beginning" diff --git a/resources/elk6/vulnwhisperer.ini b/resources/elk6/vulnwhisperer.ini index 2b92761..2e8c687 100644 --- a/resources/elk6/vulnwhisperer.ini +++ b/resources/elk6/vulnwhisperer.ini @@ -4,8 +4,8 @@ hostname=localhost port=8834 username=nessus_username password=nessus_password -write_path=/opt/vulnwhisperer/data/nessus/ -db_path=/opt/vulnwhisperer/database +write_path=/opt/VulnWhisperer/data/nessus/ +db_path=/opt/VulnWhisperer/database trash=false verbose=true @@ -15,7 +15,7 @@ hostname=cloud.tenable.com port=443 username=tenable.io_username password=tenable.io_password -write_path=/opt/vulnwhisperer/data/tenable/ +write_path=/opt/VulnWhisperer/data/tenable/ db_path=/opt/VulnWhisperer/data/database trash=false verbose=true @@ -26,8 +26,8 @@ enabled = true hostname = qualysapi.qg2.apps.qualys.com username = exampleuser password = examplepass -write_path=/opt/vulnwhisperer/data/qualys/ -db_path=/opt/vulnwhisperer/data/database +write_path=/opt/VulnWhisperer/data/qualys/ +db_path=/opt/VulnWhisperer/data/database verbose=true # Set the maximum number of retries each connection should attempt. @@ -42,8 +42,8 @@ enabled = true hostname = qualysapi.qg2.apps.qualys.com username = exampleuser password = examplepass -write_path=/opt/vulnwhisperer/data/qualys/ -db_path=/opt/vulnwhisperer/data/database +write_path=/opt/VulnWhisperer/data/qualys/ +db_path=/opt/VulnWhisperer/data/database verbose=true # Set the maximum number of retries each connection should attempt. @@ -60,8 +60,8 @@ hostname = api.detectify.com username = exampleuser #password variable used as secretKey password = examplepass -write_path =/opt/vulnwhisperer/data/detectify/ -db_path = /opt/vulnwhisperer/data/database +write_path =/opt/VulnWhisperer/data/detectify/ +db_path = /opt/VulnWhisperer/data/database verbose = true [openvas] @@ -70,8 +70,8 @@ hostname = localhost port = 4000 username = exampleuser password = examplepass -write_path=/opt/vulnwhisperer/data/openvas/ -db_path=/opt/vulnwhisperer/data/database +write_path=/opt/VulnWhisperer/data/openvas/ +db_path=/opt/VulnWhisperer/data/database verbose=true #[proxy] @@ -92,8 +92,8 @@ verbose=true hostname = jira-host username = username password = password -write_path = /opt/vulnwhisperer/data/jira/ -db_path = /opt/vulnwhisperer/data/database +write_path = /opt/VulnWhisperer/data/jira/ +db_path = /opt/VulnWhisperer/data/database verbose = true dns_resolv = False