From fc5f9b5b7c45753bb0c894598a4e8b9e7303df83 Mon Sep 17 00:00:00 2001 From: qmontal Date: Mon, 20 Aug 2018 15:20:58 +0200 Subject: [PATCH] Fix docker-compose logstash config (#92) * ignore nessus requests warnings * docker-compose fully working with vulnwhisperer integrated * remove comments docker-compose * documenting docker-compose * Readme corrections * fix after recheck everything works out of the box * fix exits that break the no specified section execution mode * fix docker qualysapi issue, updated README * revert change on deps/qualysapi/qualysapi/util.py (no effect) * temporarily changed Dockerfile link to the working one * fix docker-compose logstash config * permissions needed for logstash container to work * changing default path qualys, there are no folders --- README.md | 14 ++++++++++---- docker-compose.yml | 8 ++++---- docker/2000_qualys_web_scans.conf | 4 ++-- docker/3000_openvas.conf | 2 +- docker/logstash.yml | 11 +++++------ logstash/2000_qualys_web_scans.conf | 2 +- 6 files changed, 23 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index ff50036..a026355 100644 --- a/README.md +++ b/README.md @@ -193,12 +193,14 @@ The docker-compose file has been tested and running on a Ubuntu 18.04 environmen mkdir data && chmod -R 666 data #data/database/report_tracker.db will need 777 to use with local vulnwhisperer ``` otherwise the users running inside the docker containers will not be able to work with it properly. If you don't apply chmod recursively, it will still work to sync the data, but only root use in localhost will have access to the created data (if you run local vulnwhisperer with the same data will break). +- docker/logstash.yml file will need other read/write permissions in order for logstash container to use the configuration file; youll need to run: +```shell +chmod 666 docker/logstash.yml +``` - You will need to rebuild the vulnwhisperer Dockerfile before launching the docker-compose, as by the way it is created right now it doesn't pull the last version of the VulnWhisperer code from Github, due to docker layering inner workings. To do this, the best way is to: ```shell - wget https://raw.githubusercontent.com/qmontal/docker_vulnwhisperer/master/Dockerfile docker build --no-cache -t hasecuritysolutions/docker_vulnwhisperer -f Dockerfile . --network=host - ``` This will create the image hasecuritysolutions/docker_vulnwhisperer:latest from scratch with the latest updates. Will soon fix that with the next VulnWhisperer version. - The vulnwhisperer container inside of docker-compose is using network_mode=host instead of the bridge mode by default; this is due to issues encountered when the container is trying to pull data from your scanners from a different VLAN than the one you currently are. The host network mode uses the DNS and interface from the host itself, fixing those issues, but it breaks the network isolation from the container (this is due to docker creating bridge interfaces to route the traffic, blocking both container's and host's network). If you change this to bridge, you might need to add your DNS to the config in order to resolve internal hostnames. @@ -213,13 +215,17 @@ docker exec -i -t 665b4a1e17b6 /bin/bash #where 665b4a1e17b6 is the container im You can also make sure that all ELK components are working by doing "curl -i host:9200 (elastic)/ host:5601 (kibana) /host:9600 (logstash). WARNING! It is possible that logstash is not exposing to the external network the port but it does to its internal docker network "esnet". - If Kibana is not showing the results, check that you are searching on the whole ES range, as by default it shows logs for the last 15 minutes (you can choose up to last 5 years) - X-Pack has been disabled by default due to the noise, plus being a trial version. You can enable it modifying the docker-compose.yml and docker/logstash.conf files. Logstash.conf contains the default credentials for the X-Pack enabled ES. +- On Logstash container, "/usr/share/logstash/pipeline/" is the default path for pipelines and "/usr/share/logstash/config/" for logstash.yml file, instead of "/etc/logstash/conf.d/" and "/etc/logstash/". +- In order to make vulnwhisperer run periodically, add to crontab the following: +```shell +0 8 * * * /usr/bin/docker-compose run vulnwhisp-vulnwhisperer +``` To launch docker-compose, do: ```shell docker-compose -f docker-compose.yml up ``` -Known issue: Qualys Vuln Management error -> QualysGuard Username: [ERROR] Could not connect to Qualys - EOF when reading a line (working on vulnwhisperer without docker) Running Nightly --------------- @@ -247,7 +253,7 @@ Authors Contributors ------------ - - [@qmontal](https://github.com/qmontal) + - [Quim Montal (@qmontal)](https://github.com/qmontal) AS SEEN ON TV ------------- diff --git a/docker-compose.yml b/docker-compose.yml index 6a21ea5..9369f6f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,10 +40,10 @@ services: image: docker.elastic.co/logstash/logstash:5.6.2 container_name: vulnwhisp-ls1 volumes: - - ./docker/1000_nessus_process_file.conf:/etc/logstash/conf.d/1000_nessus_process_file.conf - - ./docker/2000_qualys_web_scans.conf:/etc/logstash/conf.d/2000_qualys_web_scans.conf - - ./docker/3000_openvas.conf:/etc/logstash/conf.d/3000_openvas.conf - - ./docker/logstash.yml:/etc/logstash/logstash.yml + - ./docker/1000_nessus_process_file.conf:/usr/share/logstash/pipeline/1000_nessus_process_file.conf + - ./docker/2000_qualys_web_scans.conf:/usr/share/logstash/pipeline/2000_qualys_web_scans.conf + - ./docker/3000_openvas.conf:/usr/share/logstash/pipeline/3000_openvas.conf + - ./docker/logstash.yml:/usr/share/logstash/config/logstash.yml - ./data/:/opt/VulnWhisperer/data environment: - xpack.monitoring.enabled=false diff --git a/docker/2000_qualys_web_scans.conf b/docker/2000_qualys_web_scans.conf index ebeb541..9d47a1c 100644 --- a/docker/2000_qualys_web_scans.conf +++ b/docker/2000_qualys_web_scans.conf @@ -6,7 +6,7 @@ input { file { - path => "/opt/VulnWhisperer/data/qualys/scans/**/*.json" + path => "/opt/VulnWhisperer/data/qualys/*.json" type => json codec => json start_position => "beginning" @@ -146,7 +146,7 @@ output { if "qualys" in [tags] { stdout { codec => rubydebug } elasticsearch { - hosts => [ "vulnwhisp_es1.local:9200" ] + hosts => [ "vulnwhisp-es1.local:9200" ] index => "logstash-vulnwhisperer-%{+YYYY.MM}" } } diff --git a/docker/3000_openvas.conf b/docker/3000_openvas.conf index b3ee1d1..1b8c4b3 100644 --- a/docker/3000_openvas.conf +++ b/docker/3000_openvas.conf @@ -139,7 +139,7 @@ output { if "openvas" in [tags] { stdout { codec => rubydebug } elasticsearch { - hosts => [ "vulnwhisp_es1.local:9200" ] + hosts => [ "vulnwhisp-es1.local:9200" ] index => "logstash-vulnwhisperer-%{+YYYY.MM}" } } diff --git a/docker/logstash.yml b/docker/logstash.yml index 1326391..977cac8 100644 --- a/docker/logstash.yml +++ b/docker/logstash.yml @@ -1,6 +1,5 @@ -path.data: /var/lib/logstash -path.config: /etc/logstash/conf.d -#path.logs: /var/log/logstash -xpack.monitoring.elasticsearch.url: [ "vulnwhisp-es1.local:9200" ] -xpack.monitoring.elasticsearch.username: "elastic" -xpack.monitoring.elasticsearch.password: "changeme" +path.config: /usr/share/logstash/pipeline/ +xpack.monitoring.elasticsearch.password: changeme +xpack.monitoring.elasticsearch.url: vulnwhisp-es1.local:9200 +xpack.monitoring.elasticsearch.username: elastic +xpack.monitoring.enabled: false diff --git a/logstash/2000_qualys_web_scans.conf b/logstash/2000_qualys_web_scans.conf index b3bddb8..b330260 100644 --- a/logstash/2000_qualys_web_scans.conf +++ b/logstash/2000_qualys_web_scans.conf @@ -6,7 +6,7 @@ input { file { - path => "/opt/vulnwhisperer/qualys/scans/**/*.json" + path => "/opt/vulnwhisperer/qualys/*.json" type => json codec => json start_position => "beginning"