From 90908bd0c6bde38942be41316e05548ef230ba74 Mon Sep 17 00:00:00 2001 From: pemontto Date: Fri, 12 Apr 2019 11:39:49 +1000 Subject: [PATCH 1/6] Remove deps from docker image --- Dockerfile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 64d75a8..606e6a1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,14 +13,11 @@ COPY requirements.txt requirements.txt COPY setup.py setup.py COPY vulnwhisp/ vulnwhisp/ COPY bin/ bin/ -COPY deps/ deps/ COPY configs/frameworks_example.ini frameworks_example.ini RUN python setup.py clean --all RUN pip install -r requirements.txt -WORKDIR /opt/VulnWhisperer/deps/qualysapi -RUN python setup.py install WORKDIR /opt/VulnWhisperer RUN python setup.py install From 8433055f174530fbb61612177b1df16c57b3a8ad Mon Sep 17 00:00:00 2001 From: pemontto Date: Fri, 12 Apr 2019 11:40:01 +1000 Subject: [PATCH 2/6] Fix more unicode issues --- vulnwhisp/vulnwhisp.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index 782af0e..5903630 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -412,7 +412,7 @@ class vulnWhispererNessus(vulnWhispererBase): history_id, norm_time, 'csv') repls = (('\\', '_'), ('/', '_'), (' ', '_')) file_name = reduce(lambda a, kv: a.replace(*kv), repls, file_name) - relative_path_name = self.path_check(folder_name + '/' + file_name) + relative_path_name = self.path_check(folder_name + '/' + file_name).encode('utf8') if os.path.isfile(relative_path_name): if self.develop: @@ -581,7 +581,7 @@ class vulnWhispererQualys(vulnWhispererBase): + '_{last_updated}'.format(last_updated=launched_date) \ + '.{extension}'.format(extension=output_format) - relative_path_name = self.path_check(report_name) + relative_path_name = self.path_check(report_name).encode('utf8') if os.path.isfile(relative_path_name): #TODO Possibly make this optional to sync directories @@ -743,7 +743,7 @@ class vulnWhispererOpenVAS(vulnWhispererBase): report_name = 'openvas_scan_{scan_name}_{last_updated}.{extension}'.format(scan_name=scan_name, last_updated=launched_date, extension=output_format) - relative_path_name = self.path_check(report_name) + relative_path_name = self.path_check(report_name).encode('utf8') scan_reference = report_id if os.path.isfile(relative_path_name): @@ -861,7 +861,7 @@ class vulnWhispererQualysVuln(vulnWhispererBase): + '_{last_updated}'.format(last_updated=launched_date) \ + '.json' - relative_path_name = self.path_check(report_name) + relative_path_name = self.path_check(report_name).encode('utf8') if os.path.isfile(relative_path_name): #TODO Possibly make this optional to sync directories From 668efe2b7afe418f3b587331cc72fa4a4ab49cdc Mon Sep 17 00:00:00 2001 From: pemontto Date: Fri, 12 Apr 2019 11:44:04 +1000 Subject: [PATCH 3/6] Add extra test case --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index c412177..2452b5b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,6 +23,8 @@ script: # Test successful scan download and parsing - rm -rf /tmp/VulnWhisperer - vuln_whisperer -c configs/test.ini --mock --mock_dir ${TEST_PATH} + # Run a second time with no scans to import + - vuln_whisperer -c configs/test.ini --mock --mock_dir ${TEST_PATH} # Test one failed scan - rm -rf /tmp/VulnWhisperer - rm -f ${TEST_PATH}/nessus/GET_scans_exports_164_download From b35da1c79efb018b344fad592e8252c3f118694b Mon Sep 17 00:00:00 2001 From: pemontto Date: Fri, 12 Apr 2019 17:51:15 +1000 Subject: [PATCH 4/6] reduce docker layers and support test data --- Dockerfile | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 606e6a1..a2806ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,10 +2,10 @@ FROM centos:latest MAINTAINER Justin Henderson justin@hasecuritysolutions.com -RUN yum update -y -RUN yum install -y python python-devel git gcc -RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py -RUN python get-pip.py +RUN yum update -y && \ + yum install -y python python-devel git gcc && \ + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python get-pip.py WORKDIR /opt/VulnWhisperer @@ -15,12 +15,13 @@ COPY vulnwhisp/ vulnwhisp/ COPY bin/ bin/ COPY configs/frameworks_example.ini frameworks_example.ini -RUN python setup.py clean --all -RUN pip install -r requirements.txt +RUN python setup.py clean --all && \ + pip install -r requirements.txt WORKDIR /opt/VulnWhisperer -RUN python setup.py install +RUN python setup.py install && \ + ln -s /opt/VulnWhisperer /tmp/VulnWhisperer CMD vuln_whisperer -c /opt/VulnWhisperer/frameworks_example.ini From 0227636c4cd3c9d84d61e65ee8e61b4c515e6051 Mon Sep 17 00:00:00 2001 From: pemontto Date: Fri, 12 Apr 2019 17:54:17 +1000 Subject: [PATCH 5/6] unify case among config --- docker-compose-test.yml | 94 +++++++++++++++++++ docker-compose.v6.yml | 10 +- .../logstash/1000_nessus_process_file.conf | 4 +- .../logstash/2000_qualys_web_scans.conf | 2 +- .../logstash/3000_openvas.conf | 2 +- .../logstash/4000_jira.conf | 2 +- .../pipeline/1000_nessus_process_file.conf | 9 +- .../elk6/pipeline/2000_qualys_web_scans.conf | 7 +- resources/elk6/pipeline/3000_openvas.conf | 7 +- resources/elk6/pipeline/4000_jira.conf | 2 +- resources/elk6/vulnwhisperer.ini | 26 ++--- 11 files changed, 137 insertions(+), 28 deletions(-) create mode 100644 docker-compose-test.yml diff --git a/docker-compose-test.yml b/docker-compose-test.yml new file mode 100644 index 0000000..89e3ee4 --- /dev/null +++ b/docker-compose-test.yml @@ -0,0 +1,94 @@ +version: '2' +services: + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:6.6.0 + container_name: elasticsearch + environment: + - cluster.name=vulnwhisperer + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms1g -Xmx1g" + - xpack.security.enabled=false + - cluster.routing.allocation.disk.threshold_enabled=false + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 + hard: 65536 + mem_limit: 8g + volumes: + - ./docker_data/esdata1:/usr/share/elasticsearch/data + ports: + - 9200:9200 + #restart: always + networks: + esnet: + aliases: + - elasticsearch.local + + kibana: + image: docker.elastic.co/kibana/kibana:6.6.0 + container_name: kibana + environment: + SERVER_NAME: kibana + ELASTICSEARCH_URL: http://elasticsearch:9200 + ports: + - 5601:5601 + depends_on: + - elasticsearch + # volumes: + # - ./kibana-data: + networks: + esnet: + aliases: + - kibana.local + + kibana-config: + image: alpine + container_name: kibana-config + volumes: + - ./resources/elk6/init_kibana.sh:/opt/init_kibana.sh + - ./resources/elk6/kibana_APIonly.json:/opt/kibana_APIonly.json + - ./docker_data/kibana_optimize:/usr/share/kibana/optimize + command: sh -c "apk add --no-cache curl bash && chmod +x /opt/init_kibana.sh && chmod +r /opt/kibana_APIonly.json && cd /opt/ && /bin/bash /opt/init_kibana.sh" # /opt/kibana_APIonly.json" + networks: + esnet: + aliases: + - kibana-config.local + + logstash: + image: docker.elastic.co/logstash/logstash:6.6.0 + container_name: logstash + volumes: + - ./resources/elk6/pipeline/:/usr/share/logstash/pipeline + - ./docker_data/data/:/opt/VulnWhisperer/data + - ./resources/elk6/logstash.yml:/usr/share/logstash/config/logstash.yml + environment: + - xpack.monitoring.enabled=false + depends_on: + - elasticsearch + networks: + esnet: + aliases: + - logstash.local + vulnwhisperer: + image: vulnwhisperer-1.8 + container_name: vulnwhisperer + entrypoint: [ + "vuln_whisperer", + "-c", + "/opt/VulnWhisperer/vulnwhisperer.ini", + "--mock", + "--mock_dir", + "/tests/data" + ] + volumes: + # - /opt/VulnWhisperer/data/:/opt/VulnWhisperer/data + - ./docker_data/data/:/opt/VulnWhisperer/data + - ./configs/test.ini:/opt/VulnWhisperer/vulnwhisperer.ini + - ./tests/data/:/tests/data + network_mode: host + +networks: + esnet: diff --git a/docker-compose.v6.yml b/docker-compose.v6.yml index b5a833e..f53aa0c 100644 --- a/docker-compose.v6.yml +++ b/docker-compose.v6.yml @@ -56,7 +56,7 @@ services: container_name: logstash volumes: - ./resources/elk6/pipeline/:/usr/share/logstash/pipeline - - ./data/:/opt/vulnwhisperer/data + - ./data/:/opt/VulnWhisperer/data #- ./resources/elk6/logstash.yml:/usr/share/logstash/config/logstash.yml environment: - xpack.monitoring.enabled=false @@ -72,12 +72,12 @@ services: entrypoint: [ "vuln_whisperer", "-c", - "/opt/vulnwhisperer/vulnwhisperer.ini" + "/opt/VulnWhisperer/vulnwhisperer.ini" ] volumes: - - /opt/vulnwhisperer/data/:/opt/vulnwhisperer/data - - ./data/:/opt/vulnwhisperer/data - - ./resources/elk6/vulnwhisperer.ini:/opt/vulnwhisperer/vulnwhisperer.ini + - /opt/VulnWhisperer/data/:/opt/VulnWhisperer/data + - ./data/:/opt/VulnWhisperer/data + - ./resources/elk6/vulnwhisperer.ini:/opt/VulnWhisperer/vulnwhisperer.ini network_mode: host volumes: esdata1: diff --git a/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf b/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf index d8d4f92..306bd3d 100644 --- a/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf +++ b/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf @@ -7,13 +7,13 @@ input { file { - path => "/opt/vulnwhisperer/nessus/**/*" + path => "/opt/VulnWhisperer/nessus/**/*" start_position => "beginning" tags => "nessus" type => "nessus" } file { - path => "/opt/vulnwhisperer/tenable/*.csv" + path => "/opt/VulnWhisperer/tenable/*.csv" start_position => "beginning" tags => "tenable" type => "tenable" diff --git a/resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf b/resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf index 504de84..fe98ef8 100644 --- a/resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf +++ b/resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf @@ -6,7 +6,7 @@ input { file { - path => [ "/opt/vulnwhisperer/data/qualys/*.json" , "/opt/vulnwhisperer/data/qualys_web/*.json", "/opt/vulnwhisperer/data/qualys_vuln/*.json" ] + path => [ "/opt/VulnWhisperer/data/qualys/*.json" , "/opt/VulnWhisperer/data/qualys_web/*.json", "/opt/VulnWhisperer/data/qualys_vuln/*.json" ] type => json codec => json start_position => "beginning" diff --git a/resources/elk5-old_compatibility/logstash/3000_openvas.conf b/resources/elk5-old_compatibility/logstash/3000_openvas.conf index f560731..32e889c 100644 --- a/resources/elk5-old_compatibility/logstash/3000_openvas.conf +++ b/resources/elk5-old_compatibility/logstash/3000_openvas.conf @@ -6,7 +6,7 @@ input { file { - path => "/opt/vulnwhisperer/openvas/*.json" + path => "/opt/VulnWhisperer/openvas/*.json" type => json codec => json start_position => "beginning" diff --git a/resources/elk5-old_compatibility/logstash/4000_jira.conf b/resources/elk5-old_compatibility/logstash/4000_jira.conf index e4106c7..03a0b04 100644 --- a/resources/elk5-old_compatibility/logstash/4000_jira.conf +++ b/resources/elk5-old_compatibility/logstash/4000_jira.conf @@ -2,7 +2,7 @@ input { file { - path => "/opt/vulnwhisperer/jira/*.json" + path => "/opt/VulnWhisperer/jira/*.json" type => json codec => json start_position => "beginning" diff --git a/resources/elk6/pipeline/1000_nessus_process_file.conf b/resources/elk6/pipeline/1000_nessus_process_file.conf index 0c64047..0ea00c1 100644 --- a/resources/elk6/pipeline/1000_nessus_process_file.conf +++ b/resources/elk6/pipeline/1000_nessus_process_file.conf @@ -7,14 +7,14 @@ input { file { - path => "/opt/vulnwhisperer/data/nessus/**/*" + path => "/opt/VulnWhisperer/data/nessus/**/*" mode => "read" start_position => "beginning" file_completed_action => "delete" tags => "nessus" } file { - path => "/opt/vulnwhisperer/data/tenable/*.csv" + path => "/opt/VulnWhisperer/data/tenable/*.csv" mode => "read" start_position => "beginning" file_completed_action => "delete" @@ -53,11 +53,13 @@ filter { } #If using filebeats as your source, you will need to replace the "path" field to "source" + # Remove when scan name is included in event (current method is error prone) grok { match => { "path" => "(?[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.csv$" } tag_on_failure => [] } + # TODO remove when @timestamp is included in event date { match => [ "last_updated", "UNIX" ] target => "@timestamp" @@ -169,6 +171,9 @@ filter { output { if "nessus" in [tags] or "tenable" in [tags]{ + stdout { + codec => dots + } elasticsearch { hosts => [ "elasticsearch:9200" ] index => "logstash-vulnwhisperer-%{+YYYY.MM}" diff --git a/resources/elk6/pipeline/2000_qualys_web_scans.conf b/resources/elk6/pipeline/2000_qualys_web_scans.conf index fbf83ee..7c207c7 100644 --- a/resources/elk6/pipeline/2000_qualys_web_scans.conf +++ b/resources/elk6/pipeline/2000_qualys_web_scans.conf @@ -6,7 +6,7 @@ input { file { - path => [ "/opt/vulnwhisperer/data/qualys/*.json" , "/opt/vulnwhisperer/data/qualys_web/*.json", "/opt/vulnwhisperer/data/qualys_vuln/*.json"] + path => [ "/opt/VulnWhisperer/data/qualys/*.json" , "/opt/VulnWhisperer/data/qualys_web/*.json", "/opt/VulnWhisperer/data/qualys_vuln/*.json"] type => json codec => json start_position => "beginning" @@ -98,6 +98,8 @@ filter { target => "last_time_tested" } } + + # TODO remove when @timestamp is included in event date { match => [ "last_updated", "UNIX" ] target => "@timestamp" @@ -147,6 +149,9 @@ filter { } output { if "qualys" in [tags] { + stdout { + codec => dots + } elasticsearch { hosts => [ "elasticsearch:9200" ] index => "logstash-vulnwhisperer-%{+YYYY.MM}" diff --git a/resources/elk6/pipeline/3000_openvas.conf b/resources/elk6/pipeline/3000_openvas.conf index 5fcc56c..4a96ca3 100644 --- a/resources/elk6/pipeline/3000_openvas.conf +++ b/resources/elk6/pipeline/3000_openvas.conf @@ -6,7 +6,7 @@ input { file { - path => "/opt/vulnwhisperer/data/openvas/*.json" + path => "/opt/VulnWhisperer/data/openvas/*.json" type => json codec => json start_position => "beginning" @@ -92,6 +92,8 @@ filter { target => "last_time_tested" } } + + # TODO remove when @timestamp is included in event date { match => [ "last_updated", "UNIX" ] target => "@timestamp" @@ -141,6 +143,9 @@ filter { } output { if "openvas" in [tags] { + stdout { + codec => dots + } elasticsearch { hosts => [ "elasticsearch:9200" ] index => "logstash-vulnwhisperer-%{+YYYY.MM}" diff --git a/resources/elk6/pipeline/4000_jira.conf b/resources/elk6/pipeline/4000_jira.conf index 83030cc..47d978c 100644 --- a/resources/elk6/pipeline/4000_jira.conf +++ b/resources/elk6/pipeline/4000_jira.conf @@ -2,7 +2,7 @@ input { file { - path => "/opt/vulnwhisperer/data/jira/*.json" + path => "/opt/VulnWhisperer/data/jira/*.json" type => json codec => json start_position => "beginning" diff --git a/resources/elk6/vulnwhisperer.ini b/resources/elk6/vulnwhisperer.ini index 2b92761..2e8c687 100644 --- a/resources/elk6/vulnwhisperer.ini +++ b/resources/elk6/vulnwhisperer.ini @@ -4,8 +4,8 @@ hostname=localhost port=8834 username=nessus_username password=nessus_password -write_path=/opt/vulnwhisperer/data/nessus/ -db_path=/opt/vulnwhisperer/database +write_path=/opt/VulnWhisperer/data/nessus/ +db_path=/opt/VulnWhisperer/database trash=false verbose=true @@ -15,7 +15,7 @@ hostname=cloud.tenable.com port=443 username=tenable.io_username password=tenable.io_password -write_path=/opt/vulnwhisperer/data/tenable/ +write_path=/opt/VulnWhisperer/data/tenable/ db_path=/opt/VulnWhisperer/data/database trash=false verbose=true @@ -26,8 +26,8 @@ enabled = true hostname = qualysapi.qg2.apps.qualys.com username = exampleuser password = examplepass -write_path=/opt/vulnwhisperer/data/qualys/ -db_path=/opt/vulnwhisperer/data/database +write_path=/opt/VulnWhisperer/data/qualys/ +db_path=/opt/VulnWhisperer/data/database verbose=true # Set the maximum number of retries each connection should attempt. @@ -42,8 +42,8 @@ enabled = true hostname = qualysapi.qg2.apps.qualys.com username = exampleuser password = examplepass -write_path=/opt/vulnwhisperer/data/qualys/ -db_path=/opt/vulnwhisperer/data/database +write_path=/opt/VulnWhisperer/data/qualys/ +db_path=/opt/VulnWhisperer/data/database verbose=true # Set the maximum number of retries each connection should attempt. @@ -60,8 +60,8 @@ hostname = api.detectify.com username = exampleuser #password variable used as secretKey password = examplepass -write_path =/opt/vulnwhisperer/data/detectify/ -db_path = /opt/vulnwhisperer/data/database +write_path =/opt/VulnWhisperer/data/detectify/ +db_path = /opt/VulnWhisperer/data/database verbose = true [openvas] @@ -70,8 +70,8 @@ hostname = localhost port = 4000 username = exampleuser password = examplepass -write_path=/opt/vulnwhisperer/data/openvas/ -db_path=/opt/vulnwhisperer/data/database +write_path=/opt/VulnWhisperer/data/openvas/ +db_path=/opt/VulnWhisperer/data/database verbose=true #[proxy] @@ -92,8 +92,8 @@ verbose=true hostname = jira-host username = username password = password -write_path = /opt/vulnwhisperer/data/jira/ -db_path = /opt/vulnwhisperer/data/database +write_path = /opt/VulnWhisperer/data/jira/ +db_path = /opt/VulnWhisperer/data/database verbose = true dns_resolv = False From bf5070f361a72859d63ca4f12c60a0fbb3fe8696 Mon Sep 17 00:00:00 2001 From: pemontto Date: Fri, 12 Apr 2019 17:55:59 +1000 Subject: [PATCH 6/6] fix vulnwhisperer image --- docker-compose-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose-test.yml b/docker-compose-test.yml index 89e3ee4..d1f2e99 100644 --- a/docker-compose-test.yml +++ b/docker-compose-test.yml @@ -73,7 +73,7 @@ services: aliases: - logstash.local vulnwhisperer: - image: vulnwhisperer-1.8 + image: hasecuritysolutions/vulnwhisperer:latest container_name: vulnwhisperer entrypoint: [ "vuln_whisperer",