From 4ed6827ee6225ebae71cd5cf4702a9bdda2983cb Mon Sep 17 00:00:00 2001 From: pemontto Date: Thu, 11 Apr 2019 08:27:28 +1000 Subject: [PATCH 1/3] Clean config and separate qualys data --- configs/frameworks_example.ini | 24 ++----------------- configs/test.ini | 24 ++----------------- .../elk6/pipeline/2000_qualys_web_scans.conf | 3 +-- 3 files changed, 5 insertions(+), 46 deletions(-) diff --git a/configs/frameworks_example.ini b/configs/frameworks_example.ini index 77d283c..20410cb 100755 --- a/configs/frameworks_example.ini +++ b/configs/frameworks_example.ini @@ -26,7 +26,7 @@ enabled = true hostname = qualysapi.qg2.apps.qualys.com username = exampleuser password = examplepass -write_path=/opt/VulnWhisperer/data/qualys/ +write_path=/opt/VulnWhisperer/data/qualys_web/ db_path=/opt/VulnWhisperer/data/database verbose=true @@ -42,16 +42,10 @@ enabled = true hostname = qualysapi.qg2.apps.qualys.com username = exampleuser password = examplepass -write_path=/opt/VulnWhisperer/data/qualys/ +write_path=/opt/VulnWhisperer/data/qualys_vuln/ db_path=/opt/VulnWhisperer/data/database verbose=true -# Set the maximum number of retries each connection should attempt. -#Note, this applies only to failed connections and timeouts, never to requests where the server returns a response. -max_retries = 10 -# Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID. -template_id = 126024 - [detectify] #Reference https://developer.detectify.com/ enabled = false @@ -74,20 +68,6 @@ write_path=/opt/VulnWhisperer/data/openvas/ db_path=/opt/VulnWhisperer/data/database verbose=true -#[proxy] -; This section is optional. Leave it out if you're not using a proxy. -; You can use environmental variables as well: http://www.python-requests.org/en/latest/user/advanced/#proxies - -; proxy_protocol set to https, if not specified. -#proxy_url = proxy.mycorp.com - -; proxy_port will override any port specified in proxy_url -#proxy_port = 8080 - -; proxy authentication -#proxy_username = proxyuser -#proxy_password = proxypass - [jira] enabled = false hostname = jira-host diff --git a/configs/test.ini b/configs/test.ini index 468ba4a..b8ce72f 100755 --- a/configs/test.ini +++ b/configs/test.ini @@ -26,7 +26,7 @@ enabled = false hostname = qualys_web username = exampleuser password = examplepass -write_path=/tmp/VulnWhisperer/data/qualys/ +write_path=/tmp/VulnWhisperer/data/qualys_web/ db_path=/tmp/VulnWhisperer/data/database verbose=true @@ -42,16 +42,10 @@ enabled = true hostname = qualys_vuln username = exampleuser password = examplepass -write_path=/tmp/VulnWhisperer/data/qualys/ +write_path=/tmp/VulnWhisperer/data/qualys_vuln/ db_path=/tmp/VulnWhisperer/data/database verbose=true -# Set the maximum number of retries each connection should attempt. -#Note, this applies only to failed connections and timeouts, never to requests where the server returns a response. -max_retries = 10 -# Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID. -template_id = 126024 - [detectify] #Reference https://developer.detectify.com/ enabled = false @@ -74,20 +68,6 @@ write_path=/tmp/VulnWhisperer/data/openvas/ db_path=/tmp/VulnWhisperer/data/database verbose=true -#[proxy] -; This section is optional. Leave it out if you're not using a proxy. -; You can use environmental variables as well: http://www.python-requests.org/en/latest/user/advanced/#proxies - -; proxy_protocol set to https, if not specified. -#proxy_url = proxy.mycorp.com - -; proxy_port will override any port specified in proxy_url -#proxy_port = 8080 - -; proxy authentication -#proxy_username = proxyuser -#proxy_password = proxypass - [jira] enabled = false hostname = jira-host diff --git a/resources/elk6/pipeline/2000_qualys_web_scans.conf b/resources/elk6/pipeline/2000_qualys_web_scans.conf index 66b0993..fbf83ee 100644 --- a/resources/elk6/pipeline/2000_qualys_web_scans.conf +++ b/resources/elk6/pipeline/2000_qualys_web_scans.conf @@ -6,7 +6,7 @@ input { file { - path => "/opt/vulnwhisperer/data/qualys/*.json" + path => [ "/opt/vulnwhisperer/data/qualys/*.json" , "/opt/vulnwhisperer/data/qualys_web/*.json", "/opt/vulnwhisperer/data/qualys_vuln/*.json"] type => json codec => json start_position => "beginning" @@ -14,7 +14,6 @@ input { mode => "read" start_position => "beginning" file_completed_action => "delete" - } } From d2a7513ed1191b85c0e2e73b0a93548bd8ca8492 Mon Sep 17 00:00:00 2001 From: pemontto Date: Thu, 11 Apr 2019 10:36:41 +1000 Subject: [PATCH 2/3] Fix nessus logstash field cvss3_vector --- .../logstash/1000_nessus_process_file.conf | 2 +- resources/elk6/pipeline/1000_nessus_process_file.conf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf b/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf index 60e1920..d8d4f92 100644 --- a/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf +++ b/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf @@ -27,7 +27,7 @@ filter { csv { # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"] - columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss_vector", "system_type", "host_start", "host_end"] + columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"] separator => "," source => "message" } diff --git a/resources/elk6/pipeline/1000_nessus_process_file.conf b/resources/elk6/pipeline/1000_nessus_process_file.conf index dcb74a2..0c64047 100644 --- a/resources/elk6/pipeline/1000_nessus_process_file.conf +++ b/resources/elk6/pipeline/1000_nessus_process_file.conf @@ -29,7 +29,7 @@ filter { csv { # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"] - columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss_vector", "system_type", "host_start", "host_end"] + columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"] separator => "," source => "message" } From 8dc3b2f8aca3e89f540565278c99cba6b839d9d0 Mon Sep 17 00:00:00 2001 From: pemontto Date: Thu, 11 Apr 2019 10:41:13 +1000 Subject: [PATCH 3/3] Add qualys paths to elk5 logstash config --- .../elk5-old_compatibility/logstash/2000_qualys_web_scans.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf b/resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf index b330260..504de84 100644 --- a/resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf +++ b/resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf @@ -6,7 +6,7 @@ input { file { - path => "/opt/vulnwhisperer/qualys/*.json" + path => [ "/opt/vulnwhisperer/data/qualys/*.json" , "/opt/vulnwhisperer/data/qualys_web/*.json", "/opt/vulnwhisperer/data/qualys_vuln/*.json" ] type => json codec => json start_position => "beginning"