Docker-compose fully working with vulnwhisperer integrated (#90)
* ignore nessus requests warnings * docker-compose fully working with vulnwhisperer integrated * remove comments docker-compose * documenting docker-compose * Readme corrections * fix after recheck everything works out of the box * fix exits that break the no specified section execution mode * fix docker qualysapi issue, updated README * revert change on deps/qualysapi/qualysapi/util.py (no effect) * temporarily changed Dockerfile link to the working one
This commit is contained in:
qmontal
220
docker/1000_nessus_process_file.conf
Normal file
220
docker/1000_nessus_process_file.conf
Normal file
@ -0,0 +1,220 @@
|
||||
# Author: Austin Taylor and Justin Henderson
|
||||
# Email: email@austintaylor.io
|
||||
# Last Update: 12/20/2017
|
||||
# Version 0.3
|
||||
# Description: Take in nessus reports from vulnWhisperer and pumps into logstash
|
||||
|
||||
|
||||
input {
|
||||
file {
|
||||
path => "/opt/VulnWhisperer/data/nessus/**/*"
|
||||
start_position => "beginning"
|
||||
tags => "nessus"
|
||||
type => "nessus"
|
||||
}
|
||||
file {
|
||||
path => "/opt/VulnWhisperer/data/tenable/*.csv"
|
||||
start_position => "beginning"
|
||||
tags => "tenable"
|
||||
type => "tenable"
|
||||
}
|
||||
}
|
||||
|
||||
filter {
|
||||
if "nessus" in [tags] or "tenable" in [tags] {
|
||||
# Drop the header column
|
||||
if [message] =~ "^Plugin ID" { drop {} }
|
||||
|
||||
csv {
|
||||
# columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"]
|
||||
columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss_vector", "system_type", "host_start", "host_end"]
|
||||
separator => ","
|
||||
source => "message"
|
||||
}
|
||||
|
||||
ruby {
|
||||
code => "if event.get('description')
|
||||
event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
|
||||
end
|
||||
if event.get('synopsis')
|
||||
event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
|
||||
end
|
||||
if event.get('solution')
|
||||
event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
|
||||
end
|
||||
if event.get('see_also')
|
||||
event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
|
||||
end
|
||||
if event.get('plugin_output')
|
||||
event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
|
||||
end"
|
||||
}
|
||||
|
||||
#If using filebeats as your source, you will need to replace the "path" field to "source"
|
||||
grok {
|
||||
match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.csv$" }
|
||||
tag_on_failure => []
|
||||
}
|
||||
|
||||
date {
|
||||
match => [ "last_updated", "UNIX" ]
|
||||
target => "@timestamp"
|
||||
remove_field => ["last_updated"]
|
||||
}
|
||||
|
||||
if [risk] == "None" {
|
||||
mutate { add_field => { "risk_number" => 0 }}
|
||||
}
|
||||
if [risk] == "Low" {
|
||||
mutate { add_field => { "risk_number" => 1 }}
|
||||
}
|
||||
if [risk] == "Medium" {
|
||||
mutate { add_field => { "risk_number" => 2 }}
|
||||
}
|
||||
if [risk] == "High" {
|
||||
mutate { add_field => { "risk_number" => 3 }}
|
||||
}
|
||||
if [risk] == "Critical" {
|
||||
mutate { add_field => { "risk_number" => 4 }}
|
||||
}
|
||||
|
||||
if ![cve] or [cve] == "nan" {
|
||||
mutate { remove_field => [ "cve" ] }
|
||||
}
|
||||
if ![cvss] or [cvss] == "nan" {
|
||||
mutate { remove_field => [ "cvss" ] }
|
||||
}
|
||||
if ![cvss_base] or [cvss_base] == "nan" {
|
||||
mutate { remove_field => [ "cvss_base" ] }
|
||||
}
|
||||
if ![cvss_temporal] or [cvss_temporal] == "nan" {
|
||||
mutate { remove_field => [ "cvss_temporal" ] }
|
||||
}
|
||||
if ![cvss_temporal_vector] or [cvss_temporal_vector] == "nan" {
|
||||
mutate { remove_field => [ "cvss_temporal_vector" ] }
|
||||
}
|
||||
if ![cvss_vector] or [cvss_vector] == "nan" {
|
||||
mutate { remove_field => [ "cvss_vector" ] }
|
||||
}
|
||||
if ![cvss3_base] or [cvss3_base] == "nan" {
|
||||
mutate { remove_field => [ "cvss3_base" ] }
|
||||
}
|
||||
if ![cvss3_temporal] or [cvss3_temporal] == "nan" {
|
||||
mutate { remove_field => [ "cvss3_temporal" ] }
|
||||
}
|
||||
if ![cvss3_temporal_vector] or [cvss3_temporal_vector] == "nan" {
|
||||
mutate { remove_field => [ "cvss3_temporal_vector" ] }
|
||||
}
|
||||
if ![description] or [description] == "nan" {
|
||||
mutate { remove_field => [ "description" ] }
|
||||
}
|
||||
if ![mac_address] or [mac_address] == "nan" {
|
||||
mutate { remove_field => [ "mac_address" ] }
|
||||
}
|
||||
if ![netbios] or [netbios] == "nan" {
|
||||
mutate { remove_field => [ "netbios" ] }
|
||||
}
|
||||
if ![operating_system] or [operating_system] == "nan" {
|
||||
mutate { remove_field => [ "operating_system" ] }
|
||||
}
|
||||
if ![plugin_output] or [plugin_output] == "nan" {
|
||||
mutate { remove_field => [ "plugin_output" ] }
|
||||
}
|
||||
if ![see_also] or [see_also] == "nan" {
|
||||
mutate { remove_field => [ "see_also" ] }
|
||||
}
|
||||
if ![synopsis] or [synopsis] == "nan" {
|
||||
mutate { remove_field => [ "synopsis" ] }
|
||||
}
|
||||
if ![system_type] or [system_type] == "nan" {
|
||||
mutate { remove_field => [ "system_type" ] }
|
||||
}
|
||||
|
||||
mutate {
|
||||
remove_field => [ "message" ]
|
||||
add_field => { "risk_score" => "%{cvss}" }
|
||||
}
|
||||
mutate {
|
||||
convert => { "risk_score" => "float" }
|
||||
}
|
||||
if [risk_score] == 0 {
|
||||
mutate {
|
||||
add_field => { "risk_score_name" => "info" }
|
||||
}
|
||||
}
|
||||
if [risk_score] > 0 and [risk_score] < 3 {
|
||||
mutate {
|
||||
add_field => { "risk_score_name" => "low" }
|
||||
}
|
||||
}
|
||||
if [risk_score] >= 3 and [risk_score] < 6 {
|
||||
mutate {
|
||||
add_field => { "risk_score_name" => "medium" }
|
||||
}
|
||||
}
|
||||
if [risk_score] >=6 and [risk_score] < 9 {
|
||||
mutate {
|
||||
add_field => { "risk_score_name" => "high" }
|
||||
}
|
||||
}
|
||||
if [risk_score] >= 9 {
|
||||
mutate {
|
||||
add_field => { "risk_score_name" => "critical" }
|
||||
}
|
||||
}
|
||||
|
||||
# Compensating controls - adjust risk_score
|
||||
# Adobe and Java are not allowed to run in browser unless whitelisted
|
||||
# Therefore, lower score by dividing by 3 (score is subjective to risk)
|
||||
|
||||
#Modify and uncomment when ready to use
|
||||
#if [risk_score] != 0 {
|
||||
# if [plugin_name] =~ "Adobe" and [risk_score] > 6 or [plugin_name] =~ "Java" and [risk_score] > 6 {
|
||||
# ruby {
|
||||
# code => "event.set('risk_score', event.get('risk_score') / 3)"
|
||||
# }
|
||||
# mutate {
|
||||
# add_field => { "compensating_control" => "Adobe and Flash removed from browsers unless whitelisted site." }
|
||||
# }
|
||||
# }
|
||||
#}
|
||||
|
||||
# Add tags for reporting based on assets or criticality
|
||||
|
||||
if [asset] == "dc01" or [asset] == "dc02" or [asset] == "pki01" or [asset] == "192.168.0.54" or [asset] =~ "^192\.168\.0\." or [asset] =~ "^42.42.42." {
|
||||
mutate {
|
||||
add_tag => [ "critical_asset" ]
|
||||
}
|
||||
}
|
||||
#if [asset] =~ "^192\.168\.[45][0-9][0-9]\.1$" or [asset] =~ "^192.168\.[50]\.[0-9]{1,2}\.1$"{
|
||||
# mutate {
|
||||
# add_tag => [ "has_hipaa_data" ]
|
||||
# }
|
||||
#}
|
||||
#if [asset] =~ "^192\.168\.[45][0-9][0-9]\." {
|
||||
# mutate {
|
||||
# add_tag => [ "hipaa_asset" ]
|
||||
# }
|
||||
#}
|
||||
if [asset] =~ "^hr" {
|
||||
mutate {
|
||||
add_tag => [ "pci_asset" ]
|
||||
}
|
||||
}
|
||||
#if [asset] =~ "^10\.0\.50\." {
|
||||
# mutate {
|
||||
# add_tag => [ "web_servers" ]
|
||||
# }
|
||||
#}
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
if "nessus" in [tags] or "tenable" in [tags] or [type] in [ "nessus", "tenable" ] {
|
||||
# stdout { codec => rubydebug }
|
||||
elasticsearch {
|
||||
hosts => [ "vulnwhisp-es1.local:9200" ]
|
||||
index => "logstash-vulnwhisperer-%{+YYYY.MM}"
|
||||
}
|
||||
}
|
||||
}
|
||||
153
docker/2000_qualys_web_scans.conf
Normal file
153
docker/2000_qualys_web_scans.conf
Normal file
@ -0,0 +1,153 @@
|
||||
# Author: Austin Taylor and Justin Henderson
|
||||
# Email: austin@hasecuritysolutions.com
|
||||
# Last Update: 12/30/2017
|
||||
# Version 0.3
|
||||
# Description: Take in qualys web scan reports from vulnWhisperer and pumps into logstash
|
||||
|
||||
input {
|
||||
file {
|
||||
path => "/opt/VulnWhisperer/data/qualys/scans/**/*.json"
|
||||
type => json
|
||||
codec => json
|
||||
start_position => "beginning"
|
||||
tags => [ "qualys" ]
|
||||
}
|
||||
}
|
||||
|
||||
filter {
|
||||
if "qualys" in [tags] {
|
||||
grok {
|
||||
match => { "path" => [ "(?<tags>qualys_vuln)_scan_%{DATA}_%{INT:last_updated}.json$", "(?<tags>qualys_web)_%{INT:app_id}_%{INT:last_updated}.json$" ] }
|
||||
tag_on_failure => []
|
||||
}
|
||||
|
||||
mutate {
|
||||
replace => [ "message", "%{message}" ]
|
||||
#gsub => [
|
||||
# "message", "\|\|\|", " ",
|
||||
# "message", "\t\t", " ",
|
||||
# "message", " ", " ",
|
||||
# "message", " ", " ",
|
||||
# "message", " ", " ",
|
||||
# "message", "nan", " ",
|
||||
# "message",'\n',''
|
||||
#]
|
||||
}
|
||||
|
||||
if "qualys_web" in [tags] {
|
||||
mutate {
|
||||
add_field => { "asset" => "%{web_application_name}" }
|
||||
add_field => { "risk_score" => "%{cvss}" }
|
||||
}
|
||||
} else if "qualys_vuln" in [tags] {
|
||||
mutate {
|
||||
add_field => { "asset" => "%{ip}" }
|
||||
add_field => { "risk_score" => "%{cvss}" }
|
||||
}
|
||||
}
|
||||
|
||||
if [risk] == "1" {
|
||||
mutate { add_field => { "risk_number" => 0 }}
|
||||
mutate { replace => { "risk" => "info" }}
|
||||
}
|
||||
if [risk] == "2" {
|
||||
mutate { add_field => { "risk_number" => 1 }}
|
||||
mutate { replace => { "risk" => "low" }}
|
||||
}
|
||||
if [risk] == "3" {
|
||||
mutate { add_field => { "risk_number" => 2 }}
|
||||
mutate { replace => { "risk" => "medium" }}
|
||||
}
|
||||
if [risk] == "4" {
|
||||
mutate { add_field => { "risk_number" => 3 }}
|
||||
mutate { replace => { "risk" => "high" }}
|
||||
}
|
||||
if [risk] == "5" {
|
||||
mutate { add_field => { "risk_number" => 4 }}
|
||||
mutate { replace => { "risk" => "critical" }}
|
||||
}
|
||||
|
||||
mutate {
|
||||
remove_field => "message"
|
||||
}
|
||||
|
||||
if [first_time_detected] {
|
||||
date {
|
||||
match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
|
||||
target => "first_time_detected"
|
||||
}
|
||||
}
|
||||
if [first_time_tested] {
|
||||
date {
|
||||
match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
|
||||
target => "first_time_tested"
|
||||
}
|
||||
}
|
||||
if [last_time_detected] {
|
||||
date {
|
||||
match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
|
||||
target => "last_time_detected"
|
||||
}
|
||||
}
|
||||
if [last_time_tested] {
|
||||
date {
|
||||
match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
|
||||
target => "last_time_tested"
|
||||
}
|
||||
}
|
||||
date {
|
||||
match => [ "last_updated", "UNIX" ]
|
||||
target => "@timestamp"
|
||||
remove_field => "last_updated"
|
||||
}
|
||||
mutate {
|
||||
convert => { "plugin_id" => "integer"}
|
||||
convert => { "id" => "integer"}
|
||||
convert => { "risk_number" => "integer"}
|
||||
convert => { "risk_score" => "float"}
|
||||
convert => { "total_times_detected" => "integer"}
|
||||
convert => { "cvss_temporal" => "float"}
|
||||
convert => { "cvss" => "float"}
|
||||
}
|
||||
if [risk_score] == 0 {
|
||||
mutate {
|
||||
add_field => { "risk_score_name" => "info" }
|
||||
}
|
||||
}
|
||||
if [risk_score] > 0 and [risk_score] < 3 {
|
||||
mutate {
|
||||
add_field => { "risk_score_name" => "low" }
|
||||
}
|
||||
}
|
||||
if [risk_score] >= 3 and [risk_score] < 6 {
|
||||
mutate {
|
||||
add_field => { "risk_score_name" => "medium" }
|
||||
}
|
||||
}
|
||||
if [risk_score] >=6 and [risk_score] < 9 {
|
||||
mutate {
|
||||
add_field => { "risk_score_name" => "high" }
|
||||
}
|
||||
}
|
||||
if [risk_score] >= 9 {
|
||||
mutate {
|
||||
add_field => { "risk_score_name" => "critical" }
|
||||
}
|
||||
}
|
||||
|
||||
if [asset] =~ "\.yourdomain\.(com|net)$" {
|
||||
mutate {
|
||||
add_tag => [ "critical_asset" ]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
output {
|
||||
if "qualys" in [tags] {
|
||||
stdout { codec => rubydebug }
|
||||
elasticsearch {
|
||||
hosts => [ "vulnwhisp_es1.local:9200" ]
|
||||
index => "logstash-vulnwhisperer-%{+YYYY.MM}"
|
||||
}
|
||||
}
|
||||
}
|
||||
146
docker/3000_openvas.conf
Normal file
146
docker/3000_openvas.conf
Normal file
@ -0,0 +1,146 @@
|
||||
# Author: Austin Taylor and Justin Henderson
|
||||
# Email: austin@hasecuritysolutions.com
|
||||
# Last Update: 03/04/2018
|
||||
# Version 0.3
|
||||
# Description: Take in qualys web scan reports from vulnWhisperer and pumps into logstash
|
||||
|
||||
input {
|
||||
file {
|
||||
path => "/opt/VulnWhisperer/data/openvas/*.json"
|
||||
type => json
|
||||
codec => json
|
||||
start_position => "beginning"
|
||||
tags => [ "openvas_scan", "openvas" ]
|
||||
}
|
||||
}
|
||||
|
||||
filter {
|
||||
if "openvas_scan" in [tags] {
|
||||
mutate {
|
||||
replace => [ "message", "%{message}" ]
|
||||
gsub => [
|
||||
"message", "\|\|\|", " ",
|
||||
"message", "\t\t", " ",
|
||||
"message", " ", " ",
|
||||
"message", " ", " ",
|
||||
"message", " ", " ",
|
||||
"message", "nan", " ",
|
||||
"message",'\n',''
|
||||
]
|
||||
}
|
||||
|
||||
|
||||
grok {
|
||||
match => { "path" => "openvas_scan_%{DATA:scan_id}_%{INT:last_updated}.json$" }
|
||||
tag_on_failure => []
|
||||
}
|
||||
|
||||
mutate {
|
||||
add_field => { "risk_score" => "%{cvss}" }
|
||||
}
|
||||
|
||||
if [risk] == "1" {
|
||||
mutate { add_field => { "risk_number" => 0 }}
|
||||
mutate { replace => { "risk" => "info" }}
|
||||
}
|
||||
if [risk] == "2" {
|
||||
mutate { add_field => { "risk_number" => 1 }}
|
||||
mutate { replace => { "risk" => "low" }}
|
||||
}
|
||||
if [risk] == "3" {
|
||||
mutate { add_field => { "risk_number" => 2 }}
|
||||
mutate { replace => { "risk" => "medium" }}
|
||||
}
|
||||
if [risk] == "4" {
|
||||
mutate { add_field => { "risk_number" => 3 }}
|
||||
mutate { replace => { "risk" => "high" }}
|
||||
}
|
||||
if [risk] == "5" {
|
||||
mutate { add_field => { "risk_number" => 4 }}
|
||||
mutate { replace => { "risk" => "critical" }}
|
||||
}
|
||||
|
||||
mutate {
|
||||
remove_field => "message"
|
||||
}
|
||||
|
||||
if [first_time_detected] {
|
||||
date {
|
||||
match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
|
||||
target => "first_time_detected"
|
||||
}
|
||||
}
|
||||
if [first_time_tested] {
|
||||
date {
|
||||
match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
|
||||
target => "first_time_tested"
|
||||
}
|
||||
}
|
||||
if [last_time_detected] {
|
||||
date {
|
||||
match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
|
||||
target => "last_time_detected"
|
||||
}
|
||||
}
|
||||
if [last_time_tested] {
|
||||
date {
|
||||
match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
|
||||
target => "last_time_tested"
|
||||
}
|
||||
}
|
||||
date {
|
||||
match => [ "last_updated", "UNIX" ]
|
||||
target => "@timestamp"
|
||||
remove_field => "last_updated"
|
||||
}
|
||||
mutate {
|
||||
convert => { "plugin_id" => "integer"}
|
||||
convert => { "id" => "integer"}
|
||||
convert => { "risk_number" => "integer"}
|
||||
convert => { "risk_score" => "float"}
|
||||
convert => { "total_times_detected" => "integer"}
|
||||
convert => { "cvss_temporal" => "float"}
|
||||
convert => { "cvss" => "float"}
|
||||
}
|
||||
if [risk_score] == 0 {
|
||||
mutate {
|
||||
add_field => { "risk_score_name" => "info" }
|
||||
}
|
||||
}
|
||||
if [risk_score] > 0 and [risk_score] < 3 {
|
||||
mutate {
|
||||
add_field => { "risk_score_name" => "low" }
|
||||
}
|
||||
}
|
||||
if [risk_score] >= 3 and [risk_score] < 6 {
|
||||
mutate {
|
||||
add_field => { "risk_score_name" => "medium" }
|
||||
}
|
||||
}
|
||||
if [risk_score] >=6 and [risk_score] < 9 {
|
||||
mutate {
|
||||
add_field => { "risk_score_name" => "high" }
|
||||
}
|
||||
}
|
||||
if [risk_score] >= 9 {
|
||||
mutate {
|
||||
add_field => { "risk_score_name" => "critical" }
|
||||
}
|
||||
}
|
||||
# Add your critical assets by subnet or by hostname. Comment this field out if you don't want to tag any, but the asset panel will break.
|
||||
if [asset] =~ "^10\.0\.100\." {
|
||||
mutate {
|
||||
add_tag => [ "critical_asset" ]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
output {
|
||||
if "openvas" in [tags] {
|
||||
stdout { codec => rubydebug }
|
||||
elasticsearch {
|
||||
hosts => [ "vulnwhisp_es1.local:9200" ]
|
||||
index => "logstash-vulnwhisperer-%{+YYYY.MM}"
|
||||
}
|
||||
}
|
||||
}
|
||||
6
docker/logstash.yml
Normal file
6
docker/logstash.yml
Normal file
@ -0,0 +1,6 @@
|
||||
path.data: /var/lib/logstash
|
||||
path.config: /etc/logstash/conf.d
|
||||
#path.logs: /var/log/logstash
|
||||
xpack.monitoring.elasticsearch.url: [ "vulnwhisp-es1.local:9200" ]
|
||||
xpack.monitoring.elasticsearch.username: "elastic"
|
||||
xpack.monitoring.elasticsearch.password: "changeme"
|
||||
Reference in New Issue
Block a user