unified logstash config
This commit is contained in:
109
resources/elk6/pipeline/1000_combined.conf
Normal file
109
resources/elk6/pipeline/1000_combined.conf
Normal file
@ -0,0 +1,109 @@
|
||||
input {
|
||||
file {
|
||||
codec => json
|
||||
mode => "read"
|
||||
path => ["/opt/VulnWhisperer/data/nessus/**/*.json", "/opt/VulnWhisperer/data/openvas/*.json", "/opt/VulnWhisperer/data/qualys_vm/*.json", "/opt/VulnWhisperer/data/qualys_was/*.json", "/opt/VulnWhisperer/data/tenable/*.json"]
|
||||
start_position => "beginning"
|
||||
file_completed_action => "delete"
|
||||
}
|
||||
file {
|
||||
codec => json
|
||||
mode => "read"
|
||||
path => "/opt/VulnWhisperer/data/jira/*.json"
|
||||
tags => [ "jira" ]
|
||||
start_position => "beginning"
|
||||
file_completed_action => "delete"
|
||||
}
|
||||
}
|
||||
|
||||
filter {
|
||||
if [scan_source] in ["nessus", "tenable", "qualys_vm", "qualys_was", "openvas"] {
|
||||
|
||||
# Parse the date/time from scan_time
|
||||
date {
|
||||
match => [ "scan_time", "UNIX" ]
|
||||
target => "@timestamp"
|
||||
remove_field => ["scan_time"]
|
||||
}
|
||||
|
||||
# Add scan_source to tags
|
||||
mutate {
|
||||
add_field => { "[tags]" => "%{scan_source}" }
|
||||
}
|
||||
|
||||
# Create a unique document_id if _unique field exists
|
||||
if [_unique] {
|
||||
# Set document ID from _unique
|
||||
mutate {
|
||||
rename => { "_unique" => "[@metadata][id]" }
|
||||
}
|
||||
}
|
||||
|
||||
# Do we need this?
|
||||
mutate {
|
||||
convert => { "cvss" => "float"}
|
||||
convert => { "cvss2" => "float"}
|
||||
convert => { "cvss2_base" => "float"}
|
||||
convert => { "cvss2_temporal" => "float"}
|
||||
convert => { "cvss3" => "float"}
|
||||
convert => { "cvss3_base" => "float"}
|
||||
convert => { "cvss3_temporal" => "float"}
|
||||
convert => { "risk_number" => "integer"}
|
||||
convert => { "total_times_detected" => "integer"}
|
||||
}
|
||||
}
|
||||
|
||||
if [scan_source] == "qualys_was" {
|
||||
if [first_time_detected] {
|
||||
date {
|
||||
match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
|
||||
target => "first_time_detected"
|
||||
}
|
||||
}
|
||||
if [first_time_tested] {
|
||||
date {
|
||||
match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
|
||||
target => "first_time_tested"
|
||||
}
|
||||
}
|
||||
if [last_time_detected] {
|
||||
date {
|
||||
match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
|
||||
target => "last_time_detected"
|
||||
}
|
||||
}
|
||||
if [last_time_tested] {
|
||||
date {
|
||||
match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
|
||||
target => "last_time_tested"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
output {
|
||||
if [scan_source] in ["nessus", "tenable", "qualys_vm", "qualys_was", "openvas"] {
|
||||
if [@metadata][id] {
|
||||
elasticsearch {
|
||||
hosts => [ "elasticsearch:9200" ]
|
||||
index => "logstash-vulnwhisperer-%{+YYYY.MM}"
|
||||
document_id => "%{[@metadata][id]}"
|
||||
manage_template => false
|
||||
}
|
||||
} else {
|
||||
elasticsearch {
|
||||
hosts => [ "elasticsearch:9200" ]
|
||||
index => "logstash-vulnwhisperer-%{+YYYY.MM}"
|
||||
manage_template => false
|
||||
}
|
||||
}
|
||||
}
|
||||
# Should these go to the same index?
|
||||
if "jira" in [tags] {
|
||||
stdout { codec => rubydebug }
|
||||
elasticsearch {
|
||||
hosts => [ "elasticsearch:9200" ]
|
||||
index => "logstash-vulnwhisperer-%{+YYYY.MM}"
|
||||
}
|
||||
}
|
||||
}
|
Reference in New Issue
Block a user