Cyberdefense/VulnWhisperer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cleanup logstash configs

Browse Source
This commit is contained in:
pemontto
2019-04-15 20:10:49 +10:00
parent 74ebf43492
commit 97d2a2606c
5 changed files with 138 additions and 188 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
resources/elk6/init_kibana.sh
View File

@ -10,7 +10,7 @@ saved_objects_file="kibana_APIonly.json"
#if [ `curl -I localhost:5601/status | head -n1 |cut -d$' ' -f2` -eq '200' ]; then echo "Loading VulnWhisperer Saved Objects"; eval $(echo $add_saved_objects$saved_objects_file); else echo "waiting for kibana"; fi #if [ `curl -I localhost:5601/status | head -n1 |cut -d$' ' -f2` -eq '200' ]; then echo "Loading VulnWhisperer Saved Objects"; eval $(echo $add_saved_objects$saved_objects_file); else echo "waiting for kibana"; fi
until curl -s "$elasticsearch_url/_cluster/health?pretty" | grep '"status"' | grep -q green; do until curl -s "$elasticsearch_url/_cluster/health?pretty" | grep '"status"' | grep -qE "green|yellow"; do
curl -s "$elasticsearch_url/_cluster/health?pretty" curl -s "$elasticsearch_url/_cluster/health?pretty"
echo "Waiting for Elasticsearch" echo "Waiting for Elasticsearch"
sleep 5 sleep 5

106
resources/elk6/pipeline/1000_nessus_process_file.conf
View File

@ -27,95 +27,49 @@ input {
filter { filter {
if "nessus" in [tags] or "tenable" in [tags] { if "nessus" in [tags] or "tenable" in [tags] {
date {
match => [ "_timestamp", "UNIX" ]
target => "@timestamp"
remove_field => ["timestamp"]
}
#If using filebeats as your source, you will need to replace the "path" field to "source" #If using filebeats as your source, you will need to replace the "path" field to "source"
# Remove when scan name is included in event (current method is error prone) # Remove when scan name is included in event (current method is error prone)
grok { grok {
match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.(csv|json)$" } match => { "path" => "([a-zA-Z0-9_.\-]+)_%{INT}_%{INT:history_id}_%{INT}.json$" }
tag_on_failure => [] tag_on_failure => []
} }
# TODO remove when @timestamp is included in event translate {
date { field => "[risk]"
match => [ "last_updated", "UNIX" ] destination => "[risk_number]"
target => "@timestamp" dictionary => {
remove_field => ["last_updated"] "None" => 0
} "Low" => 1
"Medium" => 2
if [risk] == "None" { "High" => 3
mutate { add_field => { "risk_number" => 0 }} "Critical" => 4
} }
if [risk] == "Low" {
mutate { add_field => { "risk_number" => 1 }}
}
if [risk] == "Medium" {
mutate { add_field => { "risk_number" => 2 }}
}
if [risk] == "High" {
mutate { add_field => { "risk_number" => 3 }}
}
if [risk] == "Critical" {
mutate { add_field => { "risk_number" => 4 }}
}
if ![cve] or [cve] == "nan" {
mutate { remove_field => [ "cve" ] }
}
if ![cvss] or [cvss] == "nan" {
mutate { remove_field => [ "cvss" ] }
}
if ![cvss_base] or [cvss_base] == "nan" {
mutate { remove_field => [ "cvss_base" ] }
}
if ![cvss_temporal] or [cvss_temporal] == "nan" {
mutate { remove_field => [ "cvss_temporal" ] }
}
if ![cvss_temporal_vector] or [cvss_temporal_vector] == "nan" {
mutate { remove_field => [ "cvss_temporal_vector" ] }
}
if ![cvss_vector] or [cvss_vector] == "nan" {
mutate { remove_field => [ "cvss_vector" ] }
}
if ![cvss3_base] or [cvss3_base] == "nan" {
mutate { remove_field => [ "cvss3_base" ] }
}
if ![cvss3_temporal] or [cvss3_temporal] == "nan" {
mutate { remove_field => [ "cvss3_temporal" ] }
}
if ![cvss3_temporal_vector] or [cvss3_temporal_vector] == "nan" {
mutate { remove_field => [ "cvss3_temporal_vector" ] }
}
if ![description] or [description] == "nan" {
mutate { remove_field => [ "description" ] }
}
if ![mac_address] or [mac_address] == "nan" {
mutate { remove_field => [ "mac_address" ] }
}
if ![netbios] or [netbios] == "nan" {
mutate { remove_field => [ "netbios" ] }
}
if ![operating_system] or [operating_system] == "nan" {
mutate { remove_field => [ "operating_system" ] }
}
if ![plugin_output] or [plugin_output] == "nan" {
mutate { remove_field => [ "plugin_output" ] }
}
if ![see_also] or [see_also] == "nan" {
mutate { remove_field => [ "see_also" ] }
}
if ![synopsis] or [synopsis] == "nan" {
mutate { remove_field => [ "synopsis" ] }
}
if ![system_type] or [system_type] == "nan" {
mutate { remove_field => [ "system_type" ] }
} }
mutate { mutate {
remove_field => [ "message" ]
add_field => { "risk_score" => "%{cvss}" } add_field => { "risk_score" => "%{cvss}" }
} }
mutate { mutate {
convert => { "risk_score" => "float" } convert => { "cvss_base" => "float"}
convert => { "cvss_temporal" => "float"}
convert => { "cvss" => "float"}
convert => { "cvss3_base" => "float"}
convert => { "cvss3_temporal" => "float"}
convert => { "cvss3" => "float"}
convert => { "id" => "integer"}
convert => { "plugin_id" => "integer"}
convert => { "risk_number" => "integer"}
convert => { "risk_score" => "float"}
convert => { "total_times_detected" => "integer"}
} }
if [risk_score] == 0 { if [risk_score] == 0 {
mutate { mutate {
add_field => { "risk_score_name" => "info" } add_field => { "risk_score_name" => "info" }

132
resources/elk6/pipeline/2000_qualys_web_scans.conf
View File

@ -19,101 +19,53 @@ input {
filter { filter {
if "qualys" in [tags] { if "qualys" in [tags] {
date {
match => [ "_timestamp", "UNIX" ]
target => "@timestamp"
remove_field => ["timestamp"]
}
grok { grok {
match => { "path" => [ "(?<tags>qualys_vuln)_scan_%{DATA}_%{INT:last_updated}.json$", "(?<tags>qualys_web)_%{INT:app_id}_%{INT:last_updated}.json$" ] } match => { "path" => [ "(?<tags>qualys_vuln)_scan_%{DATA}_%{INT}.json$", "(?<tags>qualys_web)_%{INT:app_id}_%{INT}.json$" ] }
tag_on_failure => [] tag_on_failure => []
} }
mutate { translate {
replace => [ "message", "%{message}" ] field => "[risk_number]"
#gsub => [ destination => "[risk]"
# "message", "\|\|\|", " ", dictionary => {
# "message", "\t\t", " ", "0" => "Info"
# "message", " ", " ", "1" => "Low"
# "message", " ", " ", "2" => "Medium"
# "message", " ", " ", "3" => "High"
# "message", "nan", " ", "4" => "Critical"
# "message",'\n','' }
#]
} }
if "qualys_web" in [tags] { if "qualys_web" in [tags] {
mutate {
add_field => { "asset" => "%{web_application_name}" }
add_field => { "risk_score" => "%{cvss}" }
}
} else if "qualys_vuln" in [tags] {
mutate { mutate {
add_field => { "asset" => "%{ip}" } add_field => { "asset" => "%{web_application_name}" }
add_field => { "risk_score" => "%{cvss}" }
} }
} }
if [risk] == "1" {
mutate { add_field => { "risk_number" => 0 }}
mutate { replace => { "risk" => "info" }}
}
if [risk] == "2" {
mutate { add_field => { "risk_number" => 1 }}
mutate { replace => { "risk" => "low" }}
}
if [risk] == "3" {
mutate { add_field => { "risk_number" => 2 }}
mutate { replace => { "risk" => "medium" }}
}
if [risk] == "4" {
mutate { add_field => { "risk_number" => 3 }}
mutate { replace => { "risk" => "high" }}
}
if [risk] == "5" {
mutate { add_field => { "risk_number" => 4 }}
mutate { replace => { "risk" => "critical" }}
}
mutate { mutate {
remove_field => "message" add_field => { "risk_score" => "%{cvss}" }
} }
if [first_time_detected] {
date {
match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
target => "first_time_detected"
}
}
if [first_time_tested] {
date {
match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
target => "first_time_tested"
}
}
if [last_time_detected] {
date {
match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
target => "last_time_detected"
}
}
if [last_time_tested] {
date {
match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
target => "last_time_tested"
}
}
# TODO remove when @timestamp is included in event
date {
match => [ "last_updated", "UNIX" ]
target => "@timestamp"
remove_field => "last_updated"
}
mutate { mutate {
convert => { "plugin_id" => "integer"} convert => { "cvss_base" => "float"}
convert => { "cvss_temporal" => "float"}
convert => { "cvss" => "float"}
convert => { "cvss3_base" => "float"}
convert => { "cvss3_temporal" => "float"}
convert => { "cvss3" => "float"}
convert => { "id" => "integer"} convert => { "id" => "integer"}
convert => { "plugin_id" => "integer"}
convert => { "risk_number" => "integer"} convert => { "risk_number" => "integer"}
convert => { "risk_score" => "float"} convert => { "risk_score" => "float"}
convert => { "total_times_detected" => "integer"} convert => { "total_times_detected" => "integer"}
convert => { "cvss_temporal" => "float"}
convert => { "cvss" => "float"}
} }
if [risk_score] == 0 { if [risk_score] == 0 {
mutate { mutate {
add_field => { "risk_score_name" => "info" } add_field => { "risk_score_name" => "info" }
@ -140,11 +92,35 @@ filter {
} }
} }
if [asset] =~ "\.yourdomain\.(com|net)$" { if [first_time_detected] {
mutate { date {
add_tag => [ "critical_asset" ] match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
target => "first_time_detected"
} }
} }
if [first_time_tested] {
date {
match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
target => "first_time_tested"
}
}
if [last_time_detected] {
date {
match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
target => "last_time_detected"
}
}
if [last_time_tested] {
date {
match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
target => "last_time_tested"
}
}
# if [asset] =~ "\.yourdomain\.(com|net)$" {
# mutate {
# add_tag => [ "critical_asset" ]
# }
# }
} }
} }
output { output {

64
resources/elk6/pipeline/3000_openvas.conf
View File

@ -20,27 +20,27 @@ input {
filter { filter {
if "openvas_scan" in [tags] { if "openvas_scan" in [tags] {
mutate { date {
replace => [ "message", "%{message}" ] match => [ "_timestamp", "UNIX" ]
gsub => [ target => "@timestamp"
"message", "\|\|\|", " ", remove_field => ["timestamp"]
"message", "\t\t", " ",
"message", " ", " ",
"message", " ", " ",
"message", " ", " ",
"message", "nan", " ",
"message",'\n',''
]
} }
grok { grok {
match => { "path" => "openvas_scan_%{DATA:scan_id}_%{INT:last_updated}.json$" } match => { "path" => "openvas_scan_%{DATA}_%{INT}.json$" }
tag_on_failure => [] tag_on_failure => []
} }
mutate { translate {
add_field => { "risk_score" => "%{cvss}" } field => "[risk_number]"
destination => "[risk]"
dictionary => {
"0" => "Info"
"1" => "Low"
"2" => "Medium"
"3" => "High"
"4" => "Critical"
}
} }
if [risk] == "1" { if [risk] == "1" {
@ -93,21 +93,24 @@ filter {
} }
} }
# TODO remove when @timestamp is included in event
date {
match => [ "last_updated", "UNIX" ]
target => "@timestamp"
remove_field => "last_updated"
}
mutate { mutate {
convert => { "plugin_id" => "integer"} add_field => { "risk_score" => "%{cvss}" }
}
mutate {
convert => { "cvss_base" => "float"}
convert => { "cvss_temporal" => "float"}
convert => { "cvss" => "float"}
convert => { "cvss3_base" => "float"}
convert => { "cvss3_temporal" => "float"}
convert => { "cvss3" => "float"}
convert => { "id" => "integer"} convert => { "id" => "integer"}
convert => { "plugin_id" => "integer"}
convert => { "risk_number" => "integer"} convert => { "risk_number" => "integer"}
convert => { "risk_score" => "float"} convert => { "risk_score" => "float"}
convert => { "total_times_detected" => "integer"} convert => { "total_times_detected" => "integer"}
convert => { "cvss_temporal" => "float"}
convert => { "cvss" => "float"}
} }
if [risk_score] == 0 { if [risk_score] == 0 {
mutate { mutate {
add_field => { "risk_score_name" => "info" } add_field => { "risk_score_name" => "info" }
@ -139,6 +142,19 @@ filter {
add_tag => [ "critical_asset" ] add_tag => [ "critical_asset" ]
} }
} }
mutate {
convert => { "plugin_id" => "integer"}
convert => { "id" => "integer"}
convert => { "risk_number" => "integer"}
convert => { "risk_score" => "float"}
convert => { "total_times_detected" => "integer"}
convert => { "cvss" => "float"}
convert => { "cvss_base" => "float"}
convert => { "cvss_temporal" => "float"}
convert => { "cvss3" => "float"}
convert => { "cvss3_base" => "float"}
convert => { "cvss3_temporal" => "float"}
}
} }
} }
output { output {

20
vulnwhisp/vulnwhisp.py
View File

@ -455,8 +455,9 @@ class vulnWhispererNessus(vulnWhispererBase):
clean_csv['scan_name'] = scan_name.encode('utf8') clean_csv['scan_name'] = scan_name.encode('utf8')
clean_csv['scan_id'] = uuid clean_csv['scan_id'] = uuid
# Add @timestamp and convert to milliseconds # Add timestamp and convert to milliseconds
clean_csv['@timestamp'] = int(norm_time) * 1000 clean_csv['_timestamp'] = norm_time
clean_csv['scan_source'] = self.CONFIG_SECTION
clean_csv.to_json(relative_path_name, orient='records', lines=True) clean_csv.to_json(relative_path_name, orient='records', lines=True)
@ -628,8 +629,9 @@ class vulnWhispererQualys(vulnWhispererBase):
# Set common fields # Set common fields
vuln_ready['scan_name'] = scan_name.encode('utf8') vuln_ready['scan_name'] = scan_name.encode('utf8')
vuln_ready['scan_id'] = report_id vuln_ready['scan_id'] = report_id
# Add @timestamp and convert to milliseconds # Add timestamp and convert to milliseconds
vuln_ready['@timestamp'] = int(launched_date) * 1000 vuln_ready['_timestamp'] = launched_date
vuln_ready['scan_source'] = self.CONFIG_SECTION
record_meta = ( record_meta = (
scan_name, scan_name,
@ -801,8 +803,9 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
# Set common fields # Set common fields
vuln_ready['scan_name'] = scan_name.encode('utf8') vuln_ready['scan_name'] = scan_name.encode('utf8')
vuln_ready['scan_id'] = report_id vuln_ready['scan_id'] = report_id
# Add @timestamp and convert to milliseconds # Add _timestamp and convert to milliseconds
vuln_ready['@timestamp'] = int(launched_date) * 1000 vuln_ready['_timestamp'] = launched_date
vuln_ready['scan_source'] = self.CONFIG_SECTION
vuln_ready.to_json(relative_path_name, orient='records', lines=True) vuln_ready.to_json(relative_path_name, orient='records', lines=True)
self.logger.info('Report written to {}'.format(report_name)) self.logger.info('Report written to {}'.format(report_name))
@ -902,8 +905,9 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
vuln_ready['scan_name'] = scan_name.encode('utf8') vuln_ready['scan_name'] = scan_name.encode('utf8')
vuln_ready['scan_id'] = report_id vuln_ready['scan_id'] = report_id
# Add @timestamp and convert to milliseconds # Add timestamp and convert to milliseconds
vuln_ready['@timestamp'] = int(launched_date) * 1000 vuln_ready['_timestamp'] = launched_date
vuln_ready['scan_source'] = self.CONFIG_SECTION
except Exception as e: except Exception as e:
self.logger.error('Could not process {}: {}'.format(report_id, str(e))) self.logger.error('Could not process {}: {}'.format(report_id, str(e)))