From 13a01fbfd050f1e266ce8f39d95de259f9af26ce Mon Sep 17 00:00:00 2001 From: Austin Taylor Date: Mon, 29 Jan 2018 22:35:45 -0500 Subject: [PATCH] Update to make nessus visualizations consistent with qualys --- elasticsearch/logstash-nessus-template.json | 143 +++--------------- .../1000_vulnWhispererBaseVisuals.json | 4 +- ...hisperer_ReportingMitigationDashboard.json | 13 +- logstash/1000_nessus_process_file.conf | 67 +++++--- 4 files changed, 72 insertions(+), 155 deletions(-) diff --git a/elasticsearch/logstash-nessus-template.json b/elasticsearch/logstash-nessus-template.json index 14928ca..eb715fc 100755 --- a/elasticsearch/logstash-nessus-template.json +++ b/elasticsearch/logstash-nessus-template.json @@ -15,150 +15,43 @@ }, "refresh_interval": "5s", "number_of_shards": "1", - "number_of_replicas": "1" + "number_of_replicas": "0" } }, "mappings": { "_default_": { + "_all": { + "enabled": true, + "norms": false + }, "dynamic_templates": [ { "message_field": { - "mapping": { - "fielddata": { - "format": "disabled" - }, - "index": "analyzed", - "omit_norms": true, - "type": "string" - }, + "path_match": "message", "match_mapping_type": "string", - "match": "message" + "mapping": { + "type": "text", + "norms": false + } } }, { "string_fields": { + "match": "*", + "match_mapping_type": "string", "mapping": { - "fielddata": { - "format": "disabled" - }, - "index": "analyzed", - "omit_norms": true, - "type": "string", + "type": "text", + "norms": false, "fields": { - "raw": { - "ignore_above": 256, - "index": "not_analyzed", - "type": "string", - "doc_values": true + "keyword": { + "type": "keyword", + "ignore_above": 256 } } - }, - "match_mapping_type": "string", - "match": "*" - } - }, - { - "ip_address_fields": { - "mapping": { - "type": "ip" - }, - "match": "*_ip" - } - }, - { - "ipv6_address_fields": { - "mapping": { - "index": "not_analyzed", - "type": "string" - }, - "match": "*_ipv6" - } - }, - { - "float_fields": { - "mapping": { - "type": "float", - "doc_values": true - }, - "match_mapping_type": "float", - "match": "*" - } - }, - { - "double_fields": { - "mapping": { - "type": "double", - "doc_values": true - }, - "match_mapping_type": "double", - "match": "*" - } - }, - { - "byte_fields": { - "mapping": { - "type": "byte", - "doc_values": true - }, - "match_mapping_type": "byte", - "match": "*" - } - }, - { - "short_fields": { - "mapping": { - "type": "short", - "doc_values": true - }, - "match_mapping_type": "short", - "match": "*" - } - }, - { - "integer_fields": { - "mapping": { - "type": "integer", - "doc_values": true - }, - "match_mapping_type": "integer", - "match": "*" - } - }, - { - "long_fields": { - "mapping": { - "type": "long", - "doc_values": true - }, - "match_mapping_type": "long", - "match": "*" - } - }, - { - "date_fields": { - "mapping": { - "type": "date", - "doc_values": true - }, - "match_mapping_type": "date", - "match": "*" - } - }, - { - "geo_point_fields": { - "mapping": { - "type": "geo_point", - "doc_values": true - }, - "match_mapping_type": "geo_point", - "match": "*" + } } } ], - "_all": { - "omit_norms": true, - "enabled": true - }, "properties": { "plugin_id": { "type": "integer" diff --git a/kibana/vuln_whisp_kibana/1000_vulnWhispererBaseVisuals.json b/kibana/vuln_whisp_kibana/1000_vulnWhispererBaseVisuals.json index 6ea9304..5b6707b 100755 --- a/kibana/vuln_whisp_kibana/1000_vulnWhispererBaseVisuals.json +++ b/kibana/vuln_whisp_kibana/1000_vulnWhispererBaseVisuals.json @@ -340,7 +340,7 @@ "_type": "visualization", "_source": { "title": "VulnWhisperer - Critical Assets", - "visState": "{\"title\":\"VulnWhisperer - Critical Assets\",\"type\":\"heatmap\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"enableHover\":true,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":4,\"colorSchema\":\"Green to Red\",\"setColorRange\":true,\"colorsRange\":[{\"from\":0,\"to\":3},{\"from\":3,\"to\":7},{\"from\":7,\"to\":9},{\"from\":9,\"to\":11}],\"invertColors\":false,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":true,\"rotate\":0,\"color\":\"white\"}}],\"type\":\"heatmap\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"risk_score\",\"customLabel\":\"Residual Risk Score\"}},{\"id\":\"2\",\"enabled\":false,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"risk_score\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"row\":true}},{\"id\":\"3\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Date\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"asset.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Critical Asset\"}}],\"listeners\":{}}", + "visState": "{\"title\":\"VulnWhisperer - Critical Assets\",\"type\":\"heatmap\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"enableHover\":true,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":4,\"colorSchema\":\"Green to Red\",\"setColorRange\":true,\"colorsRange\":[{\"from\":0,\"to\":3},{\"from\":3,\"to\":7},{\"from\":7,\"to\":9},{\"from\":9,\"to\":11}],\"invertColors\":false,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":false,\"rotate\":0,\"color\":\"white\"}}],\"type\":\"heatmap\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"risk_score\",\"customLabel\":\"Residual Risk Score\"}},{\"id\":\"2\",\"enabled\":false,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"risk_score\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"row\":true}},{\"id\":\"3\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Date\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"asset.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Critical Asset\"}}],\"listeners\":{}}", "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 3\":\"rgb(0,104,55)\",\"3 - 7\":\"rgb(135,203,103)\",\"7 - 9\":\"rgb(255,255,190)\",\"9 - 11\":\"rgb(249,142,82)\"},\"colors\":{\"8 - 10\":\"#BF1B00\",\"9 - 11\":\"#BF1B00\",\"7 - 9\":\"#EF843C\",\"3 - 7\":\"#EAB839\",\"0 - 3\":\"#7EB26D\"},\"legendOpen\":false}}", "description": "", "version": 1, @@ -410,7 +410,7 @@ "_type": "visualization", "_source": { "title": "VulnWhisperer - Risk: Medium", - "visState": "{\"title\":\"VulnWhisperer - Risk: Medium\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Background\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"bgFill\":\"white\",\"bgColor\":true,\"labelColor\":false,\"subText\":\"\",\"fontSize\":\"34\"},\"extendRange\":false}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Medium Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score:medium\"}}},\"label\":\"Medium Risk\"}]}}],\"listeners\":{}}", + "visState": "{\"title\":\"VulnWhisperer - Risk: Medium\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Background\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"bgFill\":\"white\",\"bgColor\":true,\"labelColor\":false,\"subText\":\"\",\"fontSize\":\"34\"},\"extendRange\":false},\"isDisplayWarning\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Medium Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:medium\"}}},\"label\":\"Medium Risk\"}]}}],\"listeners\":{}}", "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 10000\":\"#EAB839\"}}}", "description": "", "version": 1, diff --git a/kibana/vuln_whisp_kibana/1001_vulnWhisperer_ReportingMitigationDashboard.json b/kibana/vuln_whisp_kibana/1001_vulnWhisperer_ReportingMitigationDashboard.json index 1c38611..7b51eac 100755 --- a/kibana/vuln_whisp_kibana/1001_vulnWhisperer_ReportingMitigationDashboard.json +++ b/kibana/vuln_whisp_kibana/1001_vulnWhisperer_ReportingMitigationDashboard.json @@ -19,7 +19,7 @@ "value": 0 }, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"-vulnerability_category:\\\"INFORMATION_GATHERED\\\"\"}}}],\"highlightAll\":true,\"version\":true}" + "searchSourceJSON": "{\"filter\":[{\"query\":{\"match_all\":{}}}],\"highlightAll\":true,\"version\":true}" } } }, @@ -34,16 +34,9 @@ "optionsJSON": "{\"darkTheme\":false}", "uiStateJSON": "{\"P-11\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-2\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-20\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-21\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}},\"P-27\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-28\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}},\"P-3\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"asc\"}}}},\"P-30\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-31\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-40\":{\"vis\":{\"defaultColors\":{\"0 - 3\":\"rgb(0,104,55)\",\"3 - 7\":\"rgb(135,203,103)\",\"7 - 9\":\"rgb(255,255,190)\",\"9 - 11\":\"rgb(249,142,82)\"}}},\"P-41\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-42\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-43\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"}}},\"P-44\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-45\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-46\":{\"vis\":{\"legendOpen\":true}},\"P-47\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-48\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-49\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-5\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-50\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-51\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-6\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-8\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}", "version": 1, - "timeRestore": true, - "timeTo": "now", - "timeFrom": "now-30d", - "refreshInterval": { - "display": "Off", - "pause": false, - "value": 0 - }, + "timeRestore": false, "kibanaSavedObjectMeta": { - "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"-vulnerability_category:\\\"INFORMATION_GATHERED\\\"\"}}}],\"highlightAll\":true,\"version\":true}" + "searchSourceJSON": "{\"filter\":[{\"query\":{\"match_all\":{}}}],\"highlightAll\":true,\"version\":true}" } } } diff --git a/logstash/1000_nessus_process_file.conf b/logstash/1000_nessus_process_file.conf index 9d4e27f..019b696 100644 --- a/logstash/1000_nessus_process_file.conf +++ b/logstash/1000_nessus_process_file.conf @@ -6,7 +6,7 @@ input { file { - path => "/opt/vulnwhisp/nessus/**/*" + path => "/opt/vulnwhisperer/nessus/**/*" start_position => "beginning" tags => "nessus" type => "nessus" @@ -15,6 +15,9 @@ input { filter { if "nessus" in [tags]{ + # Drop the header column + if [message] =~ "^Plugin ID" { drop {} } + mutate { gsub => [ "message", "\|\|\|", " ", @@ -26,7 +29,7 @@ filter { } csv { - columns => ["plugin_id", "cve", "cvss", "risk", "host", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"] + columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"] separator => "," source => "message" } @@ -57,10 +60,13 @@ filter { if [risk] == "Critical" { mutate { add_field => { "risk_number" => 4 }} } - + if [cve] == "nan" { mutate { remove_field => [ "cve" ] } } + if [cvss] == "nan" { + mutate { remove_field => [ "cvss" ] } + } if [see_also] == "nan" { mutate { remove_field => [ "see_also" ] } } @@ -81,11 +87,36 @@ filter { mutate { convert => { "risk_score" => "float" } } + if [risk_score] == 0 { + mutate { + add_field => { "risk_score_name" => "info" } + } + } + if [risk_score] > 0 and [risk_score] < 3 { + mutate { + add_field => { "risk_score_name" => "low" } + } + } + if [risk_score] >= 3 and [risk_score] < 6 { + mutate { + add_field => { "risk_score_name" => "medium" } + } + } + if [risk_score] >=6 and [risk_score] < 9 { + mutate { + add_field => { "risk_score_name" => "high" } + } + } + if [risk_score] >= 9 { + mutate { + add_field => { "risk_score_name" => "critical" } + } + } # Compensating controls - adjust risk_score # Adobe and Java are not allowed to run in browser unless whitelisted # Therefore, lower score by dividing by 3 (score is subjective to risk) - + #Modify and uncomment when ready to use #if [risk_score] != 0 { # if [plugin_name] =~ "Adobe" and [risk_score] > 6 or [plugin_name] =~ "Java" and [risk_score] > 6 { @@ -100,27 +131,27 @@ filter { # Add tags for reporting based on assets or criticality - #if [host] == "192.168.0.1" or [host] == "192.168.0.50" or [host] =~ "^192\.168\.10\." or [host] =~ "^42.42.42." { - # mutate { - # add_tag => [ "critical_asset" ] - # } - #} - #if [host] =~ "^192\.168\.[45][0-9][0-9]\.1$" or [host] =~ "^192.168\.[50]\.[0-9]{1,2}\.1$"{ + if [asset] == "dc01" or [asset] == "dc02" or [asset] == "pki01" or [asset] == "192.168.0.54" or [asset] =~ "^192\.168\.0\." or [asset] =~ "^42.42.42." { + mutate { + add_tag => [ "critical_asset" ] + } + } + #if [asset] =~ "^192\.168\.[45][0-9][0-9]\.1$" or [asset] =~ "^192.168\.[50]\.[0-9]{1,2}\.1$"{ # mutate { # add_tag => [ "has_hipaa_data" ] # } #} - #if [host] =~ "^192\.168\.[45][0-9][0-9]\." { + #if [asset] =~ "^192\.168\.[45][0-9][0-9]\." { # mutate { # add_tag => [ "hipaa_asset" ] # } #} - #if [host] =~ "^192\.168\.5\." { - # mutate { - # add_tag => [ "pci_asset" ] - # } - #} - #if [host] =~ "^10\.0\.50\." { + if [asset] =~ "^hr" { + mutate { + add_tag => [ "pci_asset" ] + } + } + #if [asset] =~ "^10\.0\.50\." { # mutate { # add_tag => [ "web_servers" ] # } @@ -133,7 +164,7 @@ output { #stdout { codec => rubydebug } elasticsearch { hosts => [ "localhost:9200" ] - index => "logstash-nessus-%{+YYYY.MM}" + index => "logstash-vulnwhisperer-%{+YYYY.MM}" } } }