From 0710b38de3a956da4f6630f745484860829913db Mon Sep 17 00:00:00 2001 From: pemontto Date: Thu, 11 Apr 2019 20:27:54 +1000 Subject: [PATCH] Update nessus logstash configs --- .../docker/1000_nessus_process_file.conf | 71 ++++++++++++------- .../logstash/1000_nessus_process_file.conf | 71 ++++++++++++------- .../pipeline/1000_nessus_process_file.conf | 69 +++++++++++------- 3 files changed, 131 insertions(+), 80 deletions(-) diff --git a/resources/elk5-old_compatibility/docker/1000_nessus_process_file.conf b/resources/elk5-old_compatibility/docker/1000_nessus_process_file.conf index f28a530..fdf022b 100644 --- a/resources/elk5-old_compatibility/docker/1000_nessus_process_file.conf +++ b/resources/elk5-old_compatibility/docker/1000_nessus_process_file.conf @@ -7,52 +7,69 @@ input { file { - path => "/opt/VulnWhisperer/data/nessus/**/*" + path => "/opt/VulnWhisperer/data/nessus/**/*.csv" start_position => "beginning" tags => "nessus" type => "nessus" } + file { + path => "/opt/VulnWhisperer/data/nessus/**/*.json" + start_position => "beginning" + tags => "nessus" + type => "nessus" + codec => json + } file { path => "/opt/VulnWhisperer/data/tenable/*.csv" start_position => "beginning" tags => "tenable" type => "tenable" } + file { + path => "/opt/VulnWhisperer/data/tenable/*.json" + start_position => "beginning" + tags => "nessus" + type => "nessus" + codec => json + } } filter { if "nessus" in [tags] or "tenable" in [tags] { - # Drop the header column - if [message] =~ "^Plugin ID" { drop {} } - csv { - # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"] - columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss_vector", "system_type", "host_start", "host_end"] - separator => "," - source => "message" - } + if [path] =~ /\.csv$/ {} + # Drop the header column + if [message] =~ "^(Plugin ID|plugin_id,)" { drop {} } - ruby { - code => "if event.get('description') - event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('synopsis') - event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('solution') - event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('see_also') - event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('plugin_output') - event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end" - } + csv { + # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"] + columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"] + separator => "," + source => "message" + } + + ruby { + code => "if event.get('description') + event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) + end + if event.get('synopsis') + event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) + end + if event.get('solution') + event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) + end + if event.get('see_also') + event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) + end + if event.get('plugin_output') + event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) + end" + } + } #If using filebeats as your source, you will need to replace the "path" field to "source" grok { - match => { "path" => "(?[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.csv$" } + match => { "path" => "(?[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.(csv|json)$" } tag_on_failure => [] } diff --git a/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf b/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf index d8d4f92..ffda45a 100644 --- a/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf +++ b/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf @@ -7,52 +7,69 @@ input { file { - path => "/opt/vulnwhisperer/nessus/**/*" + path => "/opt/VulnWhisperer/data/nessus/**/*.csv" start_position => "beginning" tags => "nessus" type => "nessus" } file { - path => "/opt/vulnwhisperer/tenable/*.csv" + path => "/opt/VulnWhisperer/data/nessus/**/*.json" + start_position => "beginning" + tags => "nessus" + type => "nessus" + codec => json + } + file { + path => "/opt/VulnWhisperer/data/tenable/*.csv" start_position => "beginning" tags => "tenable" type => "tenable" } + file { + path => "/opt/VulnWhisperer/data/tenable/*.json" + start_position => "beginning" + tags => "nessus" + type => "nessus" + codec => json + } } filter { if "nessus" in [tags] or "tenable" in [tags] { - # Drop the header column - if [message] =~ "^Plugin ID" { drop {} } - csv { - # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"] - columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"] - separator => "," - source => "message" - } + if [path] =~ /\.csv$/ {} + # Drop the header column + if [message] =~ "^(Plugin ID|plugin_id,)" { drop {} } - ruby { - code => "if event.get('description') - event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('synopsis') - event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('solution') - event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('see_also') - event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('plugin_output') - event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end" + csv { + # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"] + columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"] + separator => "," + source => "message" + } + + ruby { + code => "if event.get('description') + event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) + end + if event.get('synopsis') + event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) + end + if event.get('solution') + event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) + end + if event.get('see_also') + event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) + end + if event.get('plugin_output') + event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) + end" + } } #If using filebeats as your source, you will need to replace the "path" field to "source" grok { - match => { "path" => "(?[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.csv$" } + match => { "path" => "(?[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.(csv|json)$" } tag_on_failure => [] } diff --git a/resources/elk6/pipeline/1000_nessus_process_file.conf b/resources/elk6/pipeline/1000_nessus_process_file.conf index 0c64047..8f02757 100644 --- a/resources/elk6/pipeline/1000_nessus_process_file.conf +++ b/resources/elk6/pipeline/1000_nessus_process_file.conf @@ -7,12 +7,19 @@ input { file { - path => "/opt/vulnwhisperer/data/nessus/**/*" + path => "/opt/vulnwhisperer/data/nessus/**/*.csv" mode => "read" start_position => "beginning" file_completed_action => "delete" tags => "nessus" } + file { + path => "/opt/VulnWhisperer/data/nessus/**/*.json" + start_position => "beginning" + file_completed_action => "delete" + tags => "nessus" + codec => json + } file { path => "/opt/vulnwhisperer/data/tenable/*.csv" mode => "read" @@ -20,41 +27,51 @@ input { file_completed_action => "delete" tags => "tenable" } + file { + path => "/opt/VulnWhisperer/data/tenable/*.json" + start_position => "beginning" + file_completed_action => "delete" + tags => "tenable" + codec => json + } } filter { if "nessus" in [tags] or "tenable" in [tags] { - # Drop the header column - if [message] =~ "^Plugin ID" { drop {} } - csv { - # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"] - columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"] - separator => "," - source => "message" - } + if [path] =~ /\.csv$/ {} + # Drop the header column + if [message] =~ "^(Plugin ID|plugin_id,)" { drop {} } - ruby { - code => "if event.get('description') - event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('synopsis') - event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('solution') - event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('see_also') - event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('plugin_output') - event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end" + csv { + # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"] + columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"] + separator => "," + source => "message" + } + + ruby { + code => "if event.get('description') + event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) + end + if event.get('synopsis') + event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) + end + if event.get('solution') + event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) + end + if event.get('see_also') + event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) + end + if event.get('plugin_output') + event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) + end" + } } #If using filebeats as your source, you will need to replace the "path" field to "source" grok { - match => { "path" => "(?[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.csv$" } + match => { "path" => "(?[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.(csv|json)$" } tag_on_failure => [] }