diff --git a/resources/elk5-old_compatibility/docker/1000_nessus_process_file.conf b/resources/elk5-old_compatibility/docker/1000_nessus_process_file.conf index fdf022b..13e6f6c 100644 --- a/resources/elk5-old_compatibility/docker/1000_nessus_process_file.conf +++ b/resources/elk5-old_compatibility/docker/1000_nessus_process_file.conf @@ -6,12 +6,6 @@ input { - file { - path => "/opt/VulnWhisperer/data/nessus/**/*.csv" - start_position => "beginning" - tags => "nessus" - type => "nessus" - } file { path => "/opt/VulnWhisperer/data/nessus/**/*.json" start_position => "beginning" @@ -19,12 +13,6 @@ input { type => "nessus" codec => json } - file { - path => "/opt/VulnWhisperer/data/tenable/*.csv" - start_position => "beginning" - tags => "tenable" - type => "tenable" - } file { path => "/opt/VulnWhisperer/data/tenable/*.json" start_position => "beginning" @@ -37,36 +25,6 @@ input { filter { if "nessus" in [tags] or "tenable" in [tags] { - if [path] =~ /\.csv$/ {} - # Drop the header column - if [message] =~ "^(Plugin ID|plugin_id,)" { drop {} } - - csv { - # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"] - columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"] - separator => "," - source => "message" - } - - ruby { - code => "if event.get('description') - event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('synopsis') - event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('solution') - event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('see_also') - event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('plugin_output') - event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end" - } - } - #If using filebeats as your source, you will need to replace the "path" field to "source" grok { match => { "path" => "(?[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.(csv|json)$" } diff --git a/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf b/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf index ffda45a..b1c4b1a 100644 --- a/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf +++ b/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf @@ -6,12 +6,6 @@ input { - file { - path => "/opt/VulnWhisperer/data/nessus/**/*.csv" - start_position => "beginning" - tags => "nessus" - type => "nessus" - } file { path => "/opt/VulnWhisperer/data/nessus/**/*.json" start_position => "beginning" @@ -19,12 +13,6 @@ input { type => "nessus" codec => json } - file { - path => "/opt/VulnWhisperer/data/tenable/*.csv" - start_position => "beginning" - tags => "tenable" - type => "tenable" - } file { path => "/opt/VulnWhisperer/data/tenable/*.json" start_position => "beginning" @@ -37,36 +25,6 @@ input { filter { if "nessus" in [tags] or "tenable" in [tags] { - if [path] =~ /\.csv$/ {} - # Drop the header column - if [message] =~ "^(Plugin ID|plugin_id,)" { drop {} } - - csv { - # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"] - columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"] - separator => "," - source => "message" - } - - ruby { - code => "if event.get('description') - event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('synopsis') - event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('solution') - event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('see_also') - event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('plugin_output') - event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end" - } - } - #If using filebeats as your source, you will need to replace the "path" field to "source" grok { match => { "path" => "(?[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.(csv|json)$" } diff --git a/resources/elk6/pipeline/1000_nessus_process_file.conf b/resources/elk6/pipeline/1000_nessus_process_file.conf index 8f02757..041bf4a 100644 --- a/resources/elk6/pipeline/1000_nessus_process_file.conf +++ b/resources/elk6/pipeline/1000_nessus_process_file.conf @@ -6,13 +6,6 @@ input { - file { - path => "/opt/vulnwhisperer/data/nessus/**/*.csv" - mode => "read" - start_position => "beginning" - file_completed_action => "delete" - tags => "nessus" - } file { path => "/opt/VulnWhisperer/data/nessus/**/*.json" start_position => "beginning" @@ -20,13 +13,6 @@ input { tags => "nessus" codec => json } - file { - path => "/opt/vulnwhisperer/data/tenable/*.csv" - mode => "read" - start_position => "beginning" - file_completed_action => "delete" - tags => "tenable" - } file { path => "/opt/VulnWhisperer/data/tenable/*.json" start_position => "beginning" @@ -39,36 +25,6 @@ input { filter { if "nessus" in [tags] or "tenable" in [tags] { - if [path] =~ /\.csv$/ {} - # Drop the header column - if [message] =~ "^(Plugin ID|plugin_id,)" { drop {} } - - csv { - # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"] - columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"] - separator => "," - source => "message" - } - - ruby { - code => "if event.get('description') - event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('synopsis') - event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('solution') - event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('see_also') - event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end - if event.get('plugin_output') - event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr)) - end" - } - } - #If using filebeats as your source, you will need to replace the "path" field to "source" grok { match => { "path" => "(?[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.(csv|json)$" } diff --git a/vulnwhisp/vulnwhisp.py b/vulnwhisp/vulnwhisp.py index c84e3a4..bf85f42 100755 --- a/vulnwhisp/vulnwhisp.py +++ b/vulnwhisp/vulnwhisp.py @@ -406,7 +406,7 @@ class vulnWhispererNessus(vulnWhispererBase): folder_name = next(f['name'] for f in folders if f['id'] == folder_id) if status in ['completed', 'imported']: file_name = '%s_%s_%s_%s.%s' % (scan_name, scan_id, - history_id, norm_time, 'csv') + history_id, norm_time, 'json') repls = (('\\', '_'), ('/', '_'), (' ', '_')) file_name = reduce(lambda a, kv: a.replace(*kv), repls, file_name) relative_path_name = self.path_check(folder_name + '/' + file_name) @@ -448,8 +448,8 @@ class vulnWhispererNessus(vulnWhispererBase): clean_csv['scan_name'] = scan_name.encode('utf8') clean_csv['scan_id'] = uuid - clean_csv.to_json(relative_path_name.replace('csv', 'json'), orient='records', lines=True) - clean_csv.to_csv(relative_path_name, index=False) + clean_csv.to_json(relative_path_name, orient='records', lines=True) + record_meta = ( scan_name, scan_id,