Cyberdefense/VulnWhisperer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove nessus csv completely

Browse Source
This commit is contained in:
pemontto
2019-04-11 22:22:58 +10:00
parent 778a07535f
commit 496fd23121
4 changed files with 3 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
resources/elk5-old_compatibility/docker/1000_nessus_process_file.conf
View File

@ -6,12 +6,6 @@
input {
file {
path => "/opt/VulnWhisperer/data/nessus/**/*.csv"
start_position => "beginning"
tags => "nessus"
type => "nessus"
}
file {
path => "/opt/VulnWhisperer/data/nessus/**/*.json"
start_position => "beginning"
@ -19,12 +13,6 @@ input {
type => "nessus"
codec => json
}
file {
path => "/opt/VulnWhisperer/data/tenable/*.csv"
start_position => "beginning"
tags => "tenable"
type => "tenable"
}
file {
path => "/opt/VulnWhisperer/data/tenable/*.json"
start_position => "beginning"
@ -37,36 +25,6 @@ input {
filter {
if "nessus" in [tags] or "tenable" in [tags] {
if [path] =~ /\.csv$/ {}
# Drop the header column
if [message] =~ "^(Plugin ID|plugin_id,)" { drop {} }
csv {
# columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"]
columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"]
separator => ","
source => "message"
}
ruby {
code => "if event.get('description')
event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
end
if event.get('synopsis')
event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
end
if event.get('solution')
event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
end
if event.get('see_also')
event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
end
if event.get('plugin_output')
event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
end"
}
}
#If using filebeats as your source, you will need to replace the "path" field to "source"
grok {
match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.(csv|json)$" }

42
resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf
View File

@ -6,12 +6,6 @@
input {
file {
path => "/opt/VulnWhisperer/data/nessus/**/*.csv"
start_position => "beginning"
tags => "nessus"
type => "nessus"
}
file {
path => "/opt/VulnWhisperer/data/nessus/**/*.json"
start_position => "beginning"
@ -19,12 +13,6 @@ input {
type => "nessus"
codec => json
}
file {
path => "/opt/VulnWhisperer/data/tenable/*.csv"
start_position => "beginning"
tags => "tenable"
type => "tenable"
}
file {
path => "/opt/VulnWhisperer/data/tenable/*.json"
start_position => "beginning"
@ -37,36 +25,6 @@ input {
filter {
if "nessus" in [tags] or "tenable" in [tags] {
if [path] =~ /\.csv$/ {}
# Drop the header column
if [message] =~ "^(Plugin ID|plugin_id,)" { drop {} }
csv {
# columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"]
columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"]
separator => ","
source => "message"
}
ruby {
code => "if event.get('description')
event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
end
if event.get('synopsis')
event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
end
if event.get('solution')
event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
end
if event.get('see_also')
event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
end
if event.get('plugin_output')
event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
end"
}
}
#If using filebeats as your source, you will need to replace the "path" field to "source"
grok {
match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.(csv|json)$" }

44
resources/elk6/pipeline/1000_nessus_process_file.conf
View File

@ -6,13 +6,6 @@
input {
file {
path => "/opt/vulnwhisperer/data/nessus/**/*.csv"
mode => "read"
start_position => "beginning"
file_completed_action => "delete"
tags => "nessus"
}
file {
path => "/opt/VulnWhisperer/data/nessus/**/*.json"
start_position => "beginning"
@ -20,13 +13,6 @@ input {
tags => "nessus"
codec => json
}
file {
path => "/opt/vulnwhisperer/data/tenable/*.csv"
mode => "read"
start_position => "beginning"
file_completed_action => "delete"
tags => "tenable"
}
file {
path => "/opt/VulnWhisperer/data/tenable/*.json"
start_position => "beginning"
@ -39,36 +25,6 @@ input {
filter {
if "nessus" in [tags] or "tenable" in [tags] {
if [path] =~ /\.csv$/ {}
# Drop the header column
if [message] =~ "^(Plugin ID|plugin_id,)" { drop {} }
csv {
# columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"]
columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"]
separator => ","
source => "message"
}
ruby {
code => "if event.get('description')
event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
end
if event.get('synopsis')
event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
end
if event.get('solution')
event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
end
if event.get('see_also')
event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
end
if event.get('plugin_output')
event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
end"
}
}
#If using filebeats as your source, you will need to replace the "path" field to "source"
grok {
match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.(csv|json)$" }

6
vulnwhisp/vulnwhisp.py
View File

@ -406,7 +406,7 @@ class vulnWhispererNessus(vulnWhispererBase):
folder_name = next(f['name'] for f in folders if f['id'] == folder_id)
if status in ['completed', 'imported']:
file_name = '%s_%s_%s_%s.%s' % (scan_name, scan_id,
history_id, norm_time, 'csv')
history_id, norm_time, 'json')
repls = (('\\', '_'), ('/', '_'), (' ', '_'))
file_name = reduce(lambda a, kv: a.replace(*kv), repls, file_name)
relative_path_name = self.path_check(folder_name + '/' + file_name)
@ -448,8 +448,8 @@ class vulnWhispererNessus(vulnWhispererBase):
clean_csv['scan_name'] = scan_name.encode('utf8')
clean_csv['scan_id'] = uuid
clean_csv.to_json(relative_path_name.replace('csv', 'json'), orient='records', lines=True)
clean_csv.to_csv(relative_path_name, index=False)
clean_csv.to_json(relative_path_name, orient='records', lines=True)
record_meta = (
scan_name,
scan_id,