Cyberdefense/VulnWhisperer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move to a common normalisation function

Browse Source
This commit is contained in:
pemontto
2019-04-15 21:12:07 +10:00
parent dd66414fe7
commit ee327874e5
6 changed files with 102 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
resources/elk6/pipeline/1000_nessus_process_file.conf
View File

@ -40,59 +40,42 @@ filter {
tag_on_failure => []
}
translate {
field => "[risk]"
destination => "[risk_number]"
dictionary => {
"None" => 0
"Low" => 1
"Medium" => 2
"High" => 3
"Critical" => 4
}
}
mutate {
add_field => { "risk_score" => "%{cvss}" }
}
mutate {
convert => { "cvss" => "float"}
convert => { "cvss_base" => "float"}
convert => { "cvss_temporal" => "float"}
convert => { "cvss" => "float"}
convert => { "cvss3" => "float"}
convert => { "cvss3_base" => "float"}
convert => { "cvss3_temporal" => "float"}
convert => { "cvss3" => "float"}
convert => { "id" => "integer"}
convert => { "plugin_id" => "integer"}
convert => { "risk_number" => "integer"}
convert => { "risk_score" => "float"}
convert => { "total_times_detected" => "integer"}
}
if [risk_score] == 0 {
if [cvss] == 0 {
mutate {
add_field => { "risk_score_name" => "info" }
add_field => { "cvss_severity" => "info" }
}
}
if [risk_score] > 0 and [risk_score] < 3 {
if [cvss] > 0 and [cvss] < 3 {
mutate {
add_field => { "risk_score_name" => "low" }
add_field => { "cvss_severity" => "low" }
}
}
if [risk_score] >= 3 and [risk_score] < 6 {
if [cvss] >= 3 and [cvss] < 6 {
mutate {
add_field => { "risk_score_name" => "medium" }
add_field => { "cvss_severity" => "medium" }
}
}
if [risk_score] >=6 and [risk_score] < 9 {
if [cvss] >=6 and [cvss] < 9 {
mutate {
add_field => { "risk_score_name" => "high" }
add_field => { "cvss_severity" => "high" }
}
}
if [risk_score] >= 9 {
if [cvss] >= 9 {
mutate {
add_field => { "risk_score_name" => "critical" }
add_field => { "cvss_severity" => "critical" }
}
}
}

41
resources/elk6/pipeline/2000_qualys_web_scans.conf
View File

@ -30,18 +30,6 @@ filter {
tag_on_failure => []
}
translate {
field => "[risk_number]"
destination => "[risk]"
dictionary => {
"0" => "Info"
"1" => "Low"
"2" => "Medium"
"3" => "High"
"4" => "Critical"
}
}
if "qualys_web" in [tags] {
mutate {
add_field => { "asset" => "%{web_application_name}" }
@ -49,46 +37,41 @@ filter {
}
mutate {
add_field => { "risk_score" => "%{cvss}" }
}
mutate {
convert => { "cvss" => "float"}
convert => { "cvss_base" => "float"}
convert => { "cvss_temporal" => "float"}
convert => { "cvss" => "float"}
convert => { "cvss3" => "float"}
convert => { "cvss3_base" => "float"}
convert => { "cvss3_temporal" => "float"}
convert => { "cvss3" => "float"}
convert => { "id" => "integer"}
convert => { "plugin_id" => "integer"}
convert => { "risk_number" => "integer"}
convert => { "risk_score" => "float"}
convert => { "total_times_detected" => "integer"}
}
if [risk_score] == 0 {
if [cvss] == 0 {
mutate {
add_field => { "risk_score_name" => "info" }
add_field => { "cvss_severity" => "info" }
}
}
if [risk_score] > 0 and [risk_score] < 3 {
if [cvss] > 0 and [cvss] < 3 {
mutate {
add_field => { "risk_score_name" => "low" }
add_field => { "cvss_severity" => "low" }
}
}
if [risk_score] >= 3 and [risk_score] < 6 {
if [cvss] >= 3 and [cvss] < 6 {
mutate {
add_field => { "risk_score_name" => "medium" }
add_field => { "cvss_severity" => "medium" }
}
}
if [risk_score] >=6 and [risk_score] < 9 {
if [cvss] >=6 and [cvss] < 9 {
mutate {
add_field => { "risk_score_name" => "high" }
add_field => { "cvss_severity" => "high" }
}
}
if [risk_score] >= 9 {
if [cvss] >= 9 {
mutate {
add_field => { "risk_score_name" => "critical" }
add_field => { "cvss_severity" => "critical" }
}
}

30
resources/elk6/pipeline/3000_openvas.conf
View File

@ -94,48 +94,44 @@ filter {
}
mutate {
add_field => { "risk_score" => "%{cvss}" }
}
mutate {
convert => { "cvss" => "float"}
convert => { "cvss_base" => "float"}
convert => { "cvss_temporal" => "float"}
convert => { "cvss" => "float"}
convert => { "cvss3" => "float"}
convert => { "cvss3_base" => "float"}
convert => { "cvss3_temporal" => "float"}
convert => { "cvss3" => "float"}
convert => { "id" => "integer"}
convert => { "plugin_id" => "integer"}
convert => { "risk_number" => "integer"}
convert => { "risk_score" => "float"}
convert => { "total_times_detected" => "integer"}
}
if [risk_score] == 0 {
if [cvss] == 0 {
mutate {
add_field => { "risk_score_name" => "info" }
add_field => { "cvss_severity" => "info" }
}
}
if [risk_score] > 0 and [risk_score] < 3 {
if [cvss] > 0 and [cvss] < 3 {
mutate {
add_field => { "risk_score_name" => "low" }
add_field => { "cvss_severity" => "low" }
}
}
if [risk_score] >= 3 and [risk_score] < 6 {
if [cvss] >= 3 and [cvss] < 6 {
mutate {
add_field => { "risk_score_name" => "medium" }
add_field => { "cvss_severity" => "medium" }
}
}
if [risk_score] >=6 and [risk_score] < 9 {
if [cvss] >=6 and [cvss] < 9 {
mutate {
add_field => { "risk_score_name" => "high" }
add_field => { "cvss_severity" => "high" }
}
}
if [risk_score] >= 9 {
if [cvss] >= 9 {
mutate {
add_field => { "risk_score_name" => "critical" }
add_field => { "cvss_severity" => "critical" }
}
}
# Add your critical assets by subnet or by hostname. Comment this field out if you don't want to tag any, but the asset panel will break.
if [asset] =~ "^10\.0\.100\." {
mutate {

23
vulnwhisp/frameworks/nessus.py
View File

@ -217,6 +217,7 @@ class NessusAPI(object):
self.logger.debug('Changing case of fields')
df['cve'] = df['cve'].str.upper()
df['protocol'] = df['protocol'].str.lower()
df['risk'] = df['risk'].str.lower()
# Copy asset to IP
df['ip'] = df['asset']
@ -225,27 +226,5 @@ class NessusAPI(object):
self.logger.debug('Mapping risk to severity number')
df['risk_number'] = df['risk'].str.lower().map(self.SEVERITY_MAPPING)
if self.profile == 'tenable':
self.logger.debug('Combinging CVSS vectors for tenable')
# Combine CVSS vectors
df['cvss_vector'] = (
df[['cvss_vector', 'cvss_temporal_vector']]
.apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1)
.str.rstrip('/nan')
)
df['cvss3_vector'] = (
df[['cvss3_vector', 'cvss3_temporal_vector']]
.apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1)
.str.rstrip('/nan')
)
df.drop(['cvss_temporal_vector', 'cvss3_temporal_vector'], axis=1, inplace=True)
# CVSS score = cvss3_temporal or cvss3_base or cvss_temporal or cvss_base
df['cvss'] = df['cvss_base']
df.loc[df['cvss_temporal'].notnull(), 'cvss'] = df['cvss_temporal']
df['cvss3'] = df['cvss3_base']
df.loc[df['cvss3_temporal'].notnull(), 'cvss3'] = df['cvss3_temporal']
df.fillna('', inplace=True)
return df

18
vulnwhisp/frameworks/qualys_vuln.py
View File

@ -90,6 +90,8 @@ class qualysVulnScan:
'title': 'plugin_name'
}
SEVERITY_MAPPING = {0: 'none', 1: 'low', 2: 'medium', 3: 'high',4: 'critical'}
def __init__(
self,
config=None,
@ -184,23 +186,9 @@ class qualysVulnScan:
.apply(lambda x: x[0])
)
# Combine base and temporal
df['cvss_vector'] = (
df[['cvss_vector', 'cvss_temporal_vector']]
.apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1)
.str.rstrip('/nan')
)
df.drop('cvss_temporal_vector', axis=1, inplace=True)
# Convert Qualys severity to standardised risk number
df['risk_number'] = df['severity'].astype(int)-1
# CVSS score = cvss3_temporal or cvss3_base or cvss_temporal or cvss_base
df['cvss'] = df['cvss_base']
df.loc[df['cvss_temporal'].notnull(), 'cvss'] = df['cvss_temporal']
df['cvss3'] = df['cvss3_base']
df.loc[df['cvss3_temporal'].notnull(), 'cvss3'] = df['cvss3_temporal']
df['risk'] = df['risk_number'].map(self.SEVERITY_MAPPING)
df.fillna('', inplace=True)

72
vulnwhisp/vulnwhisp.py
View File

@ -242,6 +242,52 @@ class vulnWhispererBase(object):
scan_names = []
return results
def common_normalise(self, df):
"""Map and transform common data values"""
self.logger.info('Start common mapping')
if 'cvss_base' in df:
self.logger.info('Normalising CVSS')
# CVSS = cvss_temporal or cvss_base
df['cvss'] = df['cvss_base']
df.loc[df['cvss_temporal'].notnull(), 'cvss'] = df['cvss_temporal']
# Map CVSS to severity name
df.loc[df['cvss'] == 0, 'cvss_severity'] = 'info'
df.loc[(df['cvss'] > 0) & (df['cvss'] < 3), 'cvss_severity'] = 'info'
df.loc[(df['cvss'] >= 3) & (df['cvss'] < 6), 'cvss_severity'] = 'medium'
df.loc[(df['cvss'] >= 6) & (df['cvss'] < 9), 'cvss_severity'] = 'high'
df.loc[df['cvss'] > 9, 'cvss_severity'] = 'critical'
if 'cvss3_base' in df:
self.logger.info('Normalising CVSS3')
# CVSS3 = cvss3_temporal or cvss3_base
df['cvss3'] = df['cvss3_base']
df.loc[df['cvss3_temporal'].notnull(), 'cvss3'] = df['cvss3_temporal']
# Map CVSS to severity name
df.loc[df['cvss3'] == 0, 'cvss3_severity'] = 'info'
df.loc[(df['cvss3'] > 0) & (df['cvss3'] < 3), 'cvss3_severity'] = 'info'
df.loc[(df['cvss3'] >= 3) & (df['cvss3'] < 6), 'cvss3_severity'] = 'medium'
df.loc[(df['cvss3'] >= 6) & (df['cvss3'] < 9), 'cvss3_severity'] = 'high'
df.loc[df['cvss3'] > 9, 'cvss3_severity'] = 'critical'
# Combine CVSS and CVSS3 vectors
if 'cvss_vector' in df and 'cvss_temporal_vector' in df:
self.logger.info('Normalising CVSS Vector')
df['cvss_vector'] = (
df[['cvss_vector', 'cvss_temporal_vector']]
.apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1)
.str.rstrip('/nan')
)
df.drop('cvss_temporal_vector', axis=1, inplace=True)
if 'cvss3_vector' in df and 'cvss3_temporal_vector' in df:
self.logger.info('Normalising CVSS Vector')
df['cvss3_vector'] = (
df[['cvss3_vector', 'cvss3_temporal_vector']]
.apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1)
.str.rstrip('/nan')
)
df.drop('cvss3_temporal_vector', axis=1, inplace=True)
return df
class vulnWhispererNessus(vulnWhispererBase):
@ -444,22 +490,23 @@ class vulnWhispererNessus(vulnWhispererBase):
self.exit_code += 1
continue
clean_csv = pd.read_csv(io.StringIO(file_req.decode('utf-8')))
if len(clean_csv) > 2:
vuln_ready = pd.read_csv(io.StringIO(file_req.decode('utf-8')))
if len(vuln_ready) > 2:
self.logger.info('Processing {}/{} for scan: {}'.format(scan_count, len(scan_list), scan_name.encode('utf8')))
# Map and transform fields
clean_csv = self.nessus.normalise(clean_csv)
vuln_ready = self.nessus.normalise(vuln_ready)
vuln_ready = self.common_normalise(vuln_ready)
# Set common fields
clean_csv['scan_name'] = scan_name.encode('utf8')
clean_csv['scan_id'] = uuid
vuln_ready['scan_name'] = scan_name.encode('utf8')
vuln_ready['scan_id'] = uuid
# Add timestamp and convert to milliseconds
clean_csv['_timestamp'] = norm_time
clean_csv['scan_source'] = self.CONFIG_SECTION
vuln_ready['_timestamp'] = norm_time
vuln_ready['scan_source'] = self.CONFIG_SECTION
clean_csv.to_json(relative_path_name, orient='records', lines=True)
vuln_ready.to_json(relative_path_name, orient='records', lines=True)
record_meta = (
scan_name,
@ -467,14 +514,14 @@ class vulnWhispererNessus(vulnWhispererBase):
norm_time,
file_name,
time.time(),
clean_csv.shape[0],
vuln_ready.shape[0],
self.CONFIG_SECTION,
uuid,
1,
0,
)
self.record_insert(record_meta)
self.logger.info('{filename} records written to {path} '.format(filename=clean_csv.shape[0],
self.logger.info('{filename} records written to {path} '.format(filename=vuln_ready.shape[0],
path=file_name.encode('utf8')))
else:
record_meta = (
@ -483,7 +530,7 @@ class vulnWhispererNessus(vulnWhispererBase):
norm_time,
file_name,
time.time(),
clean_csv.shape[0],
vuln_ready.shape[0],
self.CONFIG_SECTION,
uuid,
1,
@ -623,6 +670,7 @@ class vulnWhispererQualys(vulnWhispererBase):
vuln_ready = self.qualys_scan.process_data(path=self.write_path, file_id=str(generated_report_id))
# Map and transform fields
vuln_ready = self.qualys_scan.normalise(vuln_ready)
vuln_ready = self.common_normalise(vuln_ready)
# TODO remove the line below once normalising complete
vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
@ -795,6 +843,7 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
vuln_ready = self.openvas_api.process_report(report_id=report_id)
# Map and transform fields
vuln_ready = self.openvas_api.normalise(vuln_ready)
vuln_ready = self.common_normalise(vuln_ready)
# TODO move the following to the openvas_api.transform_values
vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
vuln_ready.port = vuln_ready.port.fillna(0).astype(int)
@ -900,6 +949,7 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
vuln_ready = self.qualys_scan.process_data(scan_id=report_id)
# Map and transform fields
vuln_ready = self.qualys_scan.normalise(vuln_ready)
vuln_ready = self.common_normalise(vuln_ready)
# Set common fields
vuln_ready['scan_name'] = scan_name.encode('utf8')