Cyberdefense/VulnWhisperer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move to a common normalisation function

Browse Source
This commit is contained in:
pemontto
2019-04-15 21:12:07 +10:00
parent dd66414fe7
commit ee327874e5
6 changed files with 102 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
vulnwhisp/frameworks/nessus.py
View File

@ -217,6 +217,7 @@ class NessusAPI(object):
self.logger.debug('Changing case of fields')
df['cve'] = df['cve'].str.upper()
df['protocol'] = df['protocol'].str.lower()
df['risk'] = df['risk'].str.lower()
# Copy asset to IP
df['ip'] = df['asset']
@ -225,27 +226,5 @@ class NessusAPI(object):
self.logger.debug('Mapping risk to severity number')
df['risk_number'] = df['risk'].str.lower().map(self.SEVERITY_MAPPING)
if self.profile == 'tenable':
self.logger.debug('Combinging CVSS vectors for tenable')
# Combine CVSS vectors
df['cvss_vector'] = (
df[['cvss_vector', 'cvss_temporal_vector']]
.apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1)
.str.rstrip('/nan')
)
df['cvss3_vector'] = (
df[['cvss3_vector', 'cvss3_temporal_vector']]
.apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1)
.str.rstrip('/nan')
)
df.drop(['cvss_temporal_vector', 'cvss3_temporal_vector'], axis=1, inplace=True)
# CVSS score = cvss3_temporal or cvss3_base or cvss_temporal or cvss_base
df['cvss'] = df['cvss_base']
df.loc[df['cvss_temporal'].notnull(), 'cvss'] = df['cvss_temporal']
df['cvss3'] = df['cvss3_base']
df.loc[df['cvss3_temporal'].notnull(), 'cvss3'] = df['cvss3_temporal']
df.fillna('', inplace=True)
return df

18
vulnwhisp/frameworks/qualys_vuln.py
View File

@ -90,6 +90,8 @@ class qualysVulnScan:
'title': 'plugin_name'
}
SEVERITY_MAPPING = {0: 'none', 1: 'low', 2: 'medium', 3: 'high',4: 'critical'}
def __init__(
self,
config=None,
@ -184,23 +186,9 @@ class qualysVulnScan:
.apply(lambda x: x[0])
)
# Combine base and temporal
df['cvss_vector'] = (
df[['cvss_vector', 'cvss_temporal_vector']]
.apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1)
.str.rstrip('/nan')
)
df.drop('cvss_temporal_vector', axis=1, inplace=True)
# Convert Qualys severity to standardised risk number
df['risk_number'] = df['severity'].astype(int)-1
# CVSS score = cvss3_temporal or cvss3_base or cvss_temporal or cvss_base
df['cvss'] = df['cvss_base']
df.loc[df['cvss_temporal'].notnull(), 'cvss'] = df['cvss_temporal']
df['cvss3'] = df['cvss3_base']
df.loc[df['cvss3_temporal'].notnull(), 'cvss3'] = df['cvss3_temporal']
df['risk'] = df['risk_number'].map(self.SEVERITY_MAPPING)
df.fillna('', inplace=True)

72
vulnwhisp/vulnwhisp.py
View File

@ -242,6 +242,52 @@ class vulnWhispererBase(object):
scan_names = []
return results
def common_normalise(self, df):
"""Map and transform common data values"""
self.logger.info('Start common mapping')
if 'cvss_base' in df:
self.logger.info('Normalising CVSS')
# CVSS = cvss_temporal or cvss_base
df['cvss'] = df['cvss_base']
df.loc[df['cvss_temporal'].notnull(), 'cvss'] = df['cvss_temporal']
# Map CVSS to severity name
df.loc[df['cvss'] == 0, 'cvss_severity'] = 'info'
df.loc[(df['cvss'] > 0) & (df['cvss'] < 3), 'cvss_severity'] = 'info'
df.loc[(df['cvss'] >= 3) & (df['cvss'] < 6), 'cvss_severity'] = 'medium'
df.loc[(df['cvss'] >= 6) & (df['cvss'] < 9), 'cvss_severity'] = 'high'
df.loc[df['cvss'] > 9, 'cvss_severity'] = 'critical'
if 'cvss3_base' in df:
self.logger.info('Normalising CVSS3')
# CVSS3 = cvss3_temporal or cvss3_base
df['cvss3'] = df['cvss3_base']
df.loc[df['cvss3_temporal'].notnull(), 'cvss3'] = df['cvss3_temporal']
# Map CVSS to severity name
df.loc[df['cvss3'] == 0, 'cvss3_severity'] = 'info'
df.loc[(df['cvss3'] > 0) & (df['cvss3'] < 3), 'cvss3_severity'] = 'info'
df.loc[(df['cvss3'] >= 3) & (df['cvss3'] < 6), 'cvss3_severity'] = 'medium'
df.loc[(df['cvss3'] >= 6) & (df['cvss3'] < 9), 'cvss3_severity'] = 'high'
df.loc[df['cvss3'] > 9, 'cvss3_severity'] = 'critical'
# Combine CVSS and CVSS3 vectors
if 'cvss_vector' in df and 'cvss_temporal_vector' in df:
self.logger.info('Normalising CVSS Vector')
df['cvss_vector'] = (
df[['cvss_vector', 'cvss_temporal_vector']]
.apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1)
.str.rstrip('/nan')
)
df.drop('cvss_temporal_vector', axis=1, inplace=True)
if 'cvss3_vector' in df and 'cvss3_temporal_vector' in df:
self.logger.info('Normalising CVSS Vector')
df['cvss3_vector'] = (
df[['cvss3_vector', 'cvss3_temporal_vector']]
.apply(lambda x: '{}/{}'.format(x[0], x[1]), axis=1)
.str.rstrip('/nan')
)
df.drop('cvss3_temporal_vector', axis=1, inplace=True)
return df
class vulnWhispererNessus(vulnWhispererBase):
@ -444,22 +490,23 @@ class vulnWhispererNessus(vulnWhispererBase):
self.exit_code += 1
continue
clean_csv = pd.read_csv(io.StringIO(file_req.decode('utf-8')))
if len(clean_csv) > 2:
vuln_ready = pd.read_csv(io.StringIO(file_req.decode('utf-8')))
if len(vuln_ready) > 2:
self.logger.info('Processing {}/{} for scan: {}'.format(scan_count, len(scan_list), scan_name.encode('utf8')))
# Map and transform fields
clean_csv = self.nessus.normalise(clean_csv)
vuln_ready = self.nessus.normalise(vuln_ready)
vuln_ready = self.common_normalise(vuln_ready)
# Set common fields
clean_csv['scan_name'] = scan_name.encode('utf8')
clean_csv['scan_id'] = uuid
vuln_ready['scan_name'] = scan_name.encode('utf8')
vuln_ready['scan_id'] = uuid
# Add timestamp and convert to milliseconds
clean_csv['_timestamp'] = norm_time
clean_csv['scan_source'] = self.CONFIG_SECTION
vuln_ready['_timestamp'] = norm_time
vuln_ready['scan_source'] = self.CONFIG_SECTION
clean_csv.to_json(relative_path_name, orient='records', lines=True)
vuln_ready.to_json(relative_path_name, orient='records', lines=True)
record_meta = (
scan_name,
@ -467,14 +514,14 @@ class vulnWhispererNessus(vulnWhispererBase):
norm_time,
file_name,
time.time(),
clean_csv.shape[0],
vuln_ready.shape[0],
self.CONFIG_SECTION,
uuid,
1,
0,
)
self.record_insert(record_meta)
self.logger.info('{filename} records written to {path} '.format(filename=clean_csv.shape[0],
self.logger.info('{filename} records written to {path} '.format(filename=vuln_ready.shape[0],
path=file_name.encode('utf8')))
else:
record_meta = (
@ -483,7 +530,7 @@ class vulnWhispererNessus(vulnWhispererBase):
norm_time,
file_name,
time.time(),
clean_csv.shape[0],
vuln_ready.shape[0],
self.CONFIG_SECTION,
uuid,
1,
@ -623,6 +670,7 @@ class vulnWhispererQualys(vulnWhispererBase):
vuln_ready = self.qualys_scan.process_data(path=self.write_path, file_id=str(generated_report_id))
# Map and transform fields
vuln_ready = self.qualys_scan.normalise(vuln_ready)
vuln_ready = self.common_normalise(vuln_ready)
# TODO remove the line below once normalising complete
vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
@ -795,6 +843,7 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
vuln_ready = self.openvas_api.process_report(report_id=report_id)
# Map and transform fields
vuln_ready = self.openvas_api.normalise(vuln_ready)
vuln_ready = self.common_normalise(vuln_ready)
# TODO move the following to the openvas_api.transform_values
vuln_ready.rename(columns=self.COLUMN_MAPPING, inplace=True)
vuln_ready.port = vuln_ready.port.fillna(0).astype(int)
@ -900,6 +949,7 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
vuln_ready = self.qualys_scan.process_data(scan_id=report_id)
# Map and transform fields
vuln_ready = self.qualys_scan.normalise(vuln_ready)
vuln_ready = self.common_normalise(vuln_ready)
# Set common fields
vuln_ready['scan_name'] = scan_name.encode('utf8')