rename plugin to signature and cvss_x to cvss2_x

This commit is contained in:
pemontto
2019-05-03 16:25:29 +01:00
parent a1671a953f
commit fb76b0a1ce
8 changed files with 79 additions and 73 deletions

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -29,21 +29,21 @@
"cve": { "cve": {
"type": "keyword" "type": "keyword"
}, },
"cvss_base": {
"type": "float"
},
"cvss_severity": {
"type": "keyword"
},
"cvss_temporal": {
"type": "float"
},
"cvss_vector": {
"type": "keyword"
},
"cvss": { "cvss": {
"type": "float" "type": "float"
}, },
"cvss2_base": {
"type": "float"
},
"cvss2_severity": {
"type": "keyword"
},
"cvss2_temporal": {
"type": "float"
},
"cvss2_vector": {
"type": "keyword"
},
"cvss3_base": { "cvss3_base": {
"type": "float" "type": "float"
}, },
@ -136,10 +136,10 @@
"plugin_family": { "plugin_family": {
"type": "keyword" "type": "keyword"
}, },
"plugin_id": { "signature_id": {
"type": "keyword" "type": "keyword"
}, },
"plugin_name": { "signature": {
"type": "keyword" "type": "keyword"
}, },
"plugin_output": { "plugin_output": {

View File

@ -61,8 +61,8 @@ fi
# ((return_code = return_code + 1)) # ((return_code = return_code + 1))
# fi # fi
# Test Nessus plugin_name:Backported Security Patch Detection (FTP) # Test Nessus signature:Backported Security Patch Detection (FTP)
nessus_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_search?q=plugin_name:%22Backported%20Security%20Patch%20Detection%20(FTP)%22%20AND%20asset:176.28.50.164%20AND%20tags:nessus" | jq '.hits.hits[]._source') nessus_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_search?q=signature:%22Backported%20Security%20Patch%20Detection%20(FTP)%22%20AND%20asset:176.28.50.164%20AND%20tags:nessus" | jq '.hits.hits[]._source')
if echo $nessus_doc | jq '.risk' | grep -q "none"; then if echo $nessus_doc | jq '.risk' | grep -q "none"; then
green "✅ Passed: Nessus risk == none" green "✅ Passed: Nessus risk == none"
else else
@ -70,8 +70,8 @@ else
((return_code = return_code + 1)) ((return_code = return_code + 1))
fi fi
# Test Tenable plugin_name:Backported Security Patch Detection (FTP) # Test Tenable signature:Backported Security Patch Detection (FTP)
tenable_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_search?q=plugin_name:%22Backported%20Security%20Patch%20Detection%20(FTP)%22%20AND%20asset:176.28.50.164%20AND%20tags:tenable" | jq '.hits.hits[]._source') tenable_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_search?q=signature:%22Backported%20Security%20Patch%20Detection%20(FTP)%22%20AND%20asset:176.28.50.164%20AND%20tags:tenable" | jq '.hits.hits[]._source')
# Test asset # Test asset
if echo $tenable_doc | jq .asset | grep -q '176.28.50.164'; then if echo $tenable_doc | jq .asset | grep -q '176.28.50.164'; then
green "✅ Passed: Tenable asset == 176.28.50.164" green "✅ Passed: Tenable asset == 176.28.50.164"
@ -88,8 +88,8 @@ else
((return_code = return_code + 1)) ((return_code = return_code + 1))
fi fi
# Test Qualys plugin_name:OpenSSL Multiple Remote Security Vulnerabilities # Test Qualys signature:OpenSSL Multiple Remote Security Vulnerabilities
qualys_vuln_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_search?q=tags:qualys_vuln%20AND%20ip:%22176.28.50.164%22%20AND%20plugin_name:%22OpenSSL%20Multiple%20Remote%20Security%20Vulnerabilities%22%20AND%20port:465" | jq '.hits.hits[]._source') qualys_vuln_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-*/_search?q=tags:qualys_vuln%20AND%20ip:%22176.28.50.164%22%20AND%20signature:%22OpenSSL%20Multiple%20Remote%20Security%20Vulnerabilities%22%20AND%20port:465" | jq '.hits.hits[]._source')
# Test @timestamp # Test @timestamp
if echo $qualys_vuln_doc | jq '.["@timestamp"]' | grep -q '2019-03-30T10:17:41.000Z'; then if echo $qualys_vuln_doc | jq '.["@timestamp"]' | grep -q '2019-03-30T10:17:41.000Z'; then
green "✅ Passed: Qualys VM @timestamp == 2019-03-30T10:17:41.000Z" green "✅ Passed: Qualys VM @timestamp == 2019-03-30T10:17:41.000Z"

View File

@ -17,7 +17,7 @@ class NessusAPI(object):
SCANS = '/scans' SCANS = '/scans'
SCAN_ID = SCANS + '/{scan_id}' SCAN_ID = SCANS + '/{scan_id}'
HOST_VULN = SCAN_ID + '/hosts/{host_id}' HOST_VULN = SCAN_ID + '/hosts/{host_id}'
PLUGINS = HOST_VULN + '/plugins/{plugin_id}' PLUGINS = HOST_VULN + '/plugins/{signature_id}'
EXPORT = SCAN_ID + '/export' EXPORT = SCAN_ID + '/export'
EXPORT_TOKEN_DOWNLOAD = '/scans/exports/{token_id}/download' EXPORT_TOKEN_DOWNLOAD = '/scans/exports/{token_id}/download'
EXPORT_FILE_DOWNLOAD = EXPORT + '/{file_id}/download' EXPORT_FILE_DOWNLOAD = EXPORT + '/{file_id}/download'
@ -25,17 +25,19 @@ class NessusAPI(object):
EXPORT_HISTORY = EXPORT + '?history_id={history_id}' EXPORT_HISTORY = EXPORT + '?history_id={history_id}'
# All column mappings should be lowercase # All column mappings should be lowercase
COLUMN_MAPPING = { COLUMN_MAPPING = {
'cvss base score': 'cvss_base', 'cvss base score': 'cvss2_base',
'cvss temporal score': 'cvss_temporal', 'cvss temporal score': 'cvss2_temporal',
'cvss temporal vector': 'cvss_temporal_vector', 'cvss temporal vector': 'cvss2_temporal_vector',
'cvss vector': 'cvss2_vector',
'cvss3 base score': 'cvss3_base', 'cvss3 base score': 'cvss3_base',
'cvss3 temporal score': 'cvss3_temporal', 'cvss3 temporal score': 'cvss3_temporal',
'cvss3 temporal vector': 'cvss3_temporal_vector', 'cvss3 temporal vector': 'cvss3_temporal_vector',
'fqdn': 'dns', 'fqdn': 'dns',
'host': 'asset', 'host': 'asset',
'ip address': 'ip', 'ip address': 'ip',
'name': 'plugin_name', 'name': 'signature',
'os': 'operating_system', 'os': 'operating_system',
'plugin id': 'signature_id',
'see also': 'exploitability', 'see also': 'exploitability',
'system type': 'category', 'system type': 'category',
'vulnerability state': 'state' 'vulnerability state': 'state'

View File

@ -83,10 +83,10 @@ class qualysVulnScan:
'impact': 'synopsis', 'impact': 'synopsis',
'ip_status': 'state', 'ip_status': 'state',
'os': 'operating_system', 'os': 'operating_system',
'qid': 'plugin_id', 'qid': 'signature_id',
'results': 'plugin_output', 'results': 'plugin_output',
'threat': 'description', 'threat': 'description',
'title': 'plugin_name' 'title': 'signature'
} }
SEVERITY_MAPPING = {0: 'none', 1: 'low', 2: 'medium', 3: 'high',4: 'critical'} SEVERITY_MAPPING = {0: 'none', 1: 'low', 2: 'medium', 3: 'high',4: 'critical'}
@ -164,10 +164,12 @@ class qualysVulnScan:
# Contruct the CVSS vector # Contruct the CVSS vector
self.logger.info('Extracting CVSS components') self.logger.info('Extracting CVSS components')
df['cvss_vector'] = df['cvss_base'].str.extract('\((.*)\)', expand=False) df['cvss2_vector'] = df['cvss_base'].str.extract('\((.*)\)', expand=False)
df['cvss_base'] = df['cvss_base'].str.extract('^(\d+(?:\.\d+)?)', expand=False) df['cvss2_base'] = df['cvss_base'].str.extract('^(\d+(?:\.\d+)?)', expand=False)
df['cvss_temporal_vector'] = df['cvss_temporal'].str.extract('\((.*)\)', expand=False) df['cvss2_temporal_vector'] = df['cvss_temporal'].str.extract('\((.*)\)', expand=False)
df['cvss_temporal'] = df['cvss_temporal'].str.extract('^(\d+(?:\.\d+)?)', expand=False) df['cvss2_temporal'] = df['cvss_temporal'].str.extract('^(\d+(?:\.\d+)?)', expand=False)
df.drop('cvss_base', axis=1, inplace=True, errors='ignore')
df.drop('cvss_temporal', axis=1, inplace=True, errors='ignore')
# Set asset to ip # Set asset to ip
df['asset'] = df['ip'] df['asset'] = df['ip']

View File

@ -289,12 +289,12 @@ class qualysScanReport:
'DescriptionSeverity': 'synopsis', 'DescriptionSeverity': 'synopsis',
'Evidence #1': 'evidence', 'Evidence #1': 'evidence',
'Payload #1': 'payload', 'Payload #1': 'payload',
'QID': 'plugin_id', 'QID': 'signature_id',
'Request Headers #1': 'request_headers', 'Request Headers #1': 'request_headers',
'Request Method #1': 'request_method', 'Request Method #1': 'request_method',
'Request URL #1': 'request_url', 'Request URL #1': 'request_url',
'Response #1': 'plugin_output', 'Response #1': 'plugin_output',
'Title': 'plugin_name', 'Title': 'signature',
'Url': 'uri', 'Url': 'uri',
'URL': 'url', 'URL': 'url',
'Vulnerability Category': 'type', 'Vulnerability Category': 'type',

View File

@ -249,15 +249,17 @@ class vulnWhispererBase(object):
"""Map and transform common data values""" """Map and transform common data values"""
self.logger.info('Start common normalisation') self.logger.info('Start common normalisation')
df.replace({'': np.nan}, inplace=True)
self.logger.debug('Normalising CVSS') self.logger.debug('Normalising CVSS')
for cvss_version in ['cvss', 'cvss3']: for cvss_version in ['cvss2', 'cvss3']:
# cvssX = cvssX_temporal else cvssX_base
if cvss_version + '_base' in df: if cvss_version + '_base' in df:
self.logger.debug('Normalising {} base'.format(cvss_version)) self.logger.debug('Normalising {} base'.format(cvss_version))
# CVSS = cvss_temporal or cvss_base
df[cvss_version] = df[cvss_version + '_base'] df[cvss_version] = df[cvss_version + '_base']
df.loc[df[cvss_version + '_temporal'] != '', cvss_version] = df[cvss_version + '_temporal'] df[cvss_version] = df[cvss_version + '_temporal'].fillna(df[cvss_version])
# Combine CVSS and CVSS3 vectors # Combine cvssX temporal and base vectors
if cvss_version + '_vector' in df and cvss_version + '_temporal_vector' in df: if cvss_version + '_vector' in df and cvss_version + '_temporal_vector' in df:
self.logger.debug('Normalising {} vector'.format(cvss_version)) self.logger.debug('Normalising {} vector'.format(cvss_version))
df[cvss_version + '_vector'] = ( df[cvss_version + '_vector'] = (
@ -267,16 +269,22 @@ class vulnWhispererBase(object):
) )
df.drop(cvss_version + '_temporal_vector', axis=1, inplace=True) df.drop(cvss_version + '_temporal_vector', axis=1, inplace=True)
# Map cvssX to severity name
if cvss_version in df: if cvss_version in df:
self.logger.debug('Normalising {} severity'.format(cvss_version)) self.logger.debug('Normalising {} severity'.format(cvss_version))
# Map CVSS to severity name
df.loc[df[cvss_version].astype(str) == '', cvss_version] = None
df[cvss_version] = df[cvss_version].astype('float') df[cvss_version] = df[cvss_version].astype('float')
df[cvss_version + '_severity'] = 'informational' df[cvss_version + '_severity'] = 'informational'
df.loc[(df[cvss_version] > 0) & (df[cvss_version] < 3), cvss_version + '_severity'] = 'low' df.loc[(df[cvss_version] > 0) & (df[cvss_version] < 3), cvss_version + '_severity'] = 'low'
df.loc[(df[cvss_version] >= 3) & (df[cvss_version] < 6), cvss_version + '_severity'] = 'medium' df.loc[(df[cvss_version] >= 3) & (df[cvss_version] < 6), cvss_version + '_severity'] = 'medium'
df.loc[(df[cvss_version] >= 6) & (df[cvss_version] < 9), cvss_version + '_severity'] = 'high' df.loc[(df[cvss_version] >= 6) & (df[cvss_version] < 9), cvss_version + '_severity'] = 'high'
df.loc[(df[cvss_version] > 9) & (df[cvss_version].notnull()), cvss_version + '_severity'] = 'critical' df.loc[df[cvss_version] > 9, cvss_version + '_severity'] = 'critical'
# Get a single cvss score derived from cvss3 else cvss2
if not 'cvss' in df:
if 'cvss3' in df:
df['cvss'] = df['cvss3'].fillna(df['cvss2'])
elif 'cvss2' in df:
df['cvss'] = df['cvss2']
self.logger.debug('Creating Unique Document ID') self.logger.debug('Creating Unique Document ID')
df['_unique'] = df.index.values df['_unique'] = df.index.values
@ -285,12 +293,6 @@ class vulnWhispererBase(object):
else: else:
df['_unique'] = df[['scan_id', '_unique']].apply(lambda x: '_'.join(x.astype(str)), axis=1) df['_unique'] = df[['scan_id', '_unique']].apply(lambda x: '_'.join(x.astype(str)), axis=1)
# Rename cvss to cvss2
# Make cvss with no suffix == cvss3 else cvss2
# cvss = cvss3 if cvss3 else cvss2
# cvss_severity = cvss3_severity if cvss3_severity else cvss2_severity
df.replace({'': np.nan}, inplace=True)
return df return df
@ -700,7 +702,7 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
'CVSS': 'cvss', 'CVSS': 'cvss',
'Severity': 'severity', 'Severity': 'severity',
'Solution Type': 'category', 'Solution Type': 'category',
'NVT Name': 'plugin_name', 'NVT Name': 'signature',
'Summary': 'synopsis', 'Summary': 'synopsis',
'Specific Result': 'plugin_output', 'Specific Result': 'plugin_output',
'NVT OID': 'nvt_oid', 'NVT OID': 'nvt_oid',
@ -1141,16 +1143,16 @@ class vulnWhispererJIRA(vulnWhispererBase):
continue continue
elif data[index]['type'] == 'Practice' or data[index]['type'] == 'Ig': elif data[index]['type'] == 'Practice' or data[index]['type'] == 'Ig':
self.logger.debug("Vulnerability '{vuln}' ignored, as it is 'Practice/Potential', not verified.".format(vuln=data[index]['plugin_name'])) self.logger.debug("Vulnerability '{vuln}' ignored, as it is 'Practice/Potential', not verified.".format(vuln=data[index]['signature']))
continue continue
if not vulnerabilities or data[index]['plugin_name'] not in [entry['title'] for entry in vulnerabilities]: if not vulnerabilities or data[index]['signature'] not in [entry['title'] for entry in vulnerabilities]:
vuln = {} vuln = {}
#vulnerabilities should have all the info for creating all JIRA labels #vulnerabilities should have all the info for creating all JIRA labels
vuln['source'] = source vuln['source'] = source
vuln['scan_name'] = scan_name vuln['scan_name'] = scan_name
#vulnerability variables #vulnerability variables
vuln['title'] = data[index]['plugin_name'] vuln['title'] = data[index]['signature']
vuln['diagnosis'] = data[index]['threat'].replace('\\n',' ') vuln['diagnosis'] = data[index]['threat'].replace('\\n',' ')
vuln['consequence'] = data[index]['impact'].replace('\\n',' ') vuln['consequence'] = data[index]['impact'].replace('\\n',' ')
vuln['solution'] = data[index]['solution'].replace('\\n',' ') vuln['solution'] = data[index]['solution'].replace('\\n',' ')
@ -1171,7 +1173,7 @@ class vulnWhispererJIRA(vulnWhispererBase):
else: else:
# grouping assets by vulnerability to open on single ticket, as each asset has its own nessus entry # grouping assets by vulnerability to open on single ticket, as each asset has its own nessus entry
for vuln in vulnerabilities: for vuln in vulnerabilities:
if vuln['title'] == data[index]['plugin_name']: if vuln['title'] == data[index]['signature']:
vuln['ips'].append("{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv))) vuln['ips'].append("{ip} - {protocol}/{port} - {dns}".format(**self.get_asset_fields(data[index], dns_resolv)))
return vulnerabilities return vulnerabilities