update qualysapi to latest + PR and refactored vulnwhisperer qualys module to qualys-web (#108)

* update qualysapi to latest + PR and refactored vulnwhisperer qualys module to qualys-web

* changing config template paths for qualys

* Update frameworks_example.ini

Will leave for now qualys local folder as "qualys" instead of changing to one for each module, as like this it will still be compatible with the current logstash and we will be able to update master to drop the qualysapi fork once the new version is uploaded to PyPI repository.
PR from qualysapi repo has already been merged, so the only missing is the upload to PyPI.

configs/frameworks_example.ini

@@ -20,7 +20,7 @@ db_path=/opt/VulnWhisperer/data/database
 trash=false
 verbose=true
 
-[qualys]
+[qualys_web]
 #Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
 enabled = true
 hostname = qualysapi.qg2.apps.qualys.com

deps/qualysapi/examples/qualysapi-section-example.py

@@ -0,0 +1,113 @@
+__author__ = 'Parag Baxi <parag.baxi@gmail.com>'
+__license__ = 'Apache License 2.0'
+
+import qualysapi
+from lxml import objectify
+from lxml.builder import E
+
+# Setup connection to QualysGuard API.
+qgc = qualysapi.connect('config.txt', 'qualys_vuln')
+#
+# API v1 call: Scan the New York & Las Vegas asset groups
+# The call is our request's first parameter.
+call = 'scan.php'
+# The parameters to append to the url is our request's second parameter.
+parameters = {'scan_title': 'Go big or go home', 'asset_groups': 'New York&Las Vegas', 'option': 'Initial+Options'}
+# Note qualysapi will automatically convert spaces into plus signs for API v1 & v2.
+# Let's call the API and store the result in xml_output.
+xml_output = qgc.request(call, parameters, concurrent_scans_retries=2, concurrent_scans_retry_delay=600)
+# concurrent_retries: Retry the call this many times if your subscription hits the concurrent scans limit.
+# concurrent_retries: Delay in seconds between retrying when subscription hits the concurrent scans limit.
+# Example XML response when this happens below:
+#     <?xml version="1.0" encoding="UTF-8"?>
+#     <ServiceResponse xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="http://localhost:50205/qps/rest/app//xsd/3.0/was/wasscan.xsd">
+#       <responseCode>INVALID_REQUEST</responseCode>
+#       <responseErrorDetails>
+#         <errorMessage>You have reached the maximum number of concurrent running scans (10) for your account</errorMessage>
+#         <errorResolution>Please wait until your previous scans have completed</errorResolution>
+#       </responseErrorDetails>
+#
+print(xml_output)
+#
+# API v1 call: Print out all IPs associated with asset group "Looneyville Texas".
+# Note that the question mark at the end is optional.
+call = 'asset_group_list.php?'
+# We can still use strings for the data (not recommended).
+parameters = 'title=Looneyville Texas'
+# Let's call the API and store the result in xml_output.
+xml_output = qgc.request(call, parameters)
+# Let's objectify the xml_output string.
+root = objectify.fromstring(xml_output)
+# Print out the IPs.
+print(root.ASSET_GROUP.SCANIPS.IP.text)
+# Prints out:
+# 10.0.0.102
+#
+# API v2 call: Print out DNS name for a range of IPs.
+call = '/api/2.0/fo/asset/host/'
+parameters = {'action': 'list', 'ips': '10.0.0.10-10.0.0.11'}
+xml_output = qgc.request(call, parameters)
+root = objectify.fromstring(xml_output)
+# Iterate hosts and print out DNS name.
+for host in root.RESPONSE.HOST_LIST.HOST:
+    print(host.IP.text, host.DNS.text)
+# Prints out:
+# 10.0.0.10 mydns1.qualys.com
+# 10.0.0.11 mydns2.qualys.com
+#
+# API v3 WAS call: Print out number of webapps.
+call = '/count/was/webapp'
+# Note that this call does not have a payload so we don't send any data parameters.
+xml_output = qgc.request(call)
+root = objectify.fromstring(xml_output)
+# Print out count of webapps.
+print(root.count.text)
+# Prints out:
+# 89
+#
+# API v3 WAS call: Print out number of webapps containing title 'Supafly'.
+call = '/count/was/webapp'
+# We can send a string XML for the data.
+parameters = '<ServiceRequest><filters><Criteria operator="CONTAINS" field="name">Supafly</Criteria></filters></ServiceRequest>'
+xml_output = qgc.request(call, parameters)
+root = objectify.fromstring(xml_output)
+# Print out count of webapps.
+print(root.count.text)
+# Prints out:
+# 3
+#
+# API v3 WAS call: Print out number of webapps containing title 'Lightsabertooth Tiger'.
+call = '/count/was/webapp'
+# We can also send an lxml.builder E object.
+parameters = (
+    E.ServiceRequest(
+        E.filters(
+            E.Criteria('Lightsabertooth Tiger', field='name',operator='CONTAINS'))))
+xml_output = qgc.request(call, parameters)
+root = objectify.fromstring(xml_output)
+# Print out count of webapps.
+print(root.count.text)
+# Prints out:
+# 0
+# Too bad, because that is an awesome webapp name!
+#
+# API v3 Asset Management call: Count tags.
+call = '/count/am/tag'
+xml_output = qgc.request(call)
+root = objectify.fromstring(xml_output)
+# We can use XPATH to find the count.
+print(root.xpath('count')[0].text)
+# Prints out:
+# 840
+#
+# API v3 Asset Management call: Find asset by name.
+call = '/search/am/tag'
+parameters = '''<ServiceRequest>
+        <preferences>
+            <limitResults>10</limitResults>
+        </preferences>
+        <filters>
+            <Criteria field="name" operator="CONTAINS">PB</Criteria>
+        </filters>
+    </ServiceRequest>'''
+xml_output = qgc.request(call, parameters)

deps/qualysapi/qualysapi/api_actions.py

@@ -5,7 +5,7 @@ from qualysapi.api_objects import *
 
 
 class QGActions(object):
-    def getHost(host):
+    def getHost(self, host):
         call = '/api/2.0/fo/asset/host/'
         parameters = {'action': 'list', 'ips': host, 'details': 'All'}
         hostData = objectify.fromstring(self.request(call, parameters)).RESPONSE

deps/qualysapi/qualysapi/api_objects.py

@@ -27,13 +27,13 @@ class AssetGroup(object):
         self.scanner_appliances = scanner_appliances
         self.title = str(title)
 
-    def addAsset(conn, ip):
+    def addAsset(self, conn, ip):
         call = '/api/2.0/fo/asset/group/'
         parameters = {'action': 'edit', 'id': self.id, 'add_ips': ip}
         conn.request(call, parameters)
         self.scanips.append(ip)
 
-    def setAssets(conn, ips):
+    def setAssets(self, conn, ips):
         call = '/api/2.0/fo/asset/group/'
         parameters = {'action': 'edit', 'id': self.id, 'set_ips': ips}
         conn.request(call, parameters)

deps/qualysapi/qualysapi/config.py

@@ -21,7 +21,6 @@ logger = logging.getLogger(__name__)
 
 
 __author__ = "Parag Baxi <parag.baxi@gmail.com> & Colin Bell <colin.bell@uwaterloo.ca>"
-__updated_by__ = "Austin Taylor <vulnWhisperer@austintaylor.io>"
 __copyright__ = "Copyright 2011-2013, Parag Baxi & University of Waterloo"
 __license__ = "BSD-new"
 
@@ -31,10 +30,10 @@ class QualysConnectConfig:
     from an ini file.
     """
 
-    def __init__(self, filename=qcs.default_filename, remember_me=False, remember_me_always=False):
+    def __init__(self, filename=qcs.default_filename, section='info', remember_me=False, remember_me_always=False):
 
         self._cfgfile = None
-
+        self._section = section
         # Prioritize local directory filename.
         # Check for file existence.
         if os.path.exists(filename):
@@ -53,50 +52,36 @@ class QualysConnectConfig:
 
             # apply bitmask to current mode to check ONLY user access permissions.
             if (mode & (stat.S_IRWXG | stat.S_IRWXO)) != 0:
-                logging.warning('%s permissions allows more than user access.' % (filename,))
+                logger.warning('%s permissions allows more than user access.' % (filename,))
 
             self._cfgparse.read(self._cfgfile)
 
-        # if 'info' doesn't exist, create the section.
+        # if 'info'/ specified section doesn't exist, create the section.
-        if not self._cfgparse.has_section('qualys'):
+        if not self._cfgparse.has_section(self._section):
-            self._cfgparse.add_section('qualys')
+            self._cfgparse.add_section(self._section)
 
         # Use default hostname (if one isn't provided).
-        if not self._cfgparse.has_option('qualys', 'hostname'):
+        if not self._cfgparse.has_option(self._section, 'hostname'):
             if self._cfgparse.has_option('DEFAULT', 'hostname'):
                 hostname = self._cfgparse.get('DEFAULT', 'hostname')
-                self._cfgparse.set('qualys', 'hostname', hostname)
+                self._cfgparse.set(self._section, 'hostname', hostname)
             else:
                 raise Exception("No 'hostname' set. QualysConnect does not know who to connect to.")
 
         # Use default max_retries (if one isn't provided).
-        if not self._cfgparse.has_option('qualys', 'max_retries'):
+        if not self._cfgparse.has_option(self._section, 'max_retries'):
             self.max_retries = qcs.defaults['max_retries']
         else:
-            self.max_retries = self._cfgparse.get('qualys', 'max_retries')
+            self.max_retries = self._cfgparse.get(self._section, 'max_retries')
             try:
                 self.max_retries = int(self.max_retries)
             except Exception:
                 logger.error('Value max_retries must be an integer.')
                 print('Value max_retries must be an integer.')
                 exit(1)
-            self._cfgparse.set('qualys', 'max_retries', str(self.max_retries))
+            self._cfgparse.set(self._section, 'max_retries', str(self.max_retries))
         self.max_retries = int(self.max_retries)
 
-        #Get template ID... user will need to set this to pull back CSV reports
-        if not self._cfgparse.has_option('qualys', 'template_id'):
-            self.report_template_id = qcs.defaults['template_id']
-        else:
-            self.report_template_id = self._cfgparse.get('qualys', 'template_id')
-            try:
-                self.report_template_id = int(self.report_template_id)
-            except Exception:
-                logger.error('Report Template ID Must be set and be an integer')
-                print('Value template ID must be an integer.')
-                exit(1)
-            self._cfgparse.set('qualys', 'template_id', str(self.report_template_id))
-        self.report_template_id = int(self.report_template_id)
-
         # Proxy support
         proxy_config = proxy_url = proxy_protocol = proxy_port = proxy_username = proxy_password = None
         # User requires proxy?
@@ -168,18 +153,16 @@ class QualysConnectConfig:
             self.proxies = None
 
         # ask username (if one doesn't exist)
-        if not self._cfgparse.has_option('qualys', 'username'):
+        if not self._cfgparse.has_option(self._section, 'username'):
             username = input('QualysGuard Username: ')
-            self._cfgparse.set('qualys', 'username', username)
+            self._cfgparse.set(self._section, 'username', username)
 
         # ask password (if one doesn't exist)
-        if not self._cfgparse.has_option('qualys', 'password'):
+        if not self._cfgparse.has_option(self._section, 'password'):
             password = getpass.getpass('QualysGuard Password: ')
-            self._cfgparse.set('qualys', 'password', password)
+            self._cfgparse.set(self._section, 'password', password)
 
-
+        logger.debug(self._cfgparse.items(self._section))
-
-        logging.debug(self._cfgparse.items('qualys'))
 
         if remember_me or remember_me_always:
             # Let's create that config file for next time...
@@ -211,11 +194,8 @@ class QualysConnectConfig:
 
     def get_auth(self):
         ''' Returns username from the configfile. '''
-        return (self._cfgparse.get('qualys', 'username'), self._cfgparse.get('qualys', 'password'))
+        return (self._cfgparse.get(self._section, 'username'), self._cfgparse.get(self._section, 'password'))
 
     def get_hostname(self):
         ''' Returns hostname. '''
-        return self._cfgparse.get('qualys', 'hostname')
+        return self._cfgparse.get(self._section, 'hostname')
-
-    def get_template_id(self):
-        return self._cfgparse.get('qualys','template_id')

deps/qualysapi/qualysapi/connector.py

@@ -159,7 +159,7 @@ class QGConnector(api_actions.QGActions):
             if api_call_endpoint in self.api_methods['was get']:
                 return 'get'
             # Post calls with no payload will result in HTTPError: 415 Client Error: Unsupported Media Type.
-            if data is None:
+            if not data:
                 # No post data. Some calls change to GET with no post data.
                 if api_call_endpoint in self.api_methods['was no data get']:
                     return 'get'
@@ -220,8 +220,7 @@ class QGConnector(api_actions.QGActions):
                 data = data.lstrip('?')
                 data = data.rstrip('&')
                 # Convert to dictionary.
-                #data = urllib.parse.parse_qs(data)
+                data = urlparse.parse_qs(data)
-                data = urlparse(data)
                 logger.debug('Converted:\n%s' % str(data))
         elif api_version in ('am', 'was', 'am2'):
             if type(data) == etree._Element:
@@ -258,7 +257,7 @@ class QGConnector(api_actions.QGActions):
         url = self.url_api_version(api_version)
         #
         # Set up headers.
-        headers = {"X-Requested-With": "QualysAPI (python) v%s - VulnWhisperer" % (qualysapi.version.__version__,)}
+        headers = {"X-Requested-With": "Parag Baxi QualysAPI (python) v%s" % (qualysapi.version.__version__,)}
         logger.debug('headers =\n%s' % (str(headers)))
         # Portal API takes in XML text, requiring custom header.
         if api_version in ('am', 'was', 'am2'):
@@ -318,7 +317,7 @@ class QGConnector(api_actions.QGActions):
                 logger.debug(e)
                 pass
             # Response received.
-            response = str(request.content)
+            response = request.text
             logger.debug('response text =\n%s' % (response))
             # Keep track of how many retries.
             retries += 1

deps/qualysapi/qualysapi/contrib.py

@@ -1,290 +0,0 @@
-# File for 3rd party contributions.
-
-from __future__ import absolute_import
-from __future__ import print_function
-import six
-from six.moves import range
-
-__author__ = 'Parag Baxi <parag.baxi@gmail.com>'
-__license__ = 'Apache License 2.0'
-
-import logging
-import time
-import types
-import unicodedata
-from collections import defaultdict
-
-from lxml import etree, objectify
-
-
-# Set module level logger.
-logger = logging.getLogger(__name__)
-
-
-def generate_vm_report(self, report_details, startup_delay=60, polling_delay=30, max_checks=10):
-    ''' Spool and download QualysGuard VM report.
-
-    startup_delay: Time in seconds to wait before initially checking.
-    polling_delay: Time in seconds to wait between checks.
-    max_checks: Maximum number of times to check for report spooling completion.
-
-    '''
-    # Merge parameters.
-    report_details['action'] = 'launch'
-    logger.debug(report_details)
-    xml_output = qualysapi_instance.request(2, 'report', report_details)
-    report_id = etree.XML(xml_output).find('.//VALUE').text
-    logger.debug('report_id: %s' % (report_id))
-    # Wait for report to finish spooling.
-    # Maximum number of times to check for report.  About 10 minutes.
-    MAX_CHECKS = 10
-    logger.info('Report sent to spooler. Checking for report in %s seconds.' % (startup_delay))
-    time.sleep(startup_delay)
-    for n in range(0, max_checks):
-        # Check to see if report is done.
-        xml_output = qualysapi_instance.request(2, 'report', {'action': 'list', 'id': report_id})
-        tag_status = etree.XML(xml_output).findtext(".//STATE")
-        logger.debug('tag_status: %s' % (tag_status))
-        tag_status = etree.XML(xml_output).findtext(".//STATE")
-        logger.debug('tag_status: %s' % (tag_status))
-        if tag_status is not None:
-            # Report is showing up in the Report Center.
-            if tag_status == 'Finished':
-                # Report creation complete.
-                break
-        # Report not finished, wait.
-        logger.info('Report still spooling. Trying again in %s seconds.' % (polling_delay))
-        time.sleep(polling_delay)
-    # We now have to fetch the report.  Use the report id.
-    report_xml = qualysapi_instance.request(2, 'report', {'action': 'fetch', 'id': report_id})
-    return report_xml
-
-
-def qg_html_to_ascii(qg_html_text):
-    """Convert and return QualysGuard's quasi HTML text to ASCII text."""
-    text = qg_html_text
-    # Handle tagged line breaks (<p>, <br>)
-    text = re.sub(r'(?i)<br>[ ]*', '\n', text)
-    text = re.sub(r'(?i)<p>[ ]*', '\n', text)
-    # Remove consecutive line breaks
-    text = re.sub(r"^\s+", "", text, flags=re.MULTILINE)
-    # Remove empty lines at the end.
-    text = re.sub('[\n]+$', '$', text)
-    # Store anchor tags href attribute
-    links = list(lxml.html.iterlinks(text))
-    # Remove anchor tags
-    html_element = lxml.html.fromstring(text)
-    # Convert anchor tags to "link_text (link: link_url )".
-    logging.debug('Converting anchor tags...')
-    text = html_element.text_content().encode('ascii', 'ignore')
-    # Convert each link.
-    for l in links:
-        # Find and replace each link.
-        link_text = l[0].text_content().encode('ascii', 'ignore').strip()
-        link_url = l[2].strip()
-        # Replacing link_text
-        if link_text != link_url:
-            # Link text is different, most likely a description.
-            text = string.replace(text, link_text, '%s (link: %s )' % (link_text, link_url))
-        else:
-            # Link text is the same as the href.  No need to duplicate link.
-            text = string.replace(text, link_text, '%s' % (link_url))
-    logging.debug('Done.')
-    return text
-
-
-def qg_parse_informational_qids(xml_report):
-    """Return vulnerabilities of severity 1 and 2 levels due to a restriction of
-       QualysGuard's inability to report them in the internal ticketing system.
-    """
-    # asset_group's vulnerability data map:
-    #    {'qid_number': {
-    #                    # CSV info
-    #                    'hosts': [{'ip': '10.28.0.1', 'dns': 'hostname', 'netbios': 'blah', 'vuln_id': 'remediation_ticket_number'}, {'ip': '10.28.0.3', 'dns': 'hostname2', 'netbios': '', 'vuln_id': 'remediation_ticket_number'}, ...],
-    #                    'solution': '',
-    #                    'impact': '',
-    #                    'threat': '',
-    #                    'severity': '',
-    #                   }
-    #     'qid_number2': ...
-    #     }
-    # Add all vulnerabilities to list of dictionaries.
-    # Use defaultdict in case a new QID is encountered.
-    info_vulns = defaultdict(dict)
-    # Parse vulnerabilities in xml string.
-    tree = objectify.fromstring(xml_report)
-    # Write IP, DNS, & Result into each QID CSV file.
-    logging.debug('Parsing report...')
-    # TODO:  Check against c_args.max to prevent creating CSV content for QIDs that we won't use.
-    for host in tree.HOST_LIST.HOST:
-        # Extract possible extra hostname information.
-        try:
-            netbios = unicodedata.normalize('NFKD', six.text_type(host.NETBIOS)).encode('ascii', 'ignore').strip()
-        except AttributeError:
-            netbios = ''
-        try:
-            dns = unicodedata.normalize('NFKD', six.text_type(host.DNS)).encode('ascii', 'ignore').strip()
-        except AttributeError:
-            dns = ''
-        ip = unicodedata.normalize('NFKD', six.text_type(host.IP)).encode('ascii', 'ignore').strip()
-        # Extract vulnerabilities host is affected by.
-        for vuln in host.VULN_INFO_LIST.VULN_INFO:
-            try:
-                result = unicodedata.normalize('NFKD', six.text_type(vuln.RESULT)).encode('ascii', 'ignore').strip()
-            except AttributeError:
-                result = ''
-            qid = unicodedata.normalize('NFKD', six.text_type(vuln.QID)).encode('ascii', 'ignore').strip()
-            # Attempt to add host to QID's list of affected hosts.
-            try:
-                info_vulns[qid]['hosts'].append({'ip': '%s' % (ip),
-                                                 'dns': '%s' % (dns),
-                                                 'netbios': '%s' % (netbios),
-                                                 'vuln_id': '',
-                                                 # Informational QIDs do not have vuln_id numbers.  This is a flag to write the CSV file.
-                                                 'result': '%s' % (result), })
-            except KeyError:
-                # New QID.
-                logging.debug('New QID found: %s' % (qid))
-                info_vulns[qid]['hosts'] = []
-                info_vulns[qid]['hosts'].append({'ip': '%s' % (ip),
-                                                 'dns': '%s' % (dns),
-                                                 'netbios': '%s' % (netbios),
-                                                 'vuln_id': '',
-                                                 # Informational QIDs do not have vuln_id numbers.  This is a flag to write the CSV file.
-                                                 'result': '%s' % (result), })
-    # All vulnerabilities added.
-    # Add all vulnerabilty information.
-    for vuln_details in tree.GLOSSARY.VULN_DETAILS_LIST.VULN_DETAILS:
-        qid = unicodedata.normalize('NFKD', six.text_type(vuln_details.QID)).encode('ascii', 'ignore').strip()
-        info_vulns[qid]['title'] = unicodedata.normalize('NFKD', six.text_type(vuln_details.TITLE)).encode('ascii',
-                                                                                                           'ignore').strip()
-        info_vulns[qid]['severity'] = unicodedata.normalize('NFKD', six.text_type(vuln_details.SEVERITY)).encode('ascii',
-                                                                                                                 'ignore').strip()
-        info_vulns[qid]['solution'] = qg_html_to_ascii(
-            unicodedata.normalize('NFKD', six.text_type(vuln_details.SOLUTION)).encode('ascii', 'ignore').strip())
-        info_vulns[qid]['threat'] = qg_html_to_ascii(
-            unicodedata.normalize('NFKD', six.text_type(vuln_details.THREAT)).encode('ascii', 'ignore').strip())
-        info_vulns[qid]['impact'] = qg_html_to_ascii(
-            unicodedata.normalize('NFKD', six.text_type(vuln_details.IMPACT)).encode('ascii', 'ignore').strip())
-    # Ready to report informational vulnerabilities.
-    return info_vulns
-
-
-# TODO: Implement required function qg_remediation_tickets(asset_group, status, qids)
-# TODO: Remove static 'report_template' value.  Parameterize and document required report template.
-def qg_ticket_list(asset_group, severity, qids=None):
-    """Return dictionary of each vulnerability reported against asset_group of severity."""
-    global asset_group_details
-    # All vulnerabilities imported to list of dictionaries.
-    vulns = qg_remediation_tickets(asset_group, 'OPEN', qids)  # vulns now holds all open remediation tickets.
-    if not vulns:
-        # No tickets to report.
-        return False
-    #
-    # Sort the vulnerabilities in order of prevalence -- number of hosts affected.
-    vulns = OrderedDict(sorted(list(vulns.items()), key=lambda t: len(t[1]['hosts'])))
-    logging.debug('vulns sorted = %s' % (vulns))
-    #
-    # Remove QIDs that have duplicate patches.
-    #
-    # Read in patch report.
-    # TODO:  Allow for lookup of report_template.
-    # Report template is Patch report "Sev 5 confirmed patchable".
-    logging.debug('Retrieving patch report from QualysGuard.')
-    print('Retrieving patch report from QualysGuard.')
-    report_template = '1063695'
-    # Call QualysGuard for patch report.
-    csv_output = qg_command(2, 'report', {'action': 'launch', 'output_format': 'csv',
-                                          'asset_group_ids': asset_group_details['qg_asset_group_id'],
-                                          'template_id': report_template,
-                                          'report_title': 'QGIR Patch %s' % (asset_group)})
-    logging.debug('csv_output =')
-    logging.debug(csv_output)
-    logging.debug('Improving remediation efficiency by removing unneeded, redundant patches.')
-    print('Improving remediation efficiency by removing unneeded, redundant patches.')
-    # Find the line for Patches by Host data.
-    logging.debug('Header found at %s.' % (csv_output.find('Patch QID, IP, DNS, NetBIOS, OS, Vulnerability Count')))
-
-    starting_pos = csv_output.find('Patch QID, IP, DNS, NetBIOS, OS, Vulnerability Count') + 52
-    logging.debug('starting_pos = %s' % str(starting_pos))
-    # Data resides between line ending in 'Vulnerability Count' and a blank line.
-    patches_by_host = csv_output[starting_pos:csv_output[starting_pos:].find(
-        'Host Vulnerabilities Fixed by Patch') + starting_pos - 3]
-    logging.debug('patches_by_host =')
-    logging.debug(patches_by_host)
-    # Read in string patches_by_host csv to a dictionary.
-    f = patches_by_host.split(os.linesep)
-    reader = csv.DictReader(f, ['Patch QID', 'IP', 'DNS', 'NetBIOS', 'OS', 'Vulnerability Count'], delimiter=',')
-    # Mark Patch QIDs that fix multiple vulnerabilities with associated IP addresses.
-    redundant_qids = defaultdict(list)
-    for row in reader:
-        if int(row['Vulnerability Count']) > 1:
-            # Add to list of redundant QIDs.
-            redundant_qids[row['Patch QID']].append(row['IP'])
-            logging.debug('%s, %s, %s, %s' % (
-                row['Patch QID'],
-                row['IP'],
-                int(row['Vulnerability Count']),
-                redundant_qids[row['Patch QID']]))
-    # Log for debugging.
-    logging.debug('len(redundant_qids) = %s, redundant_qids =' % (len(redundant_qids)))
-    for patch_qid in list(redundant_qids.keys()):
-        logging.debug('%s, %s' % (str(patch_qid), str(redundant_qids[patch_qid])))
-    # Extract redundant QIDs with associated IP addresses.
-    # Find the line for Patches by Host data.
-    starting_pos = csv_output.find('Patch QID, IP, QID, Severity, Type, Title, Instance, Last Detected') + 66
-    # Data resides between line ending in 'Vulnerability Count' and end of string.
-    host_vulnerabilities_fixed_by_patch = csv_output[starting_pos:]
-    # Read in string host_vulnerabilities_fixed_by_patch csv to a dictionary.
-    f = host_vulnerabilities_fixed_by_patch.split(os.linesep)
-    reader = csv.DictReader(f, ['Patch QID', 'IP', 'QID', 'Severity', 'Type', 'Title', 'Instance', 'Last Detected'],
-                            delimiter=',')
-    # Remove IP addresses associated with redundant QIDs.
-    qids_to_remove = defaultdict(list)
-    for row in reader:
-        # If the row's IP address's Patch QID was found to have multiple vulnerabilities...
-        if len(redundant_qids[row['Patch QID']]) > 0 and redundant_qids[row['Patch QID']].count(row['IP']) > 0:
-            # Add the QID column to the list of dictionaries {QID: [IP address, IP address, ...], QID2: [IP address], ...}
-            qids_to_remove[row['QID']].append(row['IP'])
-    # Log for debugging.
-    logging.debug('len(qids_to_remove) = %s, qids_to_remove =' % (len(qids_to_remove)))
-    for a_qid in list(qids_to_remove.keys()):
-        logging.debug('%s, %s' % (str(a_qid), str(qids_to_remove[a_qid])))
-    #
-    # Diff vulns against qids_to_remove and against open incidents.
-    #
-    vulns_length = len(vulns)
-    # Iterate over list of keys rather than original dictionary as some keys may be deleted changing the size of the dictionary.
-    for a_qid in list(vulns.keys()):
-        # Debug log original qid's hosts.
-        logging.debug('Before diffing vulns[%s] =' % (a_qid))
-        logging.debug(vulns[a_qid]['hosts'])
-        # Pop each host.
-        # The [:] returns a "slice" of x, which happens to contain all its elements, and is thus effectively a copy of x.
-        for host in vulns[a_qid]['hosts'][:]:
-            # If the QID for the host is a dupe or if a there is an open Reaction incident.
-            if qids_to_remove[a_qid].count(host['ip']) > 0 or reaction_open_issue(host['vuln_id']):
-                # Remove the host from the QID's list of target hosts.
-                logging.debug('Removing remediation ticket %s.' % (host['vuln_id']))
-                vulns[a_qid]['hosts'].remove(host)
-            else:
-                # Do not remove this vuln
-                logging.debug('Will report remediation %s.' % (host['vuln_id']))
-        # Debug log diff'd qid's hosts.
-        logging.debug('After diffing vulns[%s]=' % (a_qid))
-        logging.debug(vulns[a_qid]['hosts'])
-        # If there are no more hosts left to patch for the qid.
-        if len(vulns[a_qid]['hosts']) == 0:
-            # Remove the QID.
-            logging.debug('Deleting vulns[%s].' % (a_qid))
-            del vulns[a_qid]
-    # Diff completed
-    if not vulns_length == len(vulns):
-        print('A count of %s vulnerabilities have been consolidated to %s vulnerabilities, a reduction of %s%%.' % (
-            int(vulns_length),
-            int(len(vulns)),
-            int(round((int(vulns_length) - int(len(vulns))) / float(vulns_length) * 100))))
-    # Return vulns to report.
-    logging.debug('vulns =')
-    logging.debug(vulns)
-    return vulns

deps/qualysapi/qualysapi/settings.py

@@ -17,5 +17,4 @@ else:
     default_filename = ".qcrc"
 
 defaults = {'hostname': 'qualysapi.qualys.com',
-            'max_retries': '3',
+            'max_retries': '3'}
-            'template_id': '00000'}

deps/qualysapi/qualysapi/util.py

@@ -14,12 +14,12 @@ __license__ = 'Apache License 2.0'
 logger = logging.getLogger(__name__)
 
 
-def connect(config_file=qcs.default_filename, remember_me=False, remember_me_always=False):
+def connect(config_file=qcs.default_filename, section='info', remember_me=False, remember_me_always=False):
     """ Return a QGAPIConnect object for v1 API pulling settings from config
     file.
     """
     # Retrieve login credentials.
-    conf = qcconf.QualysConnectConfig(filename=config_file, remember_me=remember_me, 
+    conf = qcconf.QualysConnectConfig(filename=config_file, section=section, remember_me=remember_me,
                                       remember_me_always=remember_me_always)
     connect = qcconn.QGConnector(conf.get_auth(),
                                  conf.get_hostname(),

deps/qualysapi/qualysapi/version.py

@@ -1,3 +1,3 @@
-__author__ = 'Austin Taylor'
+__author__ = 'Parag Baxi <parag.baxi@gmail.com>'
 __pkgname__ = 'qualysapi'
-__version__ = '4.1.0'
+__version__ = '5.0.3'

deps/qualysapi/setup.cfg

@@ -0,0 +1,12 @@
+[metadata]
+# This includes the license file in the wheel.
+license_file = license
+
+[bdist_wheel]
+# This flag says to generate wheels that support both Python 2 and Python
+# 3. If your code will not run unchanged on both Python 2 and 3, you will
+# need to generate separate wheels for each Python version that you
+# support. Removing this line (or setting universal to 0) will prevent
+# bdist_wheel from trying to make a universal wheel. For more see:
+# https://packaging.python.org/tutorials/distributing-packages/#wheels
+universal=1

deps/qualysapi/setup.py

@@ -1,15 +1,15 @@
 #!/usr/bin/env python
+
 from __future__ import absolute_import
 import os
-import setuptools
+import sys
-
 try:
     from setuptools import setup
 except ImportError:
     from distutils.core import setup
 
-__author__ = 'Austin Taylor <vulnWhisperer@austintaylor.io>'
+__author__ = 'Parag Baxi <parag.baxi@gmail.com>'
-__copyright__ = 'Copyright 2017, Austin Taylor'
+__copyright__ = 'Copyright 2011-2018, Parag Baxi'
 __license__ = 'BSD-new'
 # Make pyflakes happy.
 __pkgname__ = None
@@ -27,15 +27,14 @@ def read(fname):
 
 setup(name=__pkgname__,
       version=__version__,
-      author='Austin Taylor',
+      author='Parag Baxi',
-      author_email='vulnWhisperer@austintaylor.io',
+      author_email='parag.baxi@gmail.com',
-      description='QualysGuard(R) Qualys API Package modified for VulnWhisperer',
+      description='QualysGuard(R) Qualys API Package',
       license='BSD-new',
       keywords='Qualys QualysGuard API helper network security',
-      url='https://github.com/austin-taylor/qualysapi',
+      url='https://github.com/paragbaxi/qualysapi',
       package_dir={'': '.'},
-      #packages=setuptools.find_packages(),
+      packages=['qualysapi', ],
-      packages=['qualysapi',],
       # package_data={'qualysapi':['LICENSE']},
       # scripts=['src/scripts/qhostinfo.py', 'src/scripts/qscanhist.py', 'src/scripts/qreports.py'],
       long_description=read('README.md'),
@@ -44,6 +43,10 @@ setup(name=__pkgname__,
           'Topic :: Utilities',
           'License :: OSI Approved :: Apache Software License',
           'Intended Audience :: Developers',
+          'Programming Language :: Python :: 2.7',
+          'Programming Language :: Python :: 3.4',
+          'Programming Language :: Python :: 3.5',
+          'Programming Language :: Python :: 3.6',
       ],
       install_requires=[
           'requests',

vulnwhisp/frameworks/qualys_vuln.py

@@ -17,7 +17,7 @@ class qualysWhisperAPI(object):
     def __init__(self, config=None):
         self.config = config
         try:
-            self.qgc = qualysapi.connect(config)
+            self.qgc = qualysapi.connect(config, 'qualys_vuln')
             # Fail early if we can't make a request or auth is incorrect
             self.qgc.request('about.php')
             print('[SUCCESS] - Connected to Qualys at %s' % self.qgc.server)

vulnwhisp/frameworks/qualys_web.py

@@ -35,7 +35,7 @@ class qualysWhisperAPI(object):
     def __init__(self, config=None):
         self.config = config
         try:
-            self.qgc = qualysapi.connect(config)
+            self.qgc = qualysapi.connect(config, 'qualys_web')
             print('[SUCCESS] - Connected to Qualys at %s' % self.qgc.server)
         except Exception as e:
             print('[ERROR] Could not connect to Qualys - %s' % e)

vulnwhisp/vulnwhisp.py

@@ -4,7 +4,7 @@ __author__ = 'Austin Taylor'
 
 from base.config import vwConfig
 from frameworks.nessus import NessusAPI
-from frameworks.qualys import qualysScanReport
+from frameworks.qualys_web import qualysScanReport
 from frameworks.qualys_vuln import qualysVulnScan
 from frameworks.openvas import OpenVAS_API
 from reporting.jira_api import JiraAPI
@@ -469,7 +469,7 @@ class vulnWhispererNessus(vulnWhispererBase):
 
 class vulnWhispererQualys(vulnWhispererBase):
 
-    CONFIG_SECTION = 'qualys'
+    CONFIG_SECTION = 'qualys_web'
     COLUMN_MAPPING = {'Access Path': 'access_path',
                      'Ajax Request': 'ajax_request',
                      'Ajax Request ID': 'ajax_request_id',
@@ -1176,7 +1176,7 @@ class vulnWhisperer(object):
                                      profile=self.profile)
             vw.whisper_nessus()
 
-        elif self.profile == 'qualys':
+        elif self.profile == 'qualys_web':
             vw = vulnWhispererQualys(config=self.config)
             vw.process_web_assets()