Merge pull request 'Traduction du modèle de rapport de bug' (#1) from mes-modifs into master

Reviewed-on: #1

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,41 @@
+---
+name: Rapport de bug
+about: Créez un rapport pour nous aider à nous améliorer
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+**Décrivez le bug**
+Une description claire et concise de ce qu'est le bug.
+
+**Module affecté**
+Lequel des modules ne fonctionne pas comme prévu, par exemple, Nessus, Qualys WAS, Qualys VM, OpenVAS, ELK, Jira...
+
+**Trace de débogage de VulnWhisperer**
+Si possible, veuillez joindre la trace de débogage de l'exécution pour une enquête plus approfondie (exécuter avec l'option `-d`).
+
+**Pour reproduire**
+Étapes pour reproduire le comportement :
+1. Allez à '...'
+2. Cliquez sur '....'
+3. Voir l'erreur
+
+**Comportement attendu**
+Une description claire et concise de ce à quoi vous vous attendiez.
+
+**Captures d'écran**
+Si applicable, ajoutez des captures d'écran pour aider à expliquer votre problème.
+
+**Système sur lequel VulnWhisperer s'exécute (veuillez compléter les informations suivantes) :**
+ - OS : [ex. Ubuntu Server]
+ - Version : [ex. 18.04.2 LTS]
+ - Version de VulnWhisperer : [ex. 1.7.1]
+
+**Contexte additionnel**
+Ajoutez tout autre contexte sur le problème ici.
+
+**Note importante**
+Comme VulnWhisperer s'appuie sur ELK pour l'agrégation de données, il est attendu que vous ayez déjà une instance ELK ou les connaissances pour en déployer une.
+Pour accélérer le déploiement, nous fournissons un fichier docker-compose à jour et testé qui déploie toute l'infrastructure nécessaire et nous supporterons son déploiement, mais nous ne donnerons pas de support pour les instances ELK.

.gitignore

@@ -1,3 +1,11 @@
+# Vulnwhisperer stuff
+data/
+logs/
+elk6/vulnwhisperer.ini
+resources/elk6/vulnwhisperer.ini
+configs/frameworks_example.ini
+tests/data
+
 # Byte-compiled / optimized / DLL files
 __pycache__/
 *.py[cod]
@@ -100,3 +108,6 @@ ENV/
 
 # mypy
 .mypy_cache/
+
+# Mac
+.DS_Store

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "tests/data"]
+	path = tests/data
+	url = https://github.com/HASecuritySolutions/VulnWhisperer-tests.git

.travis.yml

@@ -0,0 +1,36 @@
+group: travis_latest
+language: python
+cache: pip
+python:
+    - 2.7
+env:
+    - TEST_PATH=tests/data
+
+services:
+  - docker
+#    - 3.6
+#matrix:
+#    allow_failures:
+#        - python: 3.6 - Commenting out testing for Python 3.6 until ready
+
+before_install:
+    - mkdir -p ./data/esdata1
+    - mkdir -p ./data/es_snapshots
+    - sudo chown -R 1000:1000 ./data/es*
+    - docker build -t vulnwhisperer-local .
+    - docker-compose -f docker-compose-test.yml up -d
+install:
+    - pip install -r requirements.txt
+    - pip install flake8  # pytest  # add another testing frameworks later
+before_script:
+    # stop the build if there are Python syntax errors or undefined names
+    - flake8 . --count --exclude=deps/qualysapi --select=E901,E999,F821,F822,F823 --show-source --statistics
+    # exit-zero treats all errors as warnings.  The GitHub editor is 127 chars wide
+    - flake8 . --count --exit-zero --exclude=deps/qualysapi --max-complexity=10 --max-line-length=127 --statistics
+script:
+    - python setup.py install
+    - bash tests/test-vuln_whisperer.sh
+    - bash tests/test-docker.sh
+notifications:
+    on_success: change
+    on_failure: change  # `always` will be the setting once code changes slow down

CNAME

@@ -0,0 +1 @@
+www.vulnwhisperer.com

Dockerfile

@@ -0,0 +1,26 @@
+FROM centos:7
+
+MAINTAINER Justin Henderson justin@hasecuritysolutions.com
+
+RUN yum update -y && \
+    yum install -y python python-devel git gcc && \
+    curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+    python get-pip.py
+
+WORKDIR /opt/VulnWhisperer
+
+COPY requirements.txt requirements.txt
+COPY setup.py setup.py
+COPY vulnwhisp/ vulnwhisp/
+COPY bin/ bin/
+COPY configs/frameworks_example.ini frameworks_example.ini
+
+RUN python setup.py clean --all && \
+    pip install -r requirements.txt
+
+
+WORKDIR /opt/VulnWhisperer
+RUN python setup.py install
+
+
+CMD  vuln_whisperer -c /opt/VulnWhisperer/frameworks_example.ini

LICENSE

@@ -1,21 +1,201 @@
-MIT License
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
 
-Copyright (c) 2017 Austin Taylor
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
+   1. Definitions.
 
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,2 +1,138 @@
-# VulnWhisperer
-Create actionable data from your Vulnerability Scans
+<p align="center"><img src="https://git.gudita.com/Cyberdefense/VulnWhisperer/raw/branch/master/docs/source/vuln_whisperer_logo_s.png" width="400px"></p>
+<p align="center"> <i>Créez des <u><b>données exploitables</b></u> à partir de vos scans de vulnérabilités</i> </p> 
+
+<p align="center" style="width:400px"><img src="https://git.gudita.com/Cyberdefense/VulnWhisperer/raw/branch/master/docs/source/vulnWhispererWebApplications.png" style="width:400px"></p>
+
+
+VulnWhisperer est un outil de gestion des vulnérabilités et un agrégateur de rapports. VulnWhisperer récupère tous les rapports des différents scanners de vulnérabilités et crée un fichier avec un nom unique pour chacun, utilisant ensuite ces données pour se synchroniser avec Jira et alimenter Logstash. Jira effectue une synchronisation complète en cycle fermé avec les données fournies par les scanners, tandis que Logstash indexe et étiquette toutes les informations contenues dans le rapport (voir les fichiers logstash dans `/resources/elk6/pipeline/`). Les données sont ensuite envoyées à ElasticSearch pour être indexées, et finissent dans un format visuel et consultable dans Kibana avec des tableaux de bord déjà définis.
+
+VulnWhisperer est un projet open-source financé par la communauté. VulnWhisperer est actuellement fonctionnel mais nécessite une refonte de la documentation et une revue de code. Si vous souhaitez de l'aide, si vous êtes intéressé par de nouvelles fonctionnalités, ou si vous recherchez un support payant, veuillez nous contacter à **info@sahelcyber.com**.
+
+
+### Scanners de Vulnérabilités Supportés
+
+- [X] [Nessus (**v6**/**v7**/**v8**)](https://www.tenable.com/products/nessus/nessus-professional)
+- [X] [Qualys Web Applications](https://www.qualys.com/apps/web-app-scanning/)
+- [X] [Qualys Vulnerability Management](https://www.qualys.com/apps/vulnerability-management/)
+- [X] [OpenVAS (**v7**/**v8**/**v9**)](http://www.openvas.org/)
+- [X] [Tenable.io](https://www.tenable.com/products/tenable-io)
+- [ ] [Detectify](https://detectify.com/)
+- [ ] [Nexpose](https://www.rapid7.com/products/nexpose/)
+- [ ] [Insight VM](https://www.rapid7.com/products/insightvm/)
+- [ ] [NMAP](https://nmap.org/)
+- [ ] [Burp Suite](https://portswigger.net/burp)
+- [ ] [OWASP ZAP](https://www.zaproxy.org/)
+- [ ] Et d'autres à venir
+
+### Plateformes de Reporting Supportées
+
+- [X] [Elastic Stack (**v6**/**v7**)](https://www.elastic.co/elk-stack)
+- [ ] [OpenSearch - Envisagé pour la prochaine mise à jour](https://opensearch.org/)
+- [X] [Jira](https://www.atlassian.com/software/jira)
+- [ ] [Splunk](https://www.splunk.com/)
+
+## Démarrage
+
+1) Suivez les [prérequis d'installation](#installreq)
+2) Remplissez la section que vous souhaitez traiter dans le fichier <a href="https://git.gudita.com/Cyberdefense/VulnWhisperer/src/branch/master/configs/frameworks_example.ini">frameworks_example.ini</a>
+3) [JIRA] Si vous utilisez Jira, remplissez la configuration Jira dans le fichier de configuration mentionné ci-dessus.
+3) [ELK] Modifiez les paramètres IP dans les <a href="https://git.gudita.com/Cyberdefense/VulnWhisperer/src/branch/master/resources/elk6/pipeline">fichiers Logstash pour correspondre à votre environnement</a> et importez-les dans votre répertoire de configuration logstash (par défaut `/etc/logstash/conf.d/`)
+4) [ELK] Importez les <a href="https://git.gudita.com/Cyberdefense/VulnWhisperer/src/branch/master/resources/elk6/kibana.json">visualisations Kibana</a>
+5) [Exécutez Vulnwhisperer](#run)
+
+> **Note importante concernant les liens du Wiki :** La migration de Gitea ne transfère pas toujours le Wiki d'un projet GitHub (qui est techniquement un dépôt séparé). Si les liens vers le Wiki (comme le guide de déploiement ELK) ne fonctionnent pas, vous devrez peut-être recréer ces pages manuellement dans l'onglet "Wiki" de votre dépôt sur Gitea.
+
+Besoin d'aide ou juste envie de discuter ? Rejoignez notre [canal Slack](https://join.slack.com/t/vulnwhisperer/shared_invite/enQtNDQ5MzE4OTIyODU0LWQxZTcxYTY0MWUwYzA4MTlmMWZlYWY2Y2ZmM2EzNDFmNWVlOTM4MzNjYzI0YzdkMDA0YmQyYWRhZGI2NGUxNGI)
+
+## Prérequis
+* Python 2.7
+* Un Scanner de Vulnérabilités
+* Un Système de Reporting : Jira / ElasticStack 6.6
+
+<a id="installreq"></a>
+## Prérequis d'Installation - VulnWhisperer (peut nécessiter sudo)
+**Installez les dépendances des paquets du système d'exploitation** (pour les distributions basées sur Debian, CentOS n'en a pas besoin)
+```shell
+sudo apt-get install zlib1g-dev libxml2-dev libxslt1-dev
+
+(Optionnel) Utilisez un environnement virtuel python pour ne pas perturber les bibliothèques python de l'hôte
+
+virtualenv venv # créera l'environnement virtuel python 2.7
+
+source venv/bin/activate # démarre l'environnement, pip s'exécutera ici et devrait installer les bibliothèques sans sudo
+
+deactivate # pour quitter l'environnement virtuel une fois que vous avez terminé
+
+Installez les dépendances des bibliothèques python
+
+pip install -r /chemin/vers/VulnWhisperer/requirements.txt
+cd /chemin/vers/VulnWhisperer
+python setup.py install
+
+(Optionnel) Si vous utilisez un proxy, ajoutez l'URL du proxy comme variable d'environnement au PATH
+
+export HTTP_PROXY=[http://exemple.com:8080](http://exemple.com:8080)
+export HTTPS_PROXY=[http://exemple.com:8080](http://exemple.com:8080)
+
+Vous êtes maintenant prêt à télécharger les scans.
+
+Configuration
+Il y a quelques étapes de configuration pour mettre en place VulnWhisperer :
+
+Configurer le fichier Ini
+
+Configurer le fichier Logstash
+
+Importer les modèles ElasticSearch
+
+Importer les tableaux de bord Kibana
+
+Exécution
+Pour exécuter, remplissez le fichier de configuration avec les paramètres de votre scanner de vulnérabilités. Ensuite, vous pouvez l'exécuter depuis la ligne de commande.
+
+# (optionnel : -F -> fournit une coloration "Fantaisie" des logs, utile pour la compréhension lors de l'exécution manuelle de VulnWhisperer)
+vuln_whisperer -c configs/frameworks_example.ini -s nessus 
+# ou
+vuln_whisperer -c configs/frameworks_example.ini -s qualys
+
+Si aucune section n'est spécifiée (ex. -s nessus), vulnwhisperer vérifiera dans le fichier de configuration les modules ayant la propriété enabled=true et les exécutera séquentiellement.
+
+Docker-compose
+ELK est un monde en soi, et pour les nouveaux venus sur la plateforme, cela nécessite des compétences de base sous Linux et généralement un peu de dépannage jusqu'à ce qu'il soit déployé et fonctionne comme prévu. Comme nous ne sommes pas en mesure de fournir un support pour les problèmes ELK de chaque utilisateur, nous avons mis en place un docker-compose qui inclut :
+
+VulnWhisperer
+
+Logstash 6.6
+
+ElasticSearch 6.6
+
+Kibana 6.6
+
+Le docker-compose nécessite simplement de spécifier les chemins où les données de VulnWhisperer seront sauvegardées, et où se trouvent les fichiers de configuration. S'il est exécuté directement après un git clone, en ajoutant simplement la configuration du scanner au fichier de configuration de VulnWhisperer (/resources/elk6/vulnwhisperer.ini), il fonctionnera immédiatement.
+
+Il se charge également de charger automatiquement les tableaux de bord et les visualisations Kibana via l'API, ce qui doit être fait manuellement autrement au démarrage de Kibana.
+
+Pour plus d'informations sur le docker-compose, consultez le wiki docker-compose ou la FAQ.
+
+Feuille de route
+Notre feuille de route actuelle est la suivante :
+
+[ ] Créer un standard de vulnérabilité
+
+[ ] Mapper les résultats de chaque scanner au standard
+
+[ ] Créer des directives de module de scanner pour une intégration facile de nouveaux scanners
+
+[ ] Refactoriser le code pour réutiliser les fonctions et permettre une compatibilité totale entre les modules
+
+[ ] Changer Nessus CSV en JSON
+
+[ ] Adapter le Logstash unique au standard et aux tableaux de bord Kibana
+
+[ ] Implémenter le scanner Detectify
+
+[ ] Implémenter le reporting/tableaux de bord Splunk
+
+En plus de cela, nous essayons de nous concentrer sur la correction des bugs dès que possible, ce qui peut retarder le développement. Nous accueillons également très volontiers les PR (Pull Requests), et une fois que nous aurons implémenté le nouveau standard, il sera très facile d'ajouter la compatibilité avec de nouveaux scanners.
+
+Le standard de vulnérabilité sera initialement un nouveau JSON simple à un niveau avec toutes les informations correspondantes des différents scanners ayant des noms de variables standardisés, tout en conservant le reste des variables telles quelles.

_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-leap-day

bin/vuln_whisperer

@@ -1,39 +1,128 @@
-#!/usr/bin/env python
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+__author__ = 'Austin Taylor'
 
 
-#Written by Austin Taylor
-#www.austintaylor.io
-
 from vulnwhisp.vulnwhisp import vulnWhisperer
-from vulnwhisp.utils.cli import bcolors
-
+from vulnwhisp.base.config import vwConfig
+from vulnwhisp.test.mock import mockAPI
+import os
 import argparse
 import sys
+import logging
+
+
+def isFileValid(parser, arg):
+    if not os.path.exists(arg):
+        parser.error("The file %s does not exist!" % arg)
+    else:
+        return arg
+
 
 def main():
 
     parser = argparse.ArgumentParser(description=""" VulnWhisperer is designed to create actionable data from\
      your vulnerability scans through aggregation of historical scans.""")
     parser.add_argument('-c', '--config', dest='config', required=False, default='frameworks.ini',
-                        help='Path of config file')
+                        help='Path of config file', type=lambda x: isFileValid(parser, x.strip()))
+    parser.add_argument('-s', '--section', dest='section', required=False,
+                        help='Section in config')
+    parser.add_argument('--source', dest='source', required=False,
+                        help='JIRA required only! Source scanner to report')
+    parser.add_argument('-n', '--scanname', dest='scanname', required=False,
+                        help='JIRA required only! Scan name from scan to report')
     parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', default=True,
                         help='Prints status out to screen (defaults to True)')
+    parser.add_argument('-u', '--username', dest='username', required=False, default=None,
+                        help='The NESSUS username', type=lambda x: x.strip())
+    parser.add_argument('-p', '--password', dest='password', required=False, default=None,
+                        help='The NESSUS password', type=lambda x: x.strip())
+    parser.add_argument('-F', '--fancy', action='store_true',
+                        help='Enable colourful logging output')
+    parser.add_argument('-d', '--debug', action='store_true',
+                        help='Enable debugging messages')
+    parser.add_argument('--mock', action='store_true',
+                        help='Enable mocked API responses')
+    parser.add_argument('--mock_dir', dest='mock_dir', required=False, default=None,
+                        help='Path of test directory')
     args = parser.parse_args()
 
+    # First setup logging
+    logging.basicConfig(
+        stream=sys.stdout,
+        #format only applies when not using -F flag for colouring
+        format='%(levelname)s:%(name)s:%(funcName)s:%(message)s',
+        level=logging.DEBUG if args.debug else logging.INFO
+    )
+    logger = logging.getLogger()
+    # we set up the logger to log as well to file
+    fh = logging.FileHandler('vulnwhisperer.log')
+    fh.setLevel(logging.DEBUG if args.debug else logging.INFO)
+    fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s - %(funcName)s:%(message)s", "%Y-%m-%d %H:%M:%S"))
+    logger.addHandler(fh)
+
+    if args.fancy:
+        import coloredlogs
+        coloredlogs.install(level='DEBUG' if args.debug else 'INFO')
+
+    if args.mock:
+        mock_api = mockAPI(args.mock_dir, args.verbose)
+        mock_api.mock_endpoints()
+
+    exit_code = 0
+
     try:
+        if args.config and not args.section:
+            # this remains a print since we are in the main binary
+            print('WARNING: {warning}'.format(warning='No section was specified, vulnwhisperer will scrape enabled modules from config file. \
+                            \nPlease specify a section using -s. \
+                                                    \nExample vuln_whisperer -c config.ini -s nessus'))
+            logger.info('No section was specified, vulnwhisperer will scrape enabled modules from the config file.')
+            
+            config = vwConfig(config_in=args.config)
+            enabled_sections = config.get_sections_with_attribute('enabled')
 
-        vw = vulnWhisperer(config=args.config,
-                           verbose=args.verbose)
+            for section in enabled_sections:
+                try:
+                    vw = vulnWhisperer(config=args.config,
+                                       profile=section,
+                                       verbose=args.verbose,
+                                       username=args.username,
+                                       password=args.password,
+                                       source=args.source,
+                                       scanname=args.scanname)
+                    exit_code += vw.whisper_vulnerabilities()
+                except Exception as e:
+                    logger.error("VulnWhisperer was unable to perform the processing on '{}'".format(args.source))
+        else:
+            logger.info('Running vulnwhisperer for section {}'.format(args.section))
+            vw = vulnWhisperer(config=args.config,
+                               profile=args.section,
+                               verbose=args.verbose,
+                               username=args.username,
+                               password=args.password,
+                               source=args.source,
+                               scanname=args.scanname)
+            exit_code += vw.whisper_vulnerabilities()
 
-        vw.whisper_nessus()
-        sys.exit(1)
+        close_logging_handlers(logger)
+        sys.exit(exit_code)
 
     except Exception as e:
         if args.verbose:
-            print('{red} ERROR: {error}{endc}'.format(red=bcolors.FAIL, error=e, endc=bcolors.ENDC))
+            # this will remain a print since we are in the main binary
+            logger.error('{}'.format(str(e)))
+            print('ERROR: {error}'.format(error=e))
+        # TODO: fix this to NOT be exit 2 unless in error
+        close_logging_handlers(logger)
         sys.exit(2)
 
+    close_logging_handlers(logger)
 
+def close_logging_handlers(logger):
+    for handler in logger.handlers:
+        handler.close()
+        logger.removeFilter(handler)
 
 if __name__ == '__main__':
-    main()
+    main()

configs/frameworks_example.ini

@@ -2,10 +2,93 @@
 enabled=true
 hostname=localhost
 port=8834
+access_key=
+secret_key=
 username=nessus_username
 password=nessus_password
-write_path=/opt/vulnwhisp/scans
-db_path=/opt/vulnwhisp/database
+write_path=/opt/VulnWhisperer/data/nessus/
+db_path=/opt/VulnWhisperer/data/database
 trash=false
 verbose=true
 
+[tenable]
+enabled=true
+hostname=cloud.tenable.com
+port=443
+access_key=
+secret_key=
+username=tenable.io_username
+password=tenable.io_password
+write_path=/opt/VulnWhisperer/data/tenable/
+db_path=/opt/VulnWhisperer/data/database
+trash=false
+verbose=true
+
+[qualys_web]
+#Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
+enabled = true
+hostname = qualysapi.qg2.apps.qualys.com
+username = exampleuser
+password = examplepass
+write_path=/opt/VulnWhisperer/data/qualys_web/
+db_path=/opt/VulnWhisperer/data/database
+verbose=true
+
+# Set the maximum number of retries each connection should attempt.
+#Note, this applies only to failed connections and timeouts, never to requests where the server returns a response.
+max_retries = 10
+# Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID.
+template_id = 126024
+
+[qualys_vuln]
+#Reference https://www.qualys.com/docs/qualys-api-vmpc-user-guide.pdf to find your API
+enabled = true
+hostname = qualysapi.qg2.apps.qualys.com
+username = exampleuser
+password = examplepass
+write_path=/opt/VulnWhisperer/data/qualys_vuln/
+db_path=/opt/VulnWhisperer/data/database
+verbose=true
+
+[detectify]
+#Reference https://developer.detectify.com/
+enabled = false
+hostname = api.detectify.com
+#username variable used as apiKey
+username = exampleuser
+#password variable used as secretKey
+password = examplepass
+write_path =/opt/VulnWhisperer/data/detectify/
+db_path = /opt/VulnWhisperer/data/database
+verbose = true
+
+[openvas]
+enabled = false
+hostname = localhost
+port = 4000
+username = exampleuser
+password = examplepass
+write_path=/opt/VulnWhisperer/data/openvas/
+db_path=/opt/VulnWhisperer/data/database
+verbose=true
+
+[jira]
+enabled = false
+hostname = jira-host
+username = username
+password = password
+write_path = /opt/VulnWhisperer/data/jira/
+db_path = /opt/VulnWhisperer/data/database
+verbose = true
+dns_resolv = False
+
+#Sample jira report scan, will automatically be created for existent scans
+#[jira.qualys_vuln.test_scan]
+#source = qualys_vuln
+#scan_name = Test Scan
+#jira_project = PROJECT
+; if multiple components, separate by "," = None
+#components = 
+; minimum criticality to report (low, medium, high or critical) = None
+#min_critical_to_report = high
+

configs/test.ini

@@ -0,0 +1,94 @@
+[nessus]
+enabled=true
+hostname=nessus
+port=443
+access_key=
+secret_key=
+username=nessus_username
+password=nessus_password
+write_path=/opt/VulnWhisperer/data/nessus/
+db_path=/opt/VulnWhisperer/data/database
+trash=false
+verbose=true
+
+[tenable]
+enabled=true
+hostname=tenable
+port=443
+access_key=
+secret_key=
+username=tenable.io_username
+password=tenable.io_password
+write_path=/opt/VulnWhisperer/data/tenable/
+db_path=/opt/VulnWhisperer/data/database
+trash=false
+verbose=true
+
+[qualys_web]
+#Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
+enabled = false
+hostname = qualys_web
+username = exampleuser
+password = examplepass
+write_path=/opt/VulnWhisperer/data/qualys_web/
+db_path=/opt/VulnWhisperer/data/database
+verbose=true
+
+# Set the maximum number of retries each connection should attempt.
+#Note, this applies only to failed connections and timeouts, never to requests where the server returns a response.
+max_retries = 10
+# Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID.
+template_id = 126024
+
+[qualys_vuln]
+#Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
+enabled = true
+hostname = qualys_vuln
+username = exampleuser
+password = examplepass
+write_path=/opt/VulnWhisperer/data/qualys_vuln/
+db_path=/opt/VulnWhisperer/data/database
+verbose=true
+
+[detectify]
+#Reference https://developer.detectify.com/
+enabled = false
+hostname = detectify
+#username variable used as apiKey
+username = exampleuser
+#password variable used as secretKey
+password = examplepass
+write_path =/opt/VulnWhisperer/data/detectify/
+db_path = /opt/VulnWhisperer/data/database
+verbose = true
+
+[openvas]
+enabled = false
+hostname = openvas
+port = 4000
+username = exampleuser
+password = examplepass
+write_path=/opt/VulnWhisperer/data/openvas/
+db_path=/opt/VulnWhisperer/data/database
+verbose=true
+
+[jira]
+enabled = false
+hostname = jira-host
+username = username
+password = password
+write_path = /opt/VulnWhisperer/data/jira/
+db_path = /opt/VulnWhisperer/data/database
+verbose = true
+dns_resolv = False
+
+#Sample jira report scan, will automatically be created for existent scans
+#[jira.qualys_vuln.test_scan]
+#source = qualys_vuln
+#scan_name = Test Scan
+#jira_project = PROJECT
+; if multiple components, separate by "," = None
+#components = 
+; minimum criticality to report (low, medium, high or critical) = None
+#min_critical_to_report = high
+

docker-compose-test.yml

@@ -0,0 +1,97 @@
+version: '2'
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.6.0
+    container_name: elasticsearch
+    environment:
+      - cluster.name=vulnwhisperer
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
+      - xpack.security.enabled=false
+      - cluster.routing.allocation.disk.threshold_enabled=false
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    mem_limit: 8g
+    volumes:
+      - ./data/esdata1:/usr/share/elasticsearch/data
+      - ./data/es_snapshots:/snapshots
+    ports:
+      - 9200:9200
+    #restart: always
+    networks:
+      esnet:
+        aliases:
+          - elasticsearch.local
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:6.6.0
+    container_name: kibana
+    environment:
+      SERVER_NAME: kibana
+      ELASTICSEARCH_URL: http://elasticsearch:9200
+    ports:
+      - 5601:5601
+    depends_on:
+      - elasticsearch
+    networks:
+      esnet:
+        aliases:
+          - kibana.local
+
+  kibana-config:
+    image: alpine
+    container_name: kibana-config
+    volumes:
+      - ./resources/elk6/init_kibana.sh:/opt/init_kibana.sh
+      - ./resources/elk6/kibana_APIonly.json:/opt/kibana_APIonly.json
+      - ./resources/elk6/logstash-vulnwhisperer-template.json:/opt/index-template.json
+    command: sh -c "apk add --no-cache curl bash && chmod +x /opt/init_kibana.sh && chmod +r /opt/kibana_APIonly.json && cd /opt/ && /bin/bash /opt/init_kibana.sh" # /opt/kibana_APIonly.json"
+    networks:
+      esnet:
+        aliases:
+          - kibana-config.local
+
+  logstash:
+    image: docker.elastic.co/logstash/logstash:6.6.0
+    container_name: logstash
+    volumes:
+      - ./resources/elk6/pipeline/:/usr/share/logstash/pipeline
+      - ./data/vulnwhisperer/:/opt/VulnWhisperer/data
+      # - ./resources/elk6/logstash.yml:/usr/share/logstash/config/logstash.yml
+    environment:
+      - xpack.monitoring.enabled=false
+    depends_on:
+      - elasticsearch
+    ports:
+      - 9600:9600
+    networks:
+      esnet:
+        aliases:
+          - logstash.local
+  vulnwhisperer:
+    # image: hasecuritysolutions/vulnwhisperer:latest
+    image: vulnwhisperer-local
+    container_name: vulnwhisperer
+    entrypoint: [
+                  "vuln_whisperer",
+                  "-F",
+                  "-c",
+                  "/opt/VulnWhisperer/vulnwhisperer.ini",
+                  "--mock",
+                  "--mock_dir",
+                  "/tests/data"
+    ]    
+    volumes:
+      - ./data/vulnwhisperer/:/opt/VulnWhisperer/data
+      # - ./resources/elk6/vulnwhisperer.ini:/opt/VulnWhisperer/vulnwhisperer.ini
+      - ./configs/test.ini:/opt/VulnWhisperer/vulnwhisperer.ini
+      - ./tests/data/:/tests/data
+    network_mode: host
+
+networks:
+  esnet:

docker-compose.v6.yml

@@ -0,0 +1,86 @@
+version: '2'
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.6.0
+    container_name: elasticsearch
+    environment:
+      - cluster.name=vulnwhisperer
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
+      - xpack.security.enabled=false
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    mem_limit: 8g
+    volumes:
+      - esdata1:/usr/share/elasticsearch/data
+    ports:
+      - 9200:9200
+    #restart: always
+    networks:
+      esnet:
+        aliases:
+          - elasticsearch.local
+  kibana:
+    image: docker.elastic.co/kibana/kibana:6.6.0
+    container_name: kibana
+    environment:
+      SERVER_NAME: kibana
+      ELASTICSEARCH_URL: http://elasticsearch:9200
+    ports:
+      - 5601:5601
+    depends_on:
+      - elasticsearch
+    networks:
+      esnet:
+        aliases:
+          - kibana.local
+  kibana-config:
+    image: alpine
+    container_name: kibana-config
+    volumes:
+      - ./resources/elk6/init_kibana.sh:/opt/init_kibana.sh
+      - ./resources/elk6/kibana_APIonly.json:/opt/kibana_APIonly.json
+      - ./resources/elk6/logstash-vulnwhisperer-template.json:/opt/index-template.json
+    command: sh -c "apk add --no-cache curl bash && chmod +x /opt/init_kibana.sh && chmod +r /opt/kibana_APIonly.json && cd /opt/ && /bin/bash /opt/init_kibana.sh" # /opt/kibana_APIonly.json"
+    networks:
+      esnet:
+        aliases:
+          - kibana-config.local
+  logstash:
+    image: docker.elastic.co/logstash/logstash:6.6.0
+    container_name: logstash
+    volumes:
+      - ./resources/elk6/pipeline/:/usr/share/logstash/pipeline
+      - ./data/:/opt/VulnWhisperer/data
+      #- ./resources/elk6/logstash.yml:/usr/share/logstash/config/logstash.yml
+    environment:
+      - xpack.monitoring.enabled=false
+    depends_on:
+      - elasticsearch
+    networks:
+      esnet:
+        aliases:
+          - logstash.local
+  vulnwhisperer:
+    image: hasecuritysolutions/vulnwhisperer:latest
+    container_name: vulnwhisperer
+    entrypoint: [
+                  "vuln_whisperer",
+                  "-c",
+                  "/opt/VulnWhisperer/vulnwhisperer.ini"
+    ]    
+    volumes:
+      - ./data/:/opt/VulnWhisperer/data
+      - ./resources/elk6/vulnwhisperer.ini:/opt/VulnWhisperer/vulnwhisperer.ini
+    network_mode: host
+volumes:
+  esdata1:
+    driver: local
+
+networks:
+  esnet:

requirements.txt

@@ -0,0 +1,12 @@
+pandas==0.20.3
+setuptools==40.4.3
+pytz==2017.2
+Requests==2.20.0
+lxml==4.6.5
+future-fstrings
+bs4
+jira
+bottle
+coloredlogs
+qualysapi==6.0.0
+httpretty

resources/elk5-old_compatibility/docker-compose_ELK5_unsupported.yml

@@ -0,0 +1,72 @@
+version: '2'
+services:
+  vulnwhisp-es1:
+    image: docker.elastic.co/elasticsearch/elasticsearch:5.6.2
+    container_name: vulnwhisp-es1
+    environment:
+      - cluster.name=vulnwhisperer
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    mem_limit: 8g
+    volumes:
+      - esdata1:/usr/share/elasticsearch/data
+    ports:
+      - 9200:9200
+    environment:
+      - xpack.security.enabled=false
+    #restart: always
+    networks:
+      esnet:
+        aliases:
+          - vulnwhisp-es1.local
+  vulnwhisp-ks1:
+    image: docker.elastic.co/kibana/kibana:5.6.2
+    environment:
+      SERVER_NAME: vulnwhisp-ks1
+      ELASTICSEARCH_URL: http://vulnwhisp-es1:9200
+    ports:
+      - 5601:5601
+    depends_on:
+      - vulnwhisp-es1
+    networks:
+      esnet:
+        aliases:
+          - vulnwhisp-ks1.local
+  vulnwhisp-ls1:
+    image: docker.elastic.co/logstash/logstash:5.6.2
+    container_name: vulnwhisp-ls1
+    volumes:
+      - ./docker/1000_nessus_process_file.conf:/usr/share/logstash/pipeline/1000_nessus_process_file.conf
+      - ./docker/2000_qualys_web_scans.conf:/usr/share/logstash/pipeline/2000_qualys_web_scans.conf
+      - ./docker/3000_openvas.conf:/usr/share/logstash/pipeline/3000_openvas.conf
+      - ./docker/4000_jira.conf:/usr/share/logstash/pipeline/4000_jira.conf
+      - ./docker/logstash.yml:/usr/share/logstash/config/logstash.yml
+      - ./data/:/opt/VulnWhisperer/data
+    environment:
+      - xpack.monitoring.enabled=false
+    depends_on:
+      - vulnwhisp-es1
+    networks:
+      esnet:
+        aliases:
+          - vulnwhisp-ls1.local
+  vulnwhisp-vulnwhisperer:
+    image: hasecuritysolutions/vulnwhisperer:latest
+    container_name: vulnwhisp-vulnwhisperer
+    volumes:
+      - ./data/:/opt/VulnWhisperer/data
+      - ./configs/frameworks_example.ini:/opt/VulnWhisperer/frameworks_example.ini
+    network_mode: host
+volumes:
+  esdata1:
+    driver: local
+
+networks:
+  esnet:

resources/elk5-old_compatibility/docker/1000_nessus_process_file.conf

@@ -0,0 +1,220 @@
+# Author: Austin Taylor and Justin Henderson
+# Email: email@austintaylor.io
+# Last Update: 12/20/2017
+# Version 0.3
+# Description: Take in nessus reports from vulnWhisperer and pumps into logstash
+
+
+input {
+  file {
+    path => "/opt/VulnWhisperer/data/nessus/**/*"
+    start_position => "beginning"
+    tags => "nessus"
+    type => "nessus"
+  }
+  file {
+    path => "/opt/VulnWhisperer/data/tenable/*.csv"
+    start_position => "beginning"
+    tags => "tenable"
+    type => "tenable"
+  }
+}
+
+filter {
+  if "nessus" in [tags] or "tenable" in [tags] {
+    # Drop the header column
+    if [message] =~ "^Plugin ID" { drop {} }
+
+    csv {
+      # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"]
+      columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss_vector", "system_type", "host_start", "host_end"]
+      separator => ","
+      source => "message"
+    }
+
+    ruby {
+        code => "if event.get('description')
+                    event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
+                end
+                 if event.get('synopsis')
+                    event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
+                end
+                 if event.get('solution')
+                    event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
+                end
+                 if event.get('see_also')
+                    event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
+                end
+                 if event.get('plugin_output')
+                    event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
+                end"
+    }
+
+    #If using filebeats as your source, you will need to replace the "path" field to "source"
+    grok {
+      match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.csv$" }
+      tag_on_failure => []
+    }
+
+    date {
+      match => [ "last_updated", "UNIX" ]
+      target => "@timestamp"
+      remove_field => ["last_updated"]
+    }
+
+    if [risk] == "None" {
+      mutate { add_field => { "risk_number" => 0 }}
+    }
+    if [risk] == "Low" {
+      mutate { add_field => { "risk_number" => 1 }}
+    }
+    if [risk] == "Medium" {
+      mutate { add_field => { "risk_number" => 2 }}
+    }
+    if [risk] == "High" {
+      mutate { add_field => { "risk_number" => 3 }}
+    }
+    if [risk] == "Critical" {
+      mutate { add_field => { "risk_number" => 4 }}
+    }
+
+    if ![cve] or [cve] == "nan" {
+      mutate { remove_field => [ "cve" ] }
+    }
+    if ![cvss] or [cvss] == "nan" {
+      mutate { remove_field => [ "cvss" ] }
+    }
+    if ![cvss_base] or [cvss_base] == "nan" {
+      mutate { remove_field => [ "cvss_base" ] }
+    }
+    if ![cvss_temporal] or [cvss_temporal] == "nan" {
+      mutate { remove_field => [ "cvss_temporal" ] }
+    }
+    if ![cvss_temporal_vector] or [cvss_temporal_vector] == "nan" {
+      mutate { remove_field => [ "cvss_temporal_vector" ] }
+    }
+    if ![cvss_vector] or [cvss_vector] == "nan" {
+      mutate { remove_field => [ "cvss_vector" ] }
+    }
+    if ![cvss3_base] or [cvss3_base] == "nan" {
+      mutate { remove_field => [ "cvss3_base" ] }
+    }
+    if ![cvss3_temporal] or [cvss3_temporal] == "nan" {
+      mutate { remove_field => [ "cvss3_temporal" ] }
+    }
+    if ![cvss3_temporal_vector] or [cvss3_temporal_vector] == "nan" {
+      mutate { remove_field => [ "cvss3_temporal_vector" ] }
+    }
+    if ![description] or [description] == "nan" {
+      mutate { remove_field => [ "description" ] }
+    }
+    if ![mac_address] or [mac_address] == "nan" {
+      mutate { remove_field => [ "mac_address" ] }
+    }
+    if ![netbios] or [netbios] == "nan" {
+      mutate { remove_field => [ "netbios" ] }
+    }
+    if ![operating_system] or [operating_system] == "nan" {
+      mutate { remove_field => [ "operating_system" ] }
+    }
+    if ![plugin_output] or [plugin_output] == "nan" {
+      mutate { remove_field => [ "plugin_output" ] }
+    }
+    if ![see_also] or [see_also] == "nan" {
+      mutate { remove_field => [ "see_also" ] }
+    }
+    if ![synopsis] or [synopsis] == "nan" {
+      mutate { remove_field => [ "synopsis" ] }
+    }
+    if ![system_type] or [system_type] == "nan" {
+      mutate { remove_field => [ "system_type" ] }
+    }
+
+    mutate {
+      remove_field => [ "message" ]
+      add_field => { "risk_score" => "%{cvss}" }
+    }
+    mutate {
+      convert => { "risk_score" => "float" }
+    }
+    if [risk_score] == 0 {
+      mutate {
+        add_field => { "risk_score_name" => "info" }
+      }
+    }
+    if [risk_score] > 0 and [risk_score] < 3 {
+      mutate {
+        add_field => { "risk_score_name" => "low" }
+      }
+    }
+    if [risk_score] >= 3 and [risk_score] < 6 {
+      mutate {
+        add_field => { "risk_score_name" => "medium" }
+      }
+    }
+    if [risk_score] >=6 and [risk_score] < 9 {
+      mutate {
+        add_field => { "risk_score_name" => "high" }
+      }
+    }
+    if [risk_score] >= 9 {
+      mutate {
+        add_field => { "risk_score_name" => "critical" }
+      }
+    }
+
+    # Compensating controls - adjust risk_score
+    # Adobe and Java are not allowed to run in browser unless whitelisted
+    # Therefore, lower score by dividing by 3 (score is subjective to risk)
+
+    #Modify and uncomment when ready to use
+    #if [risk_score] != 0 {
+    #  if [plugin_name] =~ "Adobe" and [risk_score] > 6 or [plugin_name] =~ "Java" and [risk_score] > 6 {
+    #    ruby {
+    #      code => "event.set('risk_score', event.get('risk_score') / 3)"
+    #    }
+    #    mutate {
+    #      add_field => { "compensating_control" => "Adobe and Flash removed from browsers unless whitelisted site." }
+    #    }
+    #  }
+    #}
+
+    # Add tags for reporting based on assets or criticality
+
+    if [asset] == "dc01" or [asset] == "dc02" or [asset] == "pki01" or [asset] == "192.168.0.54" or [asset] =~ "^192\.168\.0\." or [asset] =~ "^42.42.42." {
+      mutate {
+        add_tag => [ "critical_asset" ]
+      }
+    }
+    #if [asset] =~ "^192\.168\.[45][0-9][0-9]\.1$" or [asset] =~ "^192.168\.[50]\.[0-9]{1,2}\.1$"{
+    #  mutate {
+    #    add_tag => [ "has_hipaa_data" ]
+    #  }
+    #}
+    #if [asset] =~ "^192\.168\.[45][0-9][0-9]\." {
+    #  mutate {
+    #    add_tag => [ "hipaa_asset" ]
+    #  }
+    #}
+    if [asset] =~ "^hr" {
+      mutate {
+        add_tag => [ "pci_asset" ]
+      }
+    }
+    #if [asset] =~ "^10\.0\.50\." {
+    #  mutate {
+    #    add_tag => [ "web_servers" ]
+    #  }
+    #}
+  }
+}
+
+output {
+  if "nessus" in [tags] or "tenable" in [tags] or [type] in [ "nessus", "tenable" ] {
+    # stdout { codec => rubydebug }
+    elasticsearch {
+      hosts => [ "vulnwhisp-es1.local:9200" ]
+      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+    }
+  }
+}

resources/elk5-old_compatibility/docker/2000_qualys_web_scans.conf

@@ -0,0 +1,153 @@
+# Author: Austin Taylor and Justin Henderson
+# Email: austin@hasecuritysolutions.com
+# Last Update: 12/30/2017
+# Version 0.3
+# Description: Take in qualys web scan reports from vulnWhisperer and pumps into logstash
+
+input {
+  file {
+    path => "/opt/VulnWhisperer/data/qualys/*.json"
+    type => json
+    codec => json
+    start_position => "beginning"
+    tags => [ "qualys" ]
+  }
+}
+
+filter {
+  if "qualys" in [tags] {
+    grok {
+      match => { "path" => [ "(?<tags>qualys_vuln)_scan_%{DATA}_%{INT:last_updated}.json$", "(?<tags>qualys_web)_%{INT:app_id}_%{INT:last_updated}.json$" ] }
+      tag_on_failure => []
+    }
+    
+    mutate {
+      replace => [ "message", "%{message}" ]
+      #gsub => [
+      #  "message", "\|\|\|", " ",
+      #  "message", "\t\t", " ",
+      #  "message", "    ", " ",
+      #  "message", "   ", " ",
+      #  "message", "  ", " ",
+      #  "message", "nan", " ",
+      #  "message",'\n',''
+      #]
+    }
+
+    if "qualys_web" in [tags] {
+    mutate {
+      add_field => { "asset" => "%{web_application_name}" }
+      add_field => { "risk_score" => "%{cvss}" }
+      }
+    } else if "qualys_vuln" in [tags] {
+      mutate {
+        add_field => { "asset" => "%{ip}" }
+        add_field => { "risk_score" => "%{cvss}" }
+      }
+    }
+
+    if [risk] == "1" {
+      mutate { add_field => { "risk_number" => 0 }}
+      mutate { replace => { "risk" => "info" }}
+    }
+    if [risk] == "2" {
+      mutate { add_field => { "risk_number" => 1 }}
+      mutate { replace => { "risk" => "low" }}
+    }
+    if [risk] == "3" {
+      mutate { add_field => { "risk_number" => 2 }}
+      mutate { replace => { "risk" => "medium" }}
+    }
+    if [risk] == "4" {
+      mutate { add_field => { "risk_number" => 3 }}
+      mutate { replace => { "risk" => "high" }}
+    }
+    if [risk] == "5" {
+      mutate { add_field => { "risk_number" => 4 }}
+      mutate { replace => { "risk" => "critical" }}
+    }
+
+    mutate {
+      remove_field => "message"
+    }
+
+    if [first_time_detected] {
+      date {
+        match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "first_time_detected"
+      }
+    }
+    if [first_time_tested] {
+      date {
+        match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "first_time_tested"
+      }
+    }
+    if [last_time_detected] {
+      date {
+        match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "last_time_detected"
+      }
+    }
+    if [last_time_tested] {
+      date {
+        match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "last_time_tested"
+      }
+    }
+    date {
+      match => [ "last_updated", "UNIX" ]
+      target => "@timestamp"
+      remove_field => "last_updated"
+    }
+    mutate {
+      convert => { "plugin_id" => "integer"}
+      convert => { "id" => "integer"}
+      convert => { "risk_number" => "integer"}
+      convert => { "risk_score" => "float"}
+      convert => { "total_times_detected" => "integer"}
+      convert => { "cvss_temporal" => "float"}
+      convert => { "cvss" => "float"}
+    }
+    if [risk_score] == 0 {
+      mutate {
+        add_field => { "risk_score_name" => "info" }
+      }
+    }
+    if [risk_score] > 0 and [risk_score] < 3 {
+      mutate {
+        add_field => { "risk_score_name" => "low" }
+      }
+    }
+    if [risk_score] >= 3 and [risk_score] < 6 {
+      mutate {
+        add_field => { "risk_score_name" => "medium" }
+      }
+    }
+    if [risk_score] >=6 and [risk_score] < 9 {
+      mutate {
+        add_field => { "risk_score_name" => "high" }
+      }
+    }
+    if [risk_score] >= 9 {
+      mutate {
+        add_field => { "risk_score_name" => "critical" }
+      }
+    }
+
+    if [asset] =~ "\.yourdomain\.(com|net)$" {
+      mutate {
+        add_tag => [ "critical_asset" ]
+      }
+    }
+  }
+}
+output {
+  if "qualys" in [tags] {
+    stdout { codec => rubydebug }
+    elasticsearch {
+      hosts => [ "vulnwhisp-es1.local:9200" ]
+      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+    }
+  }
+}

resources/elk5-old_compatibility/docker/3000_openvas.conf

@@ -0,0 +1,146 @@
+# Author: Austin Taylor and Justin Henderson
+# Email: austin@hasecuritysolutions.com
+# Last Update: 03/04/2018
+# Version 0.3
+# Description: Take in Openvas web scan reports from vulnWhisperer and pumps into logstash
+
+input {
+  file {
+    path => "/opt/VulnWhisperer/data/openvas/*.json"
+    type => json
+    codec => json
+    start_position => "beginning"
+    tags => [ "openvas_scan", "openvas" ]
+  }
+}
+
+filter {
+  if "openvas_scan" in [tags] {
+    mutate {
+      replace => [ "message", "%{message}" ]
+      gsub => [
+        "message", "\|\|\|", " ",
+        "message", "\t\t", " ",
+        "message", "    ", " ",
+        "message", "   ", " ",
+        "message", "  ", " ",
+        "message", "nan", " ",
+        "message",'\n',''
+      ]
+    }
+
+
+    grok {
+        match => { "path" => "openvas_scan_%{DATA:scan_id}_%{INT:last_updated}.json$" }
+        tag_on_failure => []
+    }
+
+    mutate {
+      add_field => { "risk_score" => "%{cvss}" }
+    }
+
+    if [risk] == "1" {
+      mutate { add_field => { "risk_number" => 0 }}
+      mutate { replace => { "risk" => "info" }}
+    }
+    if [risk] == "2" {
+      mutate { add_field => { "risk_number" => 1 }}
+      mutate { replace => { "risk" => "low" }}
+    }
+    if [risk] == "3" {
+      mutate { add_field => { "risk_number" => 2 }}
+      mutate { replace => { "risk" => "medium" }}
+    }
+    if [risk] == "4" {
+      mutate { add_field => { "risk_number" => 3 }}
+      mutate { replace => { "risk" => "high" }}
+    }
+    if [risk] == "5" {
+      mutate { add_field => { "risk_number" => 4 }}
+      mutate { replace => { "risk" => "critical" }}
+    }
+
+    mutate {
+      remove_field => "message"
+    }
+
+    if [first_time_detected] {
+      date {
+        match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "first_time_detected"
+      }
+    }
+    if [first_time_tested] {
+      date {
+        match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "first_time_tested"
+      }
+    }
+    if [last_time_detected] {
+      date {
+        match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "last_time_detected"
+      }
+    }
+    if [last_time_tested] {
+      date {
+        match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "last_time_tested"
+      }
+    }
+    date {
+      match => [ "last_updated", "UNIX" ]
+      target => "@timestamp"
+      remove_field => "last_updated"
+    }
+    mutate {
+      convert => { "plugin_id" => "integer"}
+      convert => { "id" => "integer"}
+      convert => { "risk_number" => "integer"}
+      convert => { "risk_score" => "float"}
+      convert => { "total_times_detected" => "integer"}
+      convert => { "cvss_temporal" => "float"}
+      convert => { "cvss" => "float"}
+    }
+    if [risk_score] == 0 {
+      mutate {
+        add_field => { "risk_score_name" => "info" }
+      }
+    }
+    if [risk_score] > 0 and [risk_score] < 3 {
+      mutate {
+        add_field => { "risk_score_name" => "low" }
+      }
+    }
+    if [risk_score] >= 3 and [risk_score] < 6 {
+      mutate {
+        add_field => { "risk_score_name" => "medium" }
+      }
+    }
+    if [risk_score] >=6 and [risk_score] < 9 {
+      mutate {
+        add_field => { "risk_score_name" => "high" }
+      }
+    }
+    if [risk_score] >= 9 {
+      mutate {
+        add_field => { "risk_score_name" => "critical" }
+      }
+    }
+    # Add your critical assets by subnet or by hostname. Comment this field out if you don't want to tag any, but the asset panel will break.
+    if [asset] =~ "^10\.0\.100\." {
+      mutate {
+        add_tag => [ "critical_asset" ]
+      }
+    }
+  }
+}
+output {
+  if "openvas" in [tags] {
+    stdout { codec => rubydebug }
+    elasticsearch {
+      hosts => [ "vulnwhisp-es1.local:9200" ]
+      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+    }
+  }
+}

resources/elk5-old_compatibility/docker/4000_jira.conf

@@ -0,0 +1,21 @@
+# Description: Take in jira tickets from vulnWhisperer and pumps into logstash
+
+input {
+  file {
+    path => "/opt/Vulnwhisperer/jira/*.json"
+    type => json
+    codec => json
+    start_position => "beginning"
+    tags => [ "jira" ]
+  }
+}
+
+output {
+  if "jira" in [tags] {
+    stdout { codec => rubydebug }
+    elasticsearch {
+      hosts => [ "vulnwhisp-es1.local:9200" ]
+      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+    }
+  }
+}

resources/elk5-old_compatibility/docker/logstash.yml

@@ -0,0 +1,5 @@
+path.config: /usr/share/logstash/pipeline/
+xpack.monitoring.elasticsearch.password: changeme
+xpack.monitoring.elasticsearch.url: vulnwhisp-es1.local:9200
+xpack.monitoring.elasticsearch.username: elastic
+xpack.monitoring.enabled: false

resources/elk5-old_compatibility/elasticsearch/logstash-vulnwhisperer-template.json

@@ -0,0 +1,122 @@
+{
+  "order": 0,
+  "template": "logstash-vulnwhisperer-*",
+  "settings": {
+    "index": {
+      "routing": {
+        "allocation": {
+          "total_shards_per_node": "2"
+        }
+      },
+      "mapping": {
+        "total_fields": {
+          "limit": "3000"
+        }
+      },
+      "refresh_interval": "5s",
+      "number_of_shards": "1",
+      "number_of_replicas": "0"
+    }
+  },
+  "mappings": {
+    "_default_": {
+      "_all": {
+        "enabled": false
+      },
+      "dynamic_templates": [
+        {
+          "message_field": {
+            "path_match": "message",
+            "match_mapping_type": "string",
+            "mapping": {
+              "type": "text",
+              "norms": false
+            }
+          }
+        },
+        {
+          "string_fields": {
+            "match": "*",
+            "match_mapping_type": "string",
+            "mapping": {
+              "type": "text",
+              "norms": false,
+              "fields": {
+                "keyword": {
+                  "type": "keyword",
+                  "ignore_above": 256
+                }
+              }
+            }
+          }
+        }
+      ],
+      "properties": {
+        "plugin_id": {
+          "type": "float"
+        },
+        "last_updated": {
+          "type": "date"
+        },
+        "geoip": {
+          "dynamic": true,
+          "type": "object",
+          "properties": {
+            "ip": {
+              "type": "ip"
+            },
+            "latitude": {
+              "type": "float"
+            },
+            "location": {
+              "type": "geo_point"
+            },
+            "longitude": {
+              "type": "float"
+            }
+          }
+        },
+        "risk_score": {
+          "type": "float"
+        },
+        "source": {          
+          "type": "keyword"
+        },
+        "synopsis": {
+          "type": "keyword"
+        },
+        "see_also": {
+          "type": "keyword"
+        },
+        "@timestamp": {
+          "type": "date"
+        },
+        "cve": {
+          "type": "keyword"
+        },
+        "solution": {
+          "type": "keyword"
+        },
+        "port": {
+          "type": "integer"
+        },
+        "host": {
+          "type": "text"
+        },
+        "@version": {
+          "type": "keyword"
+        },
+        "risk": {
+          "type": "keyword"
+        },
+        "assign_ip": {
+          "type": "ip"
+        },
+        "cvss": {
+          "type": "float"
+        }
+      }
+    }
+  },
+  "aliases": {}
+}

resources/elk5-old_compatibility/filebeat/filebeat.yml

@@ -0,0 +1,116 @@
+###################### Filebeat Configuration Example #########################
+
+# This file is an example configuration file highlighting only the most common
+# options. The filebeat.full.yml file from the same directory contains all the
+# supported options with more comments. You can use it as a reference.
+#
+# You can find the full configuration reference here:
+# https://www.elastic.co/guide/en/beats/filebeat/index.html
+
+#=========================== Filebeat prospectors =============================
+
+filebeat.prospectors:
+
+# Each - is a prospector. Most options can be set at the prospector level, so
+# you can use different prospectors for various configurations.
+# Below are the prospector specific configurations.
+
+- input_type: log
+  # Paths that should be crawled and fetched. Glob based paths.
+  paths:
+    # Linux Example
+    #- /var/log/*.log
+
+    #Windows Example
+    - c:\nessus\My Scans\*
+
+  # Exclude lines. A list of regular expressions to match. It drops the lines that are
+  # matching any regular expression from the list.
+  #exclude_lines: ["^DBG"]
+
+  # Include lines. A list of regular expressions to match. It exports the lines that are
+  # matching any regular expression from the list.
+  #include_lines: ["^ERR", "^WARN"]
+
+  # Exclude files. A list of regular expressions to match. Filebeat drops the files that
+  # are matching any regular expression from the list. By default, no files are dropped.
+  #exclude_files: [".gz$"]
+
+  # Optional additional fields. These field can be freely picked
+  # to add additional information to the crawled log files for filtering
+  #fields:
+  #  level: debug
+  #  review: 1
+
+  ### Multiline options
+
+  # Mutiline can be used for log messages spanning multiple lines. This is common
+  # for Java Stack Traces or C-Line Continuation
+
+  # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [
+  #multiline.pattern: ^\[
+
+  # Defines if the pattern set under pattern should be negated or not. Default is false.
+  #multiline.negate: false
+
+  # Match can be set to "after" or "before". It is used to define if lines should be append to a pattern
+  # that was (not) matched before or after or as long as a pattern is not matched based on negate.
+  # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash
+  #multiline.match: after
+
+
+#================================ General =====================================
+
+# The name of the shipper that publishes the network data. It can be used to group
+# all the transactions sent by a single shipper in the web interface.
+#name:
+
+# The tags of the shipper are included in their own field with each
+# transaction published.
+#tags: ["service-X", "web-tier"]
+
+# Optional fields that you can specify to add additional information to the
+# output.
+#fields:
+#  env: staging
+
+#================================ Outputs =====================================
+
+# Configure what outputs to use when sending the data collected by the beat.
+# Multiple outputs may be used.
+
+#-------------------------- Elasticsearch output ------------------------------
+#output.elasticsearch:
+  # Array of hosts to connect to.
+#  hosts: ["logstash01:9200"]
+
+  # Optional protocol and basic auth credentials.
+  #protocol: "https"
+  #username: "elastic"
+  #password: "changeme"
+
+#----------------------------- Logstash output --------------------------------
+output.logstash:
+  # The Logstash hosts
+  hosts: ["logstashserver1:5044", "logstashserver2:5044", "logstashserver3:5044"]
+
+  # Optional SSL. By default is off.
+  # List of root certificates for HTTPS server verifications
+  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+  # Certificate for SSL client authentication
+  #ssl.certificate: "/etc/pki/client/cert.pem"
+
+  # Client Certificate Key
+  #ssl.key: "/etc/pki/client/cert.key"
+
+#================================ Logging =====================================
+
+# Sets log level. The default log level is info.
+# Available log levels are: critical, error, warning, info, debug
+#logging.level: debug
+
+# At debug level, you can selectively enable logging only for some components.
+# To enable all selectors use ["*"]. Examples of other selectors are "beat",
+# "publish", "service".
+#logging.selectors: ["*"]

resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/1000_vulnWhispererBaseVisuals.json

@@ -0,0 +1,450 @@
+[
+  {
+    "_id": "80158c90-57c1-11e7-b484-a970fc9d150a",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - HIPAA TL",
+      "visState": "{\"type\":\"timelion\",\"title\":\"VulnWhisperer - HIPAA TL\",\"params\":{\"expression\":\".es(index=logstash-vulnwhisperer-*,q='risk_score:>9 AND tags:pci_asset').label(\\\"PCI Assets\\\"),.es(index=logstash-vulnwhisperer-*,q='risk_score:>9 AND tags:has_hipaa_data').label(\\\"Has HIPAA Data\\\"),.es(index=logstash-vulnwhisperer-*,q='risk_score:>9 AND tags:hipaa_asset').label(\\\"HIPAA Assets\\\")\",\"interval\":\"auto\"}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{}"
+      }
+    }
+  },
+  {
+    "_id": "479deab0-8a39-11e7-a58a-9bfcb3761a3d",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - TL - TaggedAssetsPluginNames",
+      "visState": "{\"title\":\"VulnWhisperer - TL - TaggedAssetsPluginNames\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*', q='tags:critical_asset OR tags:hipaa_asset OR tags:pci_asset', split=\\\"plugin_name.keyword:10\\\").bars(width=4).label(regex=\\\".*:(.+)>.*\\\",label=\\\"$1\\\")\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "84f5c370-8a38-11e7-a58a-9bfcb3761a3d",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - TL - CriticalAssetsPluginNames",
+      "visState": "{\"title\":\"VulnWhisperer - TL - CriticalAssetsPluginNames\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*', q='tags:critical_asset', split=\\\"plugin_name.keyword:10\\\").bars(width=4).label(regex=\\\".*:(.+)>.*\\\",label=\\\"$1\\\")\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "307cdae0-8a38-11e7-a58a-9bfcb3761a3d",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - TL - PluginNames",
+      "visState": "{\"title\":\"VulnWhisperer - TL - PluginNames\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*', split=\\\"plugin_name.keyword:25\\\").bars(width=4).label(regex=\\\".*:(.+)>.*\\\",label=\\\"$1\\\")\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "5093c620-44e9-11e7-8014-ede06a7e69f8",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - Mitigation Readme",
+      "visState": "{\"title\":\"VulnWhisperer - Mitigation Readme\",\"type\":\"markdown\",\"params\":{\"markdown\":\"** Legend **\\n\\n* [Common Vulnerability Scoring System (CVSS)](https://nvd.nist.gov/vuln-metrics/cvss) is the NIST vulnerability scoring system\\n* Risk Number is residual risk score calculated from CVSS, which is adjusted to be specific to the netowrk owner, which accounts for services not in use such as Java and Flash\\n* Vulnerabilities by Tag are systems tagged with HIPAA and PCI identification.\\n\\n\\n** Workflow **\\n* Select 10.0 under Risk Number to identify Critical Vulnerabilities. \\n* For more information about a CVE, scroll down and click the CVE link.\\n* To filter by tags, use one of the following filters:\\n** tags:has_hipaa_data, tags:pci_asset, tags:hipaa_asset, tags:critical_asset**\"},\"aggs\":[],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "7e7fbc90-3df2-11e7-a44e-c79ca8efb780",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer-PluginID",
+      "visState": "{\"title\":\"VulnWhisperer-PluginID\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"plugin_id\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "5a3c0340-3eb3-11e7-a192-93f36fbd9d05",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer-CVSSHeatmap",
+      "visState": "{\"title\":\"VulnWhisperer-CVSSHeatmap\",\"type\":\"heatmap\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"enableHover\":false,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":4,\"colorSchema\":\"Yellow to Red\",\"setColorRange\":false,\"colorsRange\":[],\"invertColors\":false,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":false,\"rotate\":0,\"color\":\"#555\"}}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"host.keyword\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"cvss.keyword\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"_term\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 3500\":\"rgb(255,255,204)\",\"3500 - 7000\":\"rgb(254,217,118)\",\"7000 - 10500\":\"rgb(253,141,60)\",\"10500 - 14000\":\"rgb(227,27,28)\"}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "1de9e550-3df1-11e7-a44e-c79ca8efb780",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer-Description",
+      "visState": "{\"title\":\"VulnWhisperer-Description\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"description.keyword\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Description\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "13c7d4e0-3df3-11e7-a44e-c79ca8efb780",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer-Solution",
+      "visState": "{\"title\":\"VulnWhisperer-Solution\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"solution.keyword\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Solution\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "297df800-3f7e-11e7-bd24-6903e3283192",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - Plugin Name",
+      "visState": "{\"title\":\"VulnWhisperer - Plugin Name\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"plugin_name.keyword\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Plugin Name\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "de1a5f40-3f85-11e7-97f9-3777d794626d",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - ScanName",
+      "visState": "{\"title\":\"VulnWhisperer - ScanName\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"scan_name.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Scan Name\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "ecbb99c0-3f84-11e7-97f9-3777d794626d",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - Total",
+      "visState": "{\"title\":\"VulnWhisperer - Total\",\"type\":\"metric\",\"params\":{\"handleNoResults\":true,\"fontSize\":60},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Total\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "471a3580-3f6b-11e7-88e7-df1abe6547fb",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - Vulnerabilities by Tag",
+      "visState": "{\"title\":\"VulnWhisperer - Vulnerabilities by Tag\",\"type\":\"table\",\"params\":{\"perPage\":3,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"3\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"bucket\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"tags:has_hipaa_data\",\"analyze_wildcard\":true}}},\"label\":\"Systems with HIPAA data\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"tags:pci_asset\",\"analyze_wildcard\":true}}},\"label\":\"PCI Systems\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"tags:hipaa_asset\",\"analyze_wildcard\":true}}},\"label\":\"HIPAA Systems\"}]}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "35b6d320-3f7f-11e7-bd24-6903e3283192",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - Residual Risk",
+      "visState": "{\"title\":\"VulnWhisperer - Residual Risk\",\"type\":\"table\",\"params\":{\"perPage\":15,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"},\"showTotal\":false,\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"risk_score\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Risk Number\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "a9225930-3df2-11e7-a44e-c79ca8efb780",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer-Risk",
+      "visState": "{\"title\":\"VulnWhisperer-Risk\",\"type\":\"table\",\"params\":{\"perPage\":4,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"risk\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Risk Severity\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "2f979030-44b9-11e7-a818-f5f80dfc3590",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - ScanBarChart",
+      "visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customLabel\":\"Scan Name\",\"field\":\"plugin_name.keyword\",\"order\":\"desc\",\"orderBy\":\"1\",\"size\":10},\"schema\":\"segment\",\"type\":\"terms\"}],\"listeners\":{},\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"defaultYExtents\":false,\"legendPosition\":\"right\",\"mode\":\"stacked\",\"scale\":\"linear\",\"setYExtents\":false,\"times\":[]},\"title\":\"VulnWhisperer - ScanBarChart\",\"type\":\"histogram\"}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "a6508640-897a-11e7-bbc0-33592ce0be1e",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - Critical Assets Aggregated",
+      "visState": "{\"title\":\"VulnWhisperer - Critical Assets Aggregated\",\"type\":\"heatmap\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"enableHover\":true,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":4,\"colorSchema\":\"Green to Red\",\"setColorRange\":true,\"colorsRange\":[{\"from\":0,\"to\":3},{\"from\":3,\"to\":7},{\"from\":7,\"to\":9},{\"from\":9,\"to\":11}],\"invertColors\":false,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":true,\"rotate\":0,\"color\":\"white\"}}]},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"risk_score\",\"customLabel\":\"Residual Risk Score\"}},{\"id\":\"3\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Date\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"host\",\"size\":10,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Critical Asset IP\"}},{\"id\":\"5\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"plugin_name.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"row\":true}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"colors\":{\"0 - 3\":\"#7EB26D\",\"3 - 7\":\"#EAB839\",\"7 - 9\":\"#EF843C\",\"8 - 10\":\"#BF1B00\",\"9 - 11\":\"#BF1B00\"},\"defaultColors\":{\"0 - 3\":\"rgb(0,104,55)\",\"3 - 7\":\"rgb(135,203,103)\",\"7 - 9\":\"rgb(255,255,190)\",\"9 - 11\":\"rgb(249,142,82)\"},\"legendOpen\":false}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[{\"$state\":{\"store\":\"appState\"},\"meta\":{\"alias\":\"Critical Asset\",\"disabled\":false,\"index\":\"logstash-vulnwhisperer-*\",\"key\":\"tags\",\"negate\":false,\"type\":\"phrase\",\"value\":\"critical_asset\"},\"query\":{\"match\":{\"tags\":{\"query\":\"critical_asset\",\"type\":\"phrase\"}}}}]}"
+      }
+    }
+  },
+  {
+    "_id": "099a3820-3f68-11e7-a6bd-e764d950e506",
+    "_type": "visualization",
+    "_source": {
+      "title": "Timelion VulnWhisperer Example",
+      "visState": "{\"type\":\"timelion\",\"title\":\"Timelion VulnWhisperer Example\",\"params\":{\"expression\":\".es(index=logstash-vulnwhisperer-*,q=risk:high).label(\\\"Current High Risk\\\"),.es(index=logstash-vulnwhisperer-*,q=risk:high,offset=-1y).label(\\\"Last 1 Year High Risk\\\"),.es(index=logstash-vulnwhisperer-*,q=risk:medium).label(\\\"Current Medium Risk\\\"),.es(index=logstash-vulnwhisperer-*,q=risk:medium,offset=-1y).label(\\\"Last 1 Year Medium Risk\\\")\",\"interval\":\"auto\"}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{}"
+      }
+    }
+  },
+  {
+    "_id": "67d432e0-44ec-11e7-a05f-d9719b331a27",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - TL-Critical Risk",
+      "visState": "{\"title\":\"VulnWhisperer - TL-Critical Risk\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*',q='(risk_score:>=9 AND risk_score:<=10)').label(\\\"Original\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=9 AND risk_score:<=10)',offset=-1w).label(\\\"One week offset\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=9 AND risk_score:<=10)').subtract(.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=9 AND risk_score:<=10)',offset=-1w)).label(\\\"Difference\\\").lines(steps=3,fill=2,width=1)\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "a91b9fe0-44ec-11e7-a05f-d9719b331a27",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - TL-Medium Risk",
+      "visState": "{\"title\":\"VulnWhisperer - TL-Medium Risk\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*',q='(risk_score:>=4 AND risk_score:<7)').label(\\\"Original\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=4 AND risk_score:<7)',offset=-1w).label(\\\"One week offset\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=4 AND risk_score:<7)').subtract(.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=4 AND risk_score:<7)',offset=-1w)).label(\\\"Difference\\\").lines(steps=3,fill=2,width=1)\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "8d9592d0-44ec-11e7-a05f-d9719b331a27",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - TL-High Risk",
+      "visState": "{\"title\":\"VulnWhisperer - TL-High Risk\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*',q='(risk_score:>=7 AND risk_score:<9)').label(\\\"Original\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=7 AND risk_score:<9)',offset=-1w).label(\\\"One week offset\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=7 AND risk_score:<9)').subtract(.es(index='logstash-vulnwhisperer-*',q='(risk_score:>=7 AND risk_score:<9)',offset=-1w)).label(\\\"Difference\\\").lines(steps=3,fill=2,width=1)\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "a2d66660-44ec-11e7-a05f-d9719b331a27",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - TL-Low Risk",
+      "visState": "{\"title\":\"VulnWhisperer - TL-Low Risk\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*',q='(risk_score:>0 AND risk_score:<4)').label(\\\"Original\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>0 AND risk_score:<4)',offset=-1w).label(\\\"One week offset\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk_score:>0 AND risk_score:<4)').subtract(.es(index='logstash-vulnwhisperer-*',q='(risk_score:>0 AND risk_score:<4)',offset=-1w)).label(\\\"Difference\\\").lines(steps=3,fill=2,width=1)\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "fb6eb020-49ab-11e7-8f8c-57ad64ec48a6",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - Critical Risk Score for Tagged Assets",
+      "visState": "{\"title\":\"VulnWhisperer - Critical Risk Score for Tagged Assets\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index=logstash-vulnwhisperer-*,q='risk_score:>9 AND tags:hipaa_asset').label(\\\"HIPAA Assets\\\"),.es(index=logstash-vulnwhisperer-*,q='risk_score:>9 AND tags:pci_asset').label(\\\"PCI Systems\\\"),.es(index=logstash-vulnwhisperer-*,q='risk_score:>9 AND tags:has_hipaa_data').label(\\\"Has HIPAA Data\\\")\",\"interval\":\"auto\"},\"aggs\":[],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "b2f2adb0-897f-11e7-a2d2-c57bca21b3aa",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer  - Risk: Total",
+      "visState": "{\"title\":\"VulnWhisperer  - Risk: Total\",\"type\":\"goal\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"gauge\":{\"autoExtend\":false,\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"gaugeColorMode\":\"Background\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Metric\",\"invertColors\":false,\"labels\":{\"color\":\"black\",\"show\":false},\"orientation\":\"vertical\",\"percentageMode\":false,\"scale\":{\"color\":\"#333\",\"labels\":false,\"show\":true,\"width\":2},\"style\":{\"bgColor\":true,\"bgFill\":\"white\",\"fontSize\":\"34\",\"labelColor\":false,\"subText\":\"Risk\"},\"type\":\"simple\",\"useRanges\":false,\"verticalSplit\":false},\"type\":\"gauge\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Total\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}}},\"label\":\"Critical\"}]}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"colors\":{\"0 - 10000\":\"#64B0C8\"},\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "465c5820-8977-11e7-857e-e1d56b17746d",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - Critical Assets",
+      "visState": "{\"title\":\"VulnWhisperer - Critical Assets\",\"type\":\"heatmap\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"enableHover\":true,\"legendPosition\":\"right\",\"times\":[],\"colorsNumber\":4,\"colorSchema\":\"Green to Red\",\"setColorRange\":true,\"colorsRange\":[{\"from\":0,\"to\":3},{\"from\":3,\"to\":7},{\"from\":7,\"to\":9},{\"from\":9,\"to\":11}],\"invertColors\":false,\"percentageMode\":false,\"valueAxes\":[{\"show\":false,\"id\":\"ValueAxis-1\",\"type\":\"value\",\"scale\":{\"type\":\"linear\",\"defaultYExtents\":false},\"labels\":{\"show\":false,\"rotate\":0,\"color\":\"white\"}}],\"type\":\"heatmap\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"max\",\"schema\":\"metric\",\"params\":{\"field\":\"risk_score\",\"customLabel\":\"Residual Risk Score\"}},{\"id\":\"2\",\"enabled\":false,\"type\":\"terms\",\"schema\":\"split\",\"params\":{\"field\":\"risk_score\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"row\":true}},{\"id\":\"3\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"Date\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"asset.keyword\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Critical Asset\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 3\":\"rgb(0,104,55)\",\"3 - 7\":\"rgb(135,203,103)\",\"7 - 9\":\"rgb(255,255,190)\",\"9 - 11\":\"rgb(249,142,82)\"},\"colors\":{\"8 - 10\":\"#BF1B00\",\"9 - 11\":\"#BF1B00\",\"7 - 9\":\"#EF843C\",\"3 - 7\":\"#EAB839\",\"0 - 3\":\"#7EB26D\"},\"legendOpen\":false}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[{\"meta\":{\"index\":\"logstash-vulnwhisperer-*\",\"negate\":false,\"disabled\":false,\"alias\":\"Critical Asset\",\"type\":\"phrase\",\"key\":\"tags\",\"value\":\"critical_asset\"},\"query\":{\"match\":{\"tags\":{\"query\":\"critical_asset\",\"type\":\"phrase\"}}},\"$state\":{\"store\":\"appState\"}}]}"
+      }
+    }
+  },
+  {
+    "_id": "852816e0-3eb1-11e7-90cb-918f9cb01e3d",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer-CVSS",
+      "visState": "{\"title\":\"VulnWhisperer-CVSS\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"cvss.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"CVSS Score\"}},{\"id\":\"4\",\"enabled\":true,\"type\":\"cardinality\",\"schema\":\"metric\",\"params\":{\"field\":\"asset.keyword\",\"customLabel\":\"# of Assets\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "d048c220-80b3-11e7-8790-73b60225f736",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer  - Risk: High",
+      "visState": "{\"title\":\"VulnWhisperer  - Risk: High\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Background\",\"colorsRange\":[{\"from\":0,\"to\":1000}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"bgFill\":\"white\",\"bgColor\":true,\"labelColor\":false,\"subText\":\"\",\"fontSize\":\"34\"},\"extendRange\":true}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"High Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:high\"}}},\"label\":\"\"}]}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 10000\":\"#EF843C\",\"0 - 1000\":\"#E0752D\"}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "db55bce0-80b3-11e7-8790-73b60225f736",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer  - Risk: Critical",
+      "visState": "{\"title\":\"VulnWhisperer  - Risk: Critical\",\"type\":\"goal\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"gauge\":{\"autoExtend\":false,\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"gaugeColorMode\":\"Background\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Metric\",\"invertColors\":false,\"labels\":{\"color\":\"black\",\"show\":false},\"orientation\":\"vertical\",\"percentageMode\":false,\"scale\":{\"color\":\"#333\",\"labels\":false,\"show\":true,\"width\":2},\"style\":{\"bgColor\":true,\"bgFill\":\"white\",\"fontSize\":\"34\",\"labelColor\":false,\"subText\":\"Risk\"},\"type\":\"simple\",\"useRanges\":false,\"verticalSplit\":false},\"type\":\"gauge\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Critical Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:critical\"}}},\"label\":\"Critical\"}]}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"colors\":{\"0 - 10000\":\"#BF1B00\"},\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "56f0f5f0-3ebe-11e7-a192-93f36fbd9d05",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer-RiskOverTime",
+      "visState": "{\"title\":\"VulnWhisperer-RiskOverTime\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per 12 hours\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"orderBucketsBySum\":false,\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"mode\":\"normal\",\"show\":\"true\",\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}],\"type\":\"line\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:info\"}}},\"label\":\"Info\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:low\"}}},\"label\":\"Low\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:medium\"}}},\"label\":\"Medium\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:high\"}}},\"label\":\"High\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:critical\"}}},\"label\":\"Critical\"}]}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"colors\":{\"Critical\":\"#962D82\",\"High\":\"#BF1B00\",\"Low\":\"#629E51\",\"Medium\":\"#EAB839\",\"Info\":\"#65C5DB\"}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "c1361da0-80b3-11e7-8790-73b60225f736",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer  - Risk: Medium",
+      "visState": "{\"title\":\"VulnWhisperer  - Risk: Medium\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Background\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"bgFill\":\"white\",\"bgColor\":true,\"labelColor\":false,\"subText\":\"\",\"fontSize\":\"34\"},\"extendRange\":false},\"isDisplayWarning\":false},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Medium Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:medium\"}}},\"label\":\"Medium Risk\"}]}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 10000\":\"#EAB839\"}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "e46ff7f0-897d-11e7-934b-67cec0a7da65",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer  - Risk: Low",
+      "visState": "{\"title\":\"VulnWhisperer  - Risk: Low\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Background\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"bgFill\":\"white\",\"bgColor\":true,\"labelColor\":false,\"subText\":\"\",\"fontSize\":\"34\"},\"extendRange\":false}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Low Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk_score_name:low\"}}},\"label\":\"Low Risk\"}]}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 10000\":\"#629E51\"}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "995e2280-3df3-11e7-a44e-c79ca8efb780",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer-Asset",
+      "visState": "{\"title\":\"VulnWhisperer-Asset\",\"type\":\"table\",\"params\":{\"perPage\":15,\"showMeticsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"asset.keyword\",\"size\":50,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Asset\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
+      }
+    }
+  }
+]

resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/1001_vulnWhisperer_ReportingMitigationDashboard.json

@@ -0,0 +1,43 @@
+[
+  {
+    "_id": "72051530-448e-11e7-a818-f5f80dfc3590",
+    "_type": "dashboard",
+    "_source": {
+      "title": "VulnWhisperer - Reporting",
+      "hits": 0,
+      "description": "",
+      "panelsJSON": "[{\"col\":1,\"id\":\"2f979030-44b9-11e7-a818-f5f80dfc3590\",\"panelIndex\":5,\"row\":12,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"8d9592d0-44ec-11e7-a05f-d9719b331a27\",\"panelIndex\":12,\"row\":8,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"67d432e0-44ec-11e7-a05f-d9719b331a27\",\"panelIndex\":14,\"row\":4,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":10,\"id\":\"297df800-3f7e-11e7-bd24-6903e3283192\",\"panelIndex\":15,\"row\":8,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"471a3580-3f6b-11e7-88e7-df1abe6547fb\",\"panelIndex\":20,\"row\":8,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":11,\"id\":\"995e2280-3df3-11e7-a44e-c79ca8efb780\",\"panelIndex\":22,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":9,\"id\":\"b2f2adb0-897f-11e7-a2d2-c57bca21b3aa\",\"panelIndex\":23,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"db55bce0-80b3-11e7-8790-73b60225f736\",\"panelIndex\":25,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"d048c220-80b3-11e7-8790-73b60225f736\",\"panelIndex\":26,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"e46ff7f0-897d-11e7-934b-67cec0a7da65\",\"panelIndex\":27,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":3,\"id\":\"c1361da0-80b3-11e7-8790-73b60225f736\",\"panelIndex\":28,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"479deab0-8a39-11e7-a58a-9bfcb3761a3d\",\"panelIndex\":29,\"row\":4,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"}]",
+      "optionsJSON": "{\"darkTheme\":false}",
+      "uiStateJSON": "{\"P-15\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-20\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-21\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-22\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-23\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-24\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-25\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-26\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-27\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-28\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-5\":{\"vis\":{\"legendOpen\":false}}}",
+      "version": 1,
+      "timeRestore": true,
+      "timeTo": "now",
+      "timeFrom": "now-1y",
+      "refreshInterval": {
+        "display": "Off",
+        "pause": false,
+        "value": 0
+      },
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"filter\":[{\"query\":{\"match_all\":{}}}],\"highlightAll\":true,\"version\":true}"
+      }
+    }
+  },
+  {
+    "_id": "AWCUqesWib22Ai8JwW3u",
+    "_type": "dashboard",
+    "_source": {
+      "title": "VulnWhisperer - Risk Mitigation",
+      "hits": 0,
+      "description": "",
+      "panelsJSON": "[{\"col\":11,\"id\":\"995e2280-3df3-11e7-a44e-c79ca8efb780\",\"panelIndex\":20,\"row\":8,\"size_x\":2,\"size_y\":6,\"type\":\"visualization\"},{\"col\":1,\"id\":\"852816e0-3eb1-11e7-90cb-918f9cb01e3d\",\"panelIndex\":21,\"row\":10,\"size_x\":3,\"size_y\":5,\"type\":\"visualization\"},{\"col\":4,\"id\":\"297df800-3f7e-11e7-bd24-6903e3283192\",\"panelIndex\":27,\"row\":8,\"size_x\":3,\"size_y\":5,\"type\":\"visualization\"},{\"col\":9,\"id\":\"35b6d320-3f7f-11e7-bd24-6903e3283192\",\"panelIndex\":28,\"row\":8,\"size_x\":2,\"size_y\":6,\"type\":\"visualization\"},{\"col\":11,\"id\":\"471a3580-3f6b-11e7-88e7-df1abe6547fb\",\"panelIndex\":30,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"de1a5f40-3f85-11e7-97f9-3777d794626d\",\"panelIndex\":31,\"row\":8,\"size_x\":2,\"size_y\":5,\"type\":\"visualization\"},{\"col\":10,\"id\":\"5093c620-44e9-11e7-8014-ede06a7e69f8\",\"panelIndex\":37,\"row\":4,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"columns\":[\"host\",\"risk\",\"risk_score\",\"cve\",\"plugin_name\",\"solution\",\"plugin_output\"],\"id\":\"54648700-3f74-11e7-852e-69207a3d0726\",\"panelIndex\":38,\"row\":15,\"size_x\":12,\"size_y\":6,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"fb6eb020-49ab-11e7-8f8c-57ad64ec48a6\",\"panelIndex\":39,\"row\":8,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":5,\"id\":\"465c5820-8977-11e7-857e-e1d56b17746d\",\"panelIndex\":40,\"row\":4,\"size_x\":5,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"56f0f5f0-3ebe-11e7-a192-93f36fbd9d05\",\"panelIndex\":46,\"row\":4,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"id\":\"e46ff7f0-897d-11e7-934b-67cec0a7da65\",\"panelIndex\":47,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":3,\"id\":\"c1361da0-80b3-11e7-8790-73b60225f736\",\"panelIndex\":48,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"d048c220-80b3-11e7-8790-73b60225f736\",\"panelIndex\":49,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"db55bce0-80b3-11e7-8790-73b60225f736\",\"panelIndex\":50,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":9,\"id\":\"b2f2adb0-897f-11e7-a2d2-c57bca21b3aa\",\"panelIndex\":51,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"}]",
+      "optionsJSON": "{\"darkTheme\":false}",
+      "uiStateJSON": "{\"P-11\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-2\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-20\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-21\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}},\"P-27\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-28\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}},\"P-3\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"asc\"}}}},\"P-30\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-31\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-40\":{\"vis\":{\"defaultColors\":{\"0 - 3\":\"rgb(0,104,55)\",\"3 - 7\":\"rgb(135,203,103)\",\"7 - 9\":\"rgb(255,255,190)\",\"9 - 11\":\"rgb(249,142,82)\"}}},\"P-41\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-42\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-43\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"}}},\"P-44\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-45\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-46\":{\"vis\":{\"legendOpen\":true}},\"P-47\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-48\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-49\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-5\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-50\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-51\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-6\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-8\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}",
+      "version": 1,
+      "timeRestore": false,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"filter\":[{\"query\":{\"match_all\":{}}}],\"highlightAll\":true,\"version\":true}"
+      }
+    }
+  }
+]

resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/2000_vulnWhisperer_QualysVisuals (required with Dashboard).json

@@ -0,0 +1,170 @@
+[
+  {
+    "_id": "AWCUo-jRib22Ai8JwW1N",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer  - Risk: High Qualys Scoring",
+      "visState": "{\"title\":\"VulnWhisperer  - Risk: High Qualys Scoring\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Background\",\"colorsRange\":[{\"from\":0,\"to\":1000}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"bgFill\":\"white\",\"bgColor\":true,\"labelColor\":false,\"subText\":\"\",\"fontSize\":\"34\"},\"extendRange\":true}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"High Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:high\"}}},\"label\":\"\"}]}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 10000\":\"#EF843C\",\"0 - 1000\":\"#E0752D\"}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "AWCUozGBib22Ai8JwW1B",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer  - Risk: Medium Qualys Scoring",
+      "visState": "{\"title\":\"VulnWhisperer  - Risk: Medium Qualys Scoring\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Background\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"bgFill\":\"white\",\"bgColor\":true,\"labelColor\":false,\"subText\":\"\",\"fontSize\":\"34\"},\"extendRange\":false}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Medium Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:medium\"}}},\"label\":\"Medium Risk\"}]}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 10000\":\"#EAB839\"}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "AWCUpE3Kib22Ai8JwW1c",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer  - Risk: Critical Qualys Scoring",
+      "visState": "{\"title\":\"VulnWhisperer  - Risk: Critical Qualys Scoring\",\"type\":\"goal\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"gauge\":{\"autoExtend\":false,\"backStyle\":\"Full\",\"colorSchema\":\"Green to Red\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"gaugeColorMode\":\"Background\",\"gaugeStyle\":\"Full\",\"gaugeType\":\"Metric\",\"invertColors\":false,\"labels\":{\"color\":\"black\",\"show\":false},\"orientation\":\"vertical\",\"percentageMode\":false,\"scale\":{\"color\":\"#333\",\"labels\":false,\"show\":true,\"width\":2},\"style\":{\"bgColor\":true,\"bgFill\":\"white\",\"fontSize\":\"34\",\"labelColor\":false,\"subText\":\"Risk\"},\"type\":\"simple\",\"useRanges\":false,\"verticalSplit\":false},\"type\":\"gauge\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Critical Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:critical\"}}},\"label\":\"Critical\"}]}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"colors\":{\"0 - 10000\":\"#BF1B00\"},\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "AWCUyeHGib22Ai8JwX62",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer-RiskOverTime Qualys Scoring",
+      "visState": "{\"title\":\"VulnWhisperer-RiskOverTime Qualys Scoring\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":false,\"addTooltip\":true,\"categoryAxes\":[{\"id\":\"CategoryAxis-1\",\"labels\":{\"show\":true,\"truncate\":100},\"position\":\"bottom\",\"scale\":{\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"@timestamp per 12 hours\"},\"type\":\"category\"}],\"defaultYExtents\":false,\"drawLinesBetweenPoints\":true,\"grid\":{\"categoryLines\":false,\"style\":{\"color\":\"#eee\"},\"valueAxis\":\"ValueAxis-1\"},\"interpolate\":\"linear\",\"legendPosition\":\"right\",\"orderBucketsBySum\":false,\"radiusRatio\":9,\"scale\":\"linear\",\"seriesParams\":[{\"data\":{\"id\":\"1\",\"label\":\"Count\"},\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"mode\":\"normal\",\"show\":\"true\",\"showCircles\":true,\"type\":\"line\",\"valueAxis\":\"ValueAxis-1\"}],\"setYExtents\":false,\"showCircles\":true,\"times\":[],\"valueAxes\":[{\"id\":\"ValueAxis-1\",\"labels\":{\"filter\":false,\"rotate\":0,\"show\":true,\"truncate\":100},\"name\":\"LeftAxis-1\",\"position\":\"left\",\"scale\":{\"mode\":\"normal\",\"type\":\"linear\"},\"show\":true,\"style\":{},\"title\":{\"text\":\"Count\"},\"type\":\"value\"}],\"type\":\"line\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"@timestamp\",\"interval\":\"auto\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:info\"}}},\"label\":\"Info\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:low\"}}},\"label\":\"Low\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:medium\"}}},\"label\":\"Medium\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:high\"}}},\"label\":\"High\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:critical\"}}},\"label\":\"Critical\"}]}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"colors\":{\"Critical\":\"#962D82\",\"High\":\"#BF1B00\",\"Low\":\"#629E51\",\"Medium\":\"#EAB839\",\"Info\":\"#65C5DB\"}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "AWCUos-Fib22Ai8JwW0y",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer  - Risk: Low Qualys Scoring",
+      "visState": "{\"title\":\"VulnWhisperer  - Risk: Low Qualys Scoring\",\"type\":\"goal\",\"params\":{\"addTooltip\":true,\"addLegend\":true,\"type\":\"gauge\",\"gauge\":{\"verticalSplit\":false,\"autoExtend\":false,\"percentageMode\":false,\"gaugeType\":\"Metric\",\"gaugeStyle\":\"Full\",\"backStyle\":\"Full\",\"orientation\":\"vertical\",\"useRanges\":false,\"colorSchema\":\"Green to Red\",\"gaugeColorMode\":\"Background\",\"colorsRange\":[{\"from\":0,\"to\":10000}],\"invertColors\":false,\"labels\":{\"show\":false,\"color\":\"black\"},\"scale\":{\"show\":true,\"labels\":false,\"color\":\"#333\",\"width\":2},\"type\":\"simple\",\"style\":{\"bgFill\":\"white\",\"bgColor\":true,\"labelColor\":false,\"subText\":\"\",\"fontSize\":\"34\"},\"extendRange\":false}},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{\"customLabel\":\"Low Risk\"}},{\"id\":\"2\",\"enabled\":true,\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"risk:low\"}}},\"label\":\"Low Risk\"}]}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":true,\"colors\":{\"0 - 10000\":\"#629E51\"}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "AWCg9Wsfib22Ai8Jww3v",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - Qualys: Category Description",
+      "visState": "{\"title\":\"VulnWhisperer - Qualys: Category Description\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"category_description.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Category Description\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"match_all\":{}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "AWCg88f1ib22Ai8Jww3C",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - QualysOS",
+      "visState": "{\"title\":\"VulnWhisperer - QualysOS\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"operating_system.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"match_all\":{}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "AWCg9JUAib22Ai8Jww3Y",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - QualysOwner",
+      "visState": "{\"title\":\"VulnWhisperer - QualysOwner\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"owner.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"match_all\":{}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "AWCg9tE6ib22Ai8Jww4R",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - Qualys: Impact",
+      "visState": "{\"title\":\"VulnWhisperer - Qualys: Impact\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"impact.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Impact\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"match_all\":{}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "AWCg9igvib22Ai8Jww36",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - Qualys: Level",
+      "visState": "{\"title\":\"VulnWhisperer - Qualys: Level\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"showTotal\":false,\"totalFunc\":\"sum\",\"type\":\"table\"},\"aggs\":[{\"id\":\"1\",\"enabled\":true,\"type\":\"count\",\"schema\":\"metric\",\"params\":{}},{\"id\":\"2\",\"enabled\":true,\"type\":\"terms\",\"schema\":\"bucket\",\"params\":{\"field\":\"level.keyword\",\"size\":20,\"order\":\"desc\",\"orderBy\":\"1\",\"customLabel\":\"Level\"}}],\"listeners\":{}}",
+      "uiStateJSON": "{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"match_all\":{}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "AWCUsp_3ib22Ai8JwW7R",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - TL-Critical Risk Qualys Scoring",
+      "visState": "{\"title\":\"VulnWhisperer - TL-Critical Risk Qualys Scoring\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*',q='(risk:critical)').label(\\\"Original\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk:critical)',offset=-1w).label(\\\"One week offset\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk:critical)').subtract(.es(index='logstash-vulnwhisperer-*',q='(risk:critical)',offset=-1w)).label(\\\"Difference\\\").lines(steps=3,fill=2,width=1)\",\"interval\":\"auto\",\"type\":\"timelion\"},\"aggs\":[],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  },
+  {
+    "_id": "AWCUtHETib22Ai8JwW79",
+    "_type": "visualization",
+    "_source": {
+      "title": "VulnWhisperer - TL-High Risk Qualys Scoring",
+      "visState": "{\"title\":\"VulnWhisperer - TL-High Risk Qualys Scoring\",\"type\":\"timelion\",\"params\":{\"expression\":\".es(index='logstash-vulnwhisperer-*',q='(risk:high)').label(\\\"Original\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk:high)',offset=-1w).label(\\\"One week offset\\\"),.es(index='logstash-vulnwhisperer-*',q='(risk:high)').subtract(.es(index='logstash-vulnwhisperer-*',q='(risk:high)',offset=-1w)).label(\\\"Difference\\\").lines(steps=3,fill=2,width=1)\",\"interval\":\"auto\",\"type\":\"timelion\"},\"aggs\":[],\"listeners\":{}}",
+      "uiStateJSON": "{}",
+      "description": "",
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}"
+      }
+    }
+  }
+]

resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/2001_vulnWhisperer_ReportingMitigationDashboardQualysRisk.json

@@ -0,0 +1,50 @@
+[
+  {
+    "_id": "AWCUrIBqib22Ai8JwW43",
+    "_type": "dashboard",
+    "_source": {
+      "title": "VulnWhisperer - Reporting Qualys Scoring",
+      "hits": 0,
+      "description": "",
+      "panelsJSON": "[{\"col\":1,\"id\":\"2f979030-44b9-11e7-a818-f5f80dfc3590\",\"panelIndex\":5,\"row\":11,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"col\":10,\"id\":\"297df800-3f7e-11e7-bd24-6903e3283192\",\"panelIndex\":15,\"row\":7,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":7,\"id\":\"471a3580-3f6b-11e7-88e7-df1abe6547fb\",\"panelIndex\":20,\"row\":7,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":11,\"id\":\"995e2280-3df3-11e7-a44e-c79ca8efb780\",\"panelIndex\":22,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":9,\"id\":\"b2f2adb0-897f-11e7-a2d2-c57bca21b3aa\",\"panelIndex\":23,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"479deab0-8a39-11e7-a58a-9bfcb3761a3d\",\"panelIndex\":29,\"row\":4,\"size_x\":6,\"size_y\":4,\"type\":\"visualization\"},{\"size_x\":6,\"size_y\":3,\"panelIndex\":30,\"type\":\"visualization\",\"id\":\"AWCUtHETib22Ai8JwW79\",\"col\":1,\"row\":8},{\"size_x\":6,\"size_y\":3,\"panelIndex\":31,\"type\":\"visualization\",\"id\":\"AWCUsp_3ib22Ai8JwW7R\",\"col\":7,\"row\":4},{\"size_x\":2,\"size_y\":3,\"panelIndex\":33,\"type\":\"visualization\",\"id\":\"AWCUozGBib22Ai8JwW1B\",\"col\":3,\"row\":1},{\"size_x\":2,\"size_y\":3,\"panelIndex\":34,\"type\":\"visualization\",\"id\":\"AWCUo-jRib22Ai8JwW1N\",\"col\":5,\"row\":1},{\"size_x\":2,\"size_y\":3,\"panelIndex\":35,\"type\":\"visualization\",\"id\":\"AWCUpE3Kib22Ai8JwW1c\",\"col\":7,\"row\":1},{\"size_x\":2,\"size_y\":3,\"panelIndex\":36,\"type\":\"visualization\",\"id\":\"AWCUos-Fib22Ai8JwW0y\",\"col\":1,\"row\":1}]",
+      "optionsJSON": "{\"darkTheme\":false}",
+      "uiStateJSON": "{\"P-15\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-20\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-21\":{\"vis\":{\"defaultColors\":{\"0 - 100\":\"rgb(0,104,55)\"}}},\"P-22\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-23\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-24\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-5\":{\"vis\":{\"legendOpen\":false}},\"P-33\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-34\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-35\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-27\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-28\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-26\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"}}},\"P-25\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-32\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-36\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}}}",
+      "version": 1,
+      "timeRestore": true,
+      "timeTo": "now",
+      "timeFrom": "now-30d",
+      "refreshInterval": {
+        "display": "Off",
+        "pause": false,
+        "value": 0
+      },
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"-vulnerability_category:\\\"INFORMATION_GATHERED\\\"\"}}}],\"highlightAll\":true,\"version\":true}"
+      }
+    }
+  },
+  {
+    "_id": "5dba30c0-3df3-11e7-a44e-c79ca8efb780",
+    "_type": "dashboard",
+    "_source": {
+      "title": "VulnWhisperer - Risk Mitigation Qualys Web Scoring",
+      "hits": 0,
+      "description": "",
+      "panelsJSON": "[{\"col\":11,\"id\":\"995e2280-3df3-11e7-a44e-c79ca8efb780\",\"panelIndex\":20,\"row\":8,\"size_x\":2,\"size_y\":7,\"type\":\"visualization\"},{\"col\":1,\"id\":\"852816e0-3eb1-11e7-90cb-918f9cb01e3d\",\"panelIndex\":21,\"row\":10,\"size_x\":3,\"size_y\":5,\"type\":\"visualization\"},{\"col\":4,\"id\":\"297df800-3f7e-11e7-bd24-6903e3283192\",\"panelIndex\":27,\"row\":8,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":9,\"id\":\"35b6d320-3f7f-11e7-bd24-6903e3283192\",\"panelIndex\":28,\"row\":8,\"size_x\":2,\"size_y\":7,\"type\":\"visualization\"},{\"col\":11,\"id\":\"471a3580-3f6b-11e7-88e7-df1abe6547fb\",\"panelIndex\":30,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"de1a5f40-3f85-11e7-97f9-3777d794626d\",\"panelIndex\":31,\"row\":8,\"size_x\":2,\"size_y\":4,\"type\":\"visualization\"},{\"col\":10,\"id\":\"5093c620-44e9-11e7-8014-ede06a7e69f8\",\"panelIndex\":37,\"row\":4,\"size_x\":3,\"size_y\":4,\"type\":\"visualization\"},{\"col\":1,\"columns\":[\"host\",\"risk\",\"risk_score\",\"cve\",\"plugin_name\",\"solution\",\"plugin_output\"],\"id\":\"54648700-3f74-11e7-852e-69207a3d0726\",\"panelIndex\":38,\"row\":15,\"size_x\":12,\"size_y\":6,\"sort\":[\"@timestamp\",\"desc\"],\"type\":\"search\"},{\"col\":1,\"id\":\"fb6eb020-49ab-11e7-8f8c-57ad64ec48a6\",\"panelIndex\":39,\"row\":8,\"size_x\":3,\"size_y\":2,\"type\":\"visualization\"},{\"col\":5,\"id\":\"465c5820-8977-11e7-857e-e1d56b17746d\",\"panelIndex\":40,\"row\":4,\"size_x\":5,\"size_y\":4,\"type\":\"visualization\"},{\"col\":9,\"id\":\"b2f2adb0-897f-11e7-a2d2-c57bca21b3aa\",\"panelIndex\":45,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AWCUos-Fib22Ai8JwW0y\",\"panelIndex\":47,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":3,\"id\":\"AWCUozGBib22Ai8JwW1B\",\"panelIndex\":48,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"AWCUo-jRib22Ai8JwW1N\",\"panelIndex\":49,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"AWCUpE3Kib22Ai8JwW1c\",\"panelIndex\":50,\"row\":1,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"AWCUyeHGib22Ai8JwX62\",\"panelIndex\":51,\"row\":4,\"size_x\":4,\"size_y\":4,\"type\":\"visualization\"},{\"col\":4,\"id\":\"AWCg88f1ib22Ai8Jww3C\",\"panelIndex\":52,\"row\":12,\"size_x\":3,\"size_y\":3,\"type\":\"visualization\"},{\"col\":7,\"id\":\"AWCg9JUAib22Ai8Jww3Y\",\"panelIndex\":53,\"row\":12,\"size_x\":2,\"size_y\":3,\"type\":\"visualization\"}]",
+      "optionsJSON": "{\"darkTheme\":false}",
+      "uiStateJSON": "{\"P-11\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-2\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-20\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-21\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}},\"P-27\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-28\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"desc\"}}}},\"P-3\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":0,\"direction\":\"asc\"}}}},\"P-30\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-31\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-40\":{\"vis\":{\"defaultColors\":{\"0 - 3\":\"rgb(0,104,55)\",\"3 - 7\":\"rgb(135,203,103)\",\"7 - 9\":\"rgb(255,255,190)\",\"9 - 11\":\"rgb(249,142,82)\"}}},\"P-41\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-42\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-43\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"}}},\"P-44\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-45\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-47\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-48\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-49\":{\"vis\":{\"defaultColors\":{\"0 - 1000\":\"rgb(0,104,55)\"},\"legendOpen\":false}},\"P-5\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-50\":{\"vis\":{\"defaultColors\":{\"0 - 10000\":\"rgb(0,104,55)\"}}},\"P-6\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-8\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-52\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}},\"P-53\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}}}}",
+      "version": 1,
+      "timeRestore": true,
+      "timeTo": "now",
+      "timeFrom": "now-30d",
+      "refreshInterval": {
+        "display": "Off",
+        "pause": false,
+        "value": 0
+      },
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"-vulnerability_category:\\\"INFORMATION_GATHERED\\\"\"}}}],\"highlightAll\":true,\"version\":true}"
+      }
+    }
+  }
+]

resources/elk5-old_compatibility/kibana/vuln_whisp_kibana/9000_vulnWhisperer_SavedSearch.json

@@ -0,0 +1,28 @@
+[
+  {
+    "_id": "54648700-3f74-11e7-852e-69207a3d0726",
+    "_type": "search",
+    "_source": {
+      "title": "VulnWhisperer - Saved Search",
+      "description": "",
+      "hits": 0,
+      "columns": [
+        "host",
+        "risk",
+        "risk_score",
+        "cve",
+        "plugin_name",
+        "solution",
+        "plugin_output"
+      ],
+      "sort": [
+        "@timestamp",
+        "desc"
+      ],
+      "version": 1,
+      "kibanaSavedObjectMeta": {
+        "searchSourceJSON": "{\"index\":\"logstash-vulnwhisperer-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[],\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fields\":{\"*\":{}},\"require_field_match\":false,\"fragment_size\":2147483647}}"
+      }
+    }
+  }
+]

resources/elk5-old_compatibility/logstash/0001_input_beats.conf

@@ -0,0 +1,14 @@
+input {
+  beats {
+    port => 5044
+    tags => "beats"
+  }
+}
+
+filter {
+  if [beat][hostname] == "filebeathost" {
+    mutate {
+      add_tag => ["nessus"]
+    }
+  }
+}

resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf

@@ -0,0 +1,220 @@
+# Author: Austin Taylor and Justin Henderson
+# Email: email@austintaylor.io
+# Last Update: 12/20/2017
+# Version 0.3
+# Description: Take in nessus reports from vulnWhisperer and pumps into logstash
+
+
+input {
+  file {
+    path => "/opt/VulnWhisperer/nessus/**/*"
+    start_position => "beginning"
+    tags => "nessus"
+    type => "nessus"
+  }
+  file {
+    path => "/opt/VulnWhisperer/tenable/*.csv"
+    start_position => "beginning"
+    tags => "tenable"
+    type => "tenable"
+  }
+}
+
+filter {
+  if "nessus" in [tags] or "tenable" in [tags] {
+    # Drop the header column
+    if [message] =~ "^Plugin ID" { drop {} }
+
+    csv {
+      # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"]
+      columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"]
+      separator => ","
+      source => "message"
+    }
+
+    ruby {
+        code => "if event.get('description')
+                    event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
+                end
+                 if event.get('synopsis')
+                    event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
+                end
+                 if event.get('solution')
+                    event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
+                end
+                 if event.get('see_also')
+                    event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
+                end
+                 if event.get('plugin_output')
+                    event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
+                end"
+    }
+
+    #If using filebeats as your source, you will need to replace the "path" field to "source"
+    grok {
+      match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.csv$" }
+      tag_on_failure => []
+    }
+
+    date {
+      match => [ "last_updated", "UNIX" ]
+      target => "@timestamp"
+      remove_field => ["last_updated"]
+    }
+
+    if [risk] == "None" {
+      mutate { add_field => { "risk_number" => 0 }}
+    }
+    if [risk] == "Low" {
+      mutate { add_field => { "risk_number" => 1 }}
+    }
+    if [risk] == "Medium" {
+      mutate { add_field => { "risk_number" => 2 }}
+    }
+    if [risk] == "High" {
+      mutate { add_field => { "risk_number" => 3 }}
+    }
+    if [risk] == "Critical" {
+      mutate { add_field => { "risk_number" => 4 }}
+    }
+
+    if ![cve] or [cve] == "nan" {
+      mutate { remove_field => [ "cve" ] }
+    }
+    if ![cvss] or [cvss] == "nan" {
+      mutate { remove_field => [ "cvss" ] }
+    }
+    if ![cvss_base] or [cvss_base] == "nan" {
+      mutate { remove_field => [ "cvss_base" ] }
+    }
+    if ![cvss_temporal] or [cvss_temporal] == "nan" {
+      mutate { remove_field => [ "cvss_temporal" ] }
+    }
+    if ![cvss_temporal_vector] or [cvss_temporal_vector] == "nan" {
+      mutate { remove_field => [ "cvss_temporal_vector" ] }
+    }
+    if ![cvss_vector] or [cvss_vector] == "nan" {
+      mutate { remove_field => [ "cvss_vector" ] }
+    }
+    if ![cvss3_base] or [cvss3_base] == "nan" {
+      mutate { remove_field => [ "cvss3_base" ] }
+    }
+    if ![cvss3_temporal] or [cvss3_temporal] == "nan" {
+      mutate { remove_field => [ "cvss3_temporal" ] }
+    }
+    if ![cvss3_temporal_vector] or [cvss3_temporal_vector] == "nan" {
+      mutate { remove_field => [ "cvss3_temporal_vector" ] }
+    }
+    if ![description] or [description] == "nan" {
+      mutate { remove_field => [ "description" ] }
+    }
+    if ![mac_address] or [mac_address] == "nan" {
+      mutate { remove_field => [ "mac_address" ] }
+    }
+    if ![netbios] or [netbios] == "nan" {
+      mutate { remove_field => [ "netbios" ] }
+    }
+    if ![operating_system] or [operating_system] == "nan" {
+      mutate { remove_field => [ "operating_system" ] }
+    }
+    if ![plugin_output] or [plugin_output] == "nan" {
+      mutate { remove_field => [ "plugin_output" ] }
+    }
+    if ![see_also] or [see_also] == "nan" {
+      mutate { remove_field => [ "see_also" ] }
+    }
+    if ![synopsis] or [synopsis] == "nan" {
+      mutate { remove_field => [ "synopsis" ] }
+    }
+    if ![system_type] or [system_type] == "nan" {
+      mutate { remove_field => [ "system_type" ] }
+    }
+
+    mutate {
+      remove_field => [ "message" ]
+      add_field => { "risk_score" => "%{cvss}" }
+    }
+    mutate {
+      convert => { "risk_score" => "float" }
+    }
+    if [risk_score] == 0 {
+      mutate {
+        add_field => { "risk_score_name" => "info" }
+      }
+    }
+    if [risk_score] > 0 and [risk_score] < 3 {
+      mutate {
+        add_field => { "risk_score_name" => "low" }
+      }
+    }
+    if [risk_score] >= 3 and [risk_score] < 6 {
+      mutate {
+        add_field => { "risk_score_name" => "medium" }
+      }
+    }
+    if [risk_score] >=6 and [risk_score] < 9 {
+      mutate {
+        add_field => { "risk_score_name" => "high" }
+      }
+    }
+    if [risk_score] >= 9 {
+      mutate {
+        add_field => { "risk_score_name" => "critical" }
+      }
+    }
+
+    # Compensating controls - adjust risk_score
+    # Adobe and Java are not allowed to run in browser unless whitelisted
+    # Therefore, lower score by dividing by 3 (score is subjective to risk)
+
+    #Modify and uncomment when ready to use
+    #if [risk_score] != 0 {
+    #  if [plugin_name] =~ "Adobe" and [risk_score] > 6 or [plugin_name] =~ "Java" and [risk_score] > 6 {
+    #    ruby {
+    #      code => "event.set('risk_score', event.get('risk_score') / 3)"
+    #    }
+    #    mutate {
+    #      add_field => { "compensating_control" => "Adobe and Flash removed from browsers unless whitelisted site." }
+    #    }
+    #  }
+    #}
+
+    # Add tags for reporting based on assets or criticality
+
+    if [asset] == "dc01" or [asset] == "dc02" or [asset] == "pki01" or [asset] == "192.168.0.54" or [asset] =~ "^192\.168\.0\." or [asset] =~ "^42.42.42." {
+      mutate {
+        add_tag => [ "critical_asset" ]
+      }
+    }
+    #if [asset] =~ "^192\.168\.[45][0-9][0-9]\.1$" or [asset] =~ "^192.168\.[50]\.[0-9]{1,2}\.1$"{
+    #  mutate {
+    #    add_tag => [ "has_hipaa_data" ]
+    #  }
+    #}
+    #if [asset] =~ "^192\.168\.[45][0-9][0-9]\." {
+    #  mutate {
+    #    add_tag => [ "hipaa_asset" ]
+    #  }
+    #}
+    if [asset] =~ "^hr" {
+      mutate {
+        add_tag => [ "pci_asset" ]
+      }
+    }
+    #if [asset] =~ "^10\.0\.50\." {
+    #  mutate {
+    #    add_tag => [ "web_servers" ]
+    #  }
+    #}
+  }
+}
+
+output {
+  if "nessus" in [tags] or "tenable" in [tags] or [type] in [ "nessus", "tenable" ] {
+    # stdout { codec => rubydebug }
+    elasticsearch {
+      hosts => [ "localhost:9200" ]
+      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+    }
+  }
+}

resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf

@@ -0,0 +1,153 @@
+# Author: Austin Taylor and Justin Henderson
+# Email: austin@hasecuritysolutions.com
+# Last Update: 12/30/2017
+# Version 0.3
+# Description: Take in qualys web scan reports from vulnWhisperer and pumps into logstash
+
+input {
+  file {
+    path => [ "/opt/VulnWhisperer/data/qualys/*.json" , "/opt/VulnWhisperer/data/qualys_web/*.json", "/opt/VulnWhisperer/data/qualys_vuln/*.json" ]
+    type => json
+    codec => json
+    start_position => "beginning"
+    tags => [ "qualys" ]
+  }
+}
+
+filter {
+  if "qualys" in [tags] {
+    grok {
+      match => { "path" => [ "(?<tags>qualys_vuln)_scan_%{DATA}_%{INT:last_updated}.json$", "(?<tags>qualys_web)_%{INT:app_id}_%{INT:last_updated}.json$" ] }
+      tag_on_failure => []
+    }
+    
+    mutate {
+      replace => [ "message", "%{message}" ]
+      #gsub => [
+      #  "message", "\|\|\|", " ",
+      #  "message", "\t\t", " ",
+      #  "message", "    ", " ",
+      #  "message", "   ", " ",
+      #  "message", "  ", " ",
+      #  "message", "nan", " ",
+      #  "message",'\n',''
+      #]
+    }
+
+    if "qualys_web" in [tags] {
+    mutate {
+      add_field => { "asset" => "%{web_application_name}" }
+      add_field => { "risk_score" => "%{cvss}" }
+      }
+    } else if "qualys_vuln" in [tags] {
+      mutate {
+        add_field => { "asset" => "%{ip}" }
+        add_field => { "risk_score" => "%{cvss}" }
+      }
+    }
+
+    if [risk] == "1" {
+      mutate { add_field => { "risk_number" => 0 }}
+      mutate { replace => { "risk" => "info" }}
+    }
+    if [risk] == "2" {
+      mutate { add_field => { "risk_number" => 1 }}
+      mutate { replace => { "risk" => "low" }}
+    }
+    if [risk] == "3" {
+      mutate { add_field => { "risk_number" => 2 }}
+      mutate { replace => { "risk" => "medium" }}
+    }
+    if [risk] == "4" {
+      mutate { add_field => { "risk_number" => 3 }}
+      mutate { replace => { "risk" => "high" }}
+    }
+    if [risk] == "5" {
+      mutate { add_field => { "risk_number" => 4 }}
+      mutate { replace => { "risk" => "critical" }}
+    }
+
+    mutate {
+      remove_field => "message"
+    }
+
+    if [first_time_detected] {
+      date {
+        match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "first_time_detected"
+      }
+    }
+    if [first_time_tested] {
+      date {
+        match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "first_time_tested"
+      }
+    }
+    if [last_time_detected] {
+      date {
+        match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "last_time_detected"
+      }
+    }
+    if [last_time_tested] {
+      date {
+        match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "last_time_tested"
+      }
+    }
+    date {
+      match => [ "last_updated", "UNIX" ]
+      target => "@timestamp"
+      remove_field => "last_updated"
+    }
+    mutate {
+      convert => { "plugin_id" => "integer"}
+      convert => { "id" => "integer"}
+      convert => { "risk_number" => "integer"}
+      convert => { "risk_score" => "float"}
+      convert => { "total_times_detected" => "integer"}
+      convert => { "cvss_temporal" => "float"}
+      convert => { "cvss" => "float"}
+    }
+    if [risk_score] == 0 {
+      mutate {
+        add_field => { "risk_score_name" => "info" }
+      }
+    }
+    if [risk_score] > 0 and [risk_score] < 3 {
+      mutate {
+        add_field => { "risk_score_name" => "low" }
+      }
+    }
+    if [risk_score] >= 3 and [risk_score] < 6 {
+      mutate {
+        add_field => { "risk_score_name" => "medium" }
+      }
+    }
+    if [risk_score] >=6 and [risk_score] < 9 {
+      mutate {
+        add_field => { "risk_score_name" => "high" }
+      }
+    }
+    if [risk_score] >= 9 {
+      mutate {
+        add_field => { "risk_score_name" => "critical" }
+      }
+    }
+
+    if [asset] =~ "\.yourdomain\.(com|net)$" {
+      mutate {
+        add_tag => [ "critical_asset" ]
+      }
+    }
+  }
+}
+output {
+  if "qualys" in [tags] {
+    stdout { codec => rubydebug }
+    elasticsearch {
+      hosts => [ "localhost:9200" ]
+      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+    }
+  }
+}

resources/elk5-old_compatibility/logstash/3000_openvas.conf

@@ -0,0 +1,146 @@
+# Author: Austin Taylor and Justin Henderson
+# Email: austin@hasecuritysolutions.com
+# Last Update: 03/04/2018
+# Version 0.3
+# Description: Take in qualys web scan reports from vulnWhisperer and pumps into logstash
+
+input {
+  file {
+    path => "/opt/VulnWhisperer/openvas/*.json"
+    type => json
+    codec => json
+    start_position => "beginning"
+    tags => [ "openvas_scan", "openvas" ]
+  }
+}
+
+filter {
+  if "openvas_scan" in [tags] {
+    mutate {
+      replace => [ "message", "%{message}" ]
+      gsub => [
+        "message", "\|\|\|", " ",
+        "message", "\t\t", " ",
+        "message", "    ", " ",
+        "message", "   ", " ",
+        "message", "  ", " ",
+        "message", "nan", " ",
+        "message",'\n',''
+      ]
+    }
+
+
+    grok {
+        match => { "path" => "openvas_scan_%{DATA:scan_id}_%{INT:last_updated}.json$" }
+        tag_on_failure => []
+    }
+
+    mutate {
+      add_field => { "risk_score" => "%{cvss}" }
+    }
+
+    if [risk] == "1" {
+      mutate { add_field => { "risk_number" => 0 }}
+      mutate { replace => { "risk" => "info" }}
+    }
+    if [risk] == "2" {
+      mutate { add_field => { "risk_number" => 1 }}
+      mutate { replace => { "risk" => "low" }}
+    }
+    if [risk] == "3" {
+      mutate { add_field => { "risk_number" => 2 }}
+      mutate { replace => { "risk" => "medium" }}
+    }
+    if [risk] == "4" {
+      mutate { add_field => { "risk_number" => 3 }}
+      mutate { replace => { "risk" => "high" }}
+    }
+    if [risk] == "5" {
+      mutate { add_field => { "risk_number" => 4 }}
+      mutate { replace => { "risk" => "critical" }}
+    }
+
+    mutate {
+      remove_field => "message"
+    }
+
+    if [first_time_detected] {
+      date {
+        match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "first_time_detected"
+      }
+    }
+    if [first_time_tested] {
+      date {
+        match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "first_time_tested"
+      }
+    }
+    if [last_time_detected] {
+      date {
+        match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "last_time_detected"
+      }
+    }
+    if [last_time_tested] {
+      date {
+        match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "last_time_tested"
+      }
+    }
+    date {
+      match => [ "last_updated", "UNIX" ]
+      target => "@timestamp"
+      remove_field => "last_updated"
+    }
+    mutate {
+      convert => { "plugin_id" => "integer"}
+      convert => { "id" => "integer"}
+      convert => { "risk_number" => "integer"}
+      convert => { "risk_score" => "float"}
+      convert => { "total_times_detected" => "integer"}
+      convert => { "cvss_temporal" => "float"}
+      convert => { "cvss" => "float"}
+    }
+    if [risk_score] == 0 {
+      mutate {
+        add_field => { "risk_score_name" => "info" }
+      }
+    }
+    if [risk_score] > 0 and [risk_score] < 3 {
+      mutate {
+        add_field => { "risk_score_name" => "low" }
+      }
+    }
+    if [risk_score] >= 3 and [risk_score] < 6 {
+      mutate {
+        add_field => { "risk_score_name" => "medium" }
+      }
+    }
+    if [risk_score] >=6 and [risk_score] < 9 {
+      mutate {
+        add_field => { "risk_score_name" => "high" }
+      }
+    }
+    if [risk_score] >= 9 {
+      mutate {
+        add_field => { "risk_score_name" => "critical" }
+      }
+    }
+    # Add your critical assets by subnet or by hostname. Comment this field out if you don't want to tag any, but the asset panel will break.
+    if [asset] =~ "^10\.0\.100\." {
+      mutate {
+        add_tag => [ "critical_asset" ]
+      }
+    }
+  }
+}
+output {
+  if "openvas" in [tags] {
+    stdout { codec => rubydebug }
+    elasticsearch {
+      hosts => [ "localhost:9200" ]
+      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+    }
+  }
+}

resources/elk5-old_compatibility/logstash/4000_jira.conf

@@ -0,0 +1,21 @@
+# Description: Take in jira tickets from vulnWhisperer and pumps into logstash
+
+input {
+  file {
+    path => "/opt/VulnWhisperer/jira/*.json"
+    type => json
+    codec => json
+    start_position => "beginning"
+    tags => [ "jira" ]
+  }
+}
+
+output {
+  if "jira" in [tags] {
+    stdout { codec => rubydebug }
+    elasticsearch {
+      hosts => [ "localhost:9200" ]
+      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+    }
+  }
+}

resources/elk5-old_compatibility/logstash/9998_input_broker_rabbitmq.conf

@@ -0,0 +1,13 @@
+input {
+  rabbitmq {
+    key => "nessus"
+    queue => "nessus"
+    durable => true
+    exchange => "nessus"
+    user => "logstash"    
+    password => "yourpassword"
+    host => "buffer01"
+    port => 5672
+    tags => [ "queue_nessus", "rabbitmq" ]
+  }
+}

resources/elk5-old_compatibility/logstash/9998_output_broker_rabbitmq.conf

@@ -0,0 +1,16 @@
+output {
+  if "nessus" in [tags]{
+    rabbitmq {
+      key => "nessus"
+      exchange => "nessus"
+      exchange_type => "direct"
+      user => "logstash"
+      password => "yourbufferpassword"
+      host => "buffer01"
+      port => 5672
+      durable => true
+      persistent => true
+    }
+  }
+
+}

resources/elk6/filebeat.yml

@@ -0,0 +1,116 @@
+###################### Filebeat Configuration Example #########################
+
+# This file is an example configuration file highlighting only the most common
+# options. The filebeat.full.yml file from the same directory contains all the
+# supported options with more comments. You can use it as a reference.
+#
+# You can find the full configuration reference here:
+# https://www.elastic.co/guide/en/beats/filebeat/index.html
+
+#=========================== Filebeat prospectors =============================
+
+filebeat.prospectors:
+
+# Each - is a prospector. Most options can be set at the prospector level, so
+# you can use different prospectors for various configurations.
+# Below are the prospector specific configurations.
+
+- input_type: log
+  # Paths that should be crawled and fetched. Glob based paths.
+  paths:
+    # Linux Example
+    #- /var/log/*.log
+
+    #Windows Example
+    - c:\nessus\My Scans\*
+
+  # Exclude lines. A list of regular expressions to match. It drops the lines that are
+  # matching any regular expression from the list.
+  #exclude_lines: ["^DBG"]
+
+  # Include lines. A list of regular expressions to match. It exports the lines that are
+  # matching any regular expression from the list.
+  #include_lines: ["^ERR", "^WARN"]
+
+  # Exclude files. A list of regular expressions to match. Filebeat drops the files that
+  # are matching any regular expression from the list. By default, no files are dropped.
+  #exclude_files: [".gz$"]
+
+  # Optional additional fields. These field can be freely picked
+  # to add additional information to the crawled log files for filtering
+  #fields:
+  #  level: debug
+  #  review: 1
+
+  ### Multiline options
+
+  # Mutiline can be used for log messages spanning multiple lines. This is common
+  # for Java Stack Traces or C-Line Continuation
+
+  # The regexp Pattern that has to be matched. The example pattern matches all lines starting with [
+  #multiline.pattern: ^\[
+
+  # Defines if the pattern set under pattern should be negated or not. Default is false.
+  #multiline.negate: false
+
+  # Match can be set to "after" or "before". It is used to define if lines should be append to a pattern
+  # that was (not) matched before or after or as long as a pattern is not matched based on negate.
+  # Note: After is the equivalent to previous and before is the equivalent to to next in Logstash
+  #multiline.match: after
+
+
+#================================ General =====================================
+
+# The name of the shipper that publishes the network data. It can be used to group
+# all the transactions sent by a single shipper in the web interface.
+#name:
+
+# The tags of the shipper are included in their own field with each
+# transaction published.
+#tags: ["service-X", "web-tier"]
+
+# Optional fields that you can specify to add additional information to the
+# output.
+#fields:
+#  env: staging
+
+#================================ Outputs =====================================
+
+# Configure what outputs to use when sending the data collected by the beat.
+# Multiple outputs may be used.
+
+#-------------------------- Elasticsearch output ------------------------------
+#output.elasticsearch:
+  # Array of hosts to connect to.
+#  hosts: ["logstash01:9200"]
+
+  # Optional protocol and basic auth credentials.
+  #protocol: "https"
+  #username: "elastic"
+  #password: "changeme"
+
+#----------------------------- Logstash output --------------------------------
+output.logstash:
+  # The Logstash hosts
+  hosts: ["logstashserver1:5044", "logstashserver2:5044", "logstashserver3:5044"]
+
+  # Optional SSL. By default is off.
+  # List of root certificates for HTTPS server verifications
+  #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
+
+  # Certificate for SSL client authentication
+  #ssl.certificate: "/etc/pki/client/cert.pem"
+
+  # Client Certificate Key
+  #ssl.key: "/etc/pki/client/cert.key"
+
+#================================ Logging =====================================
+
+# Sets log level. The default log level is info.
+# Available log levels are: critical, error, warning, info, debug
+#logging.level: debug
+
+# At debug level, you can selectively enable logging only for some components.
+# To enable all selectors use ["*"]. Examples of other selectors are "beat",
+# "publish", "service".
+#logging.selectors: ["*"]

resources/elk6/init_kibana.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+#kibana_url="localhost:5601"
+kibana_url="kibana.local:5601"
+elasticsearch_url="elasticsearch.local:9200"
+add_saved_objects="curl -s -u elastic:changeme -k -XPOST 'http://"$kibana_url"/api/saved_objects/_bulk_create' -H 'Content-Type: application/json' -H \"kbn-xsrf: true\" -d @"
+
+#Create all saved objects - including index pattern
+saved_objects_file="kibana_APIonly.json"
+
+#if [ `curl -I localhost:5601/status | head -n1 |cut -d$' ' -f2` -eq '200' ]; then echo "Loading VulnWhisperer Saved Objects"; eval $(echo $add_saved_objects$saved_objects_file); else echo "waiting for kibana"; fi
+
+until curl -s "$elasticsearch_url/_cluster/health?pretty" | grep '"status"' | grep -qE "green|yellow"; do
+    curl -s "$elasticsearch_url/_cluster/health?pretty"
+    echo "Waiting for Elasticsearch..."
+    sleep 5
+done
+
+count=0
+until  curl -s --fail -XPUT "http://$elasticsearch_url/_template/vulnwhisperer" -H 'Content-Type: application/json' -d '@/opt/index-template.json'; do
+    echo "Loading VulnWhisperer index template..."
+    ((count++)) && ((count==60)) && break
+    sleep 1
+done
+
+if [[ count -le 60 && $(curl -s -I http://$elasticsearch_url/_template/vulnwhisperer | head -n1 |cut -d$' ' -f2) == "200" ]]; then
+    echo -e "\n✅ VulnWhisperer index template loaded"
+else
+    echo -e "\n❌ VulnWhisperer index template failed to load"
+fi
+
+until [ "`curl -s -I "$kibana_url"/status | head -n1 |cut -d$' ' -f2`" == "200" ]; do
+    curl -s -I "$kibana_url"/status
+    echo "Waiting for Kibana..."
+    sleep 5
+done
+
+echo "Loading VulnWhisperer Saved Objects"
+echo $add_saved_objects$saved_objects_file
+eval $(echo $add_saved_objects$saved_objects_file)
+
+#set "*" as default index
+#id_default_index="87f3bcc0-8b37-11e8-83be-afaed4786d8c"
+#os.system("curl -X POST -H \"Content-Type: application/json\" -H \"kbn-xsrf: true\" -d '{\"value\":\""+id_default_index+"\"}' http://elastic:changeme@"+kibana_url+"kibana/settings/defaultIndex")
+
+#Create vulnwhisperer index pattern
+#index_name = "logstash-vulnwhisperer-*"
+#os.system(add_index+index_name+"' '-d{\"attributes\":{\"title\":\""+index_name+"\",\"timeFieldName\":\"@timestamp\"}}'")
+
+#Create jira index pattern, separated for not fill of crap variables the Discover tab by default
+#index_name = "logstash-jira-*"
+#os.system(add_index+index_name+"' '-d{\"attributes\":{\"title\":\""+index_name+"\",\"timeFieldName\":\"@timestamp\"}}'")

resources/elk6/logstash-vulnwhisperer-template.json

@@ -0,0 +1,233 @@
+{
+  "index_patterns": "logstash-vulnwhisperer-*",
+  "mappings": {
+    "doc": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "@version": {
+          "type": "keyword"
+        },
+        "asset": {
+          "type": "text",
+          "norms": false,
+          "fields": {
+            "keyword": {
+              "type": "keyword",
+              "ignore_above": 256
+            }
+          }
+        },
+        "asset_uuid": {
+          "type": "keyword"
+        },
+        "assign_ip": {
+          "type": "ip"
+        },
+        "category": {
+          "type": "keyword"
+        },
+        "cve": {
+          "type": "keyword"
+        },
+        "cvss_base": {
+          "type": "float"
+        },
+        "cvss_temporal_vector": {
+          "type": "keyword"
+        },
+        "cvss_temporal": {
+          "type": "float"
+        },
+        "cvss_vector": {
+          "type": "keyword"
+        },
+        "cvss": {
+          "type": "float"
+        },
+        "cvss3_base": {
+          "type": "float"
+        },
+        "cvss3_temporal_vector": {
+          "type": "keyword"
+        },
+        "cvss3_temporal": {
+          "type": "float"
+        },
+        "cvss3_vector": {
+          "type": "keyword"
+        },
+        "cvss3": {
+          "type": "float"
+        },
+        "description": {
+          "fields": {
+            "keyword": {
+              "ignore_above": 256,
+              "type": "keyword"
+            }
+          },
+          "norms": false,
+          "type": "text"
+        },
+        "dns": {
+          "type": "keyword"
+        },
+        "exploitability": {
+          "fields": {
+            "keyword": {
+              "ignore_above": 256,
+              "type": "keyword"
+            }
+          },
+          "norms": false,
+          "type": "text"
+        },
+        "fqdn": {
+          "type": "keyword"
+        },
+        "geoip": {
+          "dynamic": true,
+          "type": "object",
+          "properties": {
+            "ip": {
+              "type": "ip"
+            },
+            "latitude": {
+              "type": "float"
+            },
+            "location": {
+              "type": "geo_point"
+            },
+            "longitude": {
+              "type": "float"
+            }
+          }
+        },
+        "history_id": {
+          "type": "keyword"
+        },
+        "host": {
+          "type": "keyword"
+        },
+        "host_end": {
+          "type": "date"
+        },
+        "host_start": {
+          "type": "date"
+        },
+        "impact": {
+          "fields": {
+            "keyword": {
+              "ignore_above": 256,
+              "type": "keyword"
+            }
+          },
+          "norms": false,
+          "type": "text"
+        },
+        "ip_status": {
+          "type": "keyword"
+        },
+        "ip": {
+          "type": "ip"
+        },
+        "last_updated": {
+          "type": "date"
+        },
+        "operating_system": {
+          "type": "keyword"
+        },
+        "path": {
+          "type": "keyword"
+        },
+        "pci_vuln": {
+          "type": "keyword"
+        },
+        "plugin_family": {
+          "type": "keyword"
+        },
+        "plugin_id": {
+          "type": "keyword"
+        },
+        "plugin_name": {
+          "type": "keyword"
+        },
+        "plugin_output": {
+          "fields": {
+            "keyword": {
+              "ignore_above": 256,
+              "type": "keyword"
+            }
+          },
+          "norms": false,
+          "type": "text"
+        },
+        "port": {
+          "type": "integer"
+        },
+        "protocol": {
+          "type": "keyword"
+        },
+        "results": {
+          "type": "text"
+        },
+        "risk_number": {
+          "type": "integer"
+        },
+        "risk_score_name": {
+          "type": "keyword"
+        },
+        "risk_score": {
+          "type": "float"
+        },
+        "risk": {
+          "type": "keyword"
+        },
+        "scan_id": {
+          "type": "keyword"
+        },
+        "scan_name": {
+          "type": "keyword"
+        },
+        "scan_reference": {
+          "type": "keyword"
+        },
+        "see_also": {
+          "type": "keyword"
+        },
+        "solution": {
+          "type": "keyword"
+        },
+        "source": {
+          "type": "keyword"
+        },
+        "ssl": {
+          "type": "keyword"
+        },
+        "synopsis": {
+          "type": "keyword"
+        },
+        "system_type": {
+          "type": "keyword"
+        },
+        "tags": {
+          "type": "keyword"
+        },
+        "threat": {
+          "type": "text"
+        },
+        "type": {
+          "type": "keyword"
+        },
+        "vendor_reference": {
+          "type": "keyword"
+        },
+        "vulnerability_state": {
+          "type": "keyword"
+        }
+      }
+    }
+  }
+}

resources/elk6/logstash-vulnwhisperer-template_elk7.json

@@ -0,0 +1,231 @@
+{
+  "index_patterns": "logstash-vulnwhisperer-*",
+  "mappings": {
+    "properties": {
+      "@timestamp": {
+        "type": "date"
+      },
+      "@version": {
+        "type": "keyword"
+      },
+      "asset": {
+        "type": "text",
+        "norms": false,
+        "fields": {
+          "keyword": {
+            "type": "keyword",
+            "ignore_above": 256
+          }
+        }
+      },
+      "asset_uuid": {
+        "type": "keyword"
+      },
+      "assign_ip": {
+        "type": "ip"
+      },
+      "category": {
+        "type": "keyword"
+      },
+      "cve": {
+        "type": "keyword"
+      },
+      "cvss_base": {
+        "type": "float"
+      },
+      "cvss_temporal_vector": {
+        "type": "keyword"
+      },
+      "cvss_temporal": {
+        "type": "float"
+      },
+      "cvss_vector": {
+        "type": "keyword"
+      },
+      "cvss": {
+        "type": "float"
+      },
+      "cvss3_base": {
+        "type": "float"
+      },
+      "cvss3_temporal_vector": {
+        "type": "keyword"
+      },
+      "cvss3_temporal": {
+        "type": "float"
+      },
+      "cvss3_vector": {
+        "type": "keyword"
+      },
+      "cvss3": {
+        "type": "float"
+      },
+      "description": {
+        "fields": {
+          "keyword": {
+            "ignore_above": 256,
+            "type": "keyword"
+          }
+        },
+        "norms": false,
+        "type": "text"
+      },
+      "dns": {
+        "type": "keyword"
+      },
+      "exploitability": {
+        "fields": {
+          "keyword": {
+            "ignore_above": 256,
+            "type": "keyword"
+          }
+        },
+        "norms": false,
+        "type": "text"
+      },
+      "fqdn": {
+        "type": "keyword"
+      },
+      "geoip": {
+        "dynamic": true,
+        "type": "object",
+        "properties": {
+          "ip": {
+            "type": "ip"
+          },
+          "latitude": {
+            "type": "float"
+          },
+          "location": {
+            "type": "geo_point"
+          },
+          "longitude": {
+            "type": "float"
+          }
+        }
+      },
+      "history_id": {
+        "type": "keyword"
+      },
+      "host": {
+        "type": "keyword"
+      },
+      "host_end": {
+        "type": "date"
+      },
+      "host_start": {
+        "type": "date"
+      },
+      "impact": {
+        "fields": {
+          "keyword": {
+            "ignore_above": 256,
+            "type": "keyword"
+          }
+        },
+        "norms": false,
+        "type": "text"
+      },
+      "ip_status": {
+        "type": "keyword"
+      },
+      "ip": {
+        "type": "ip"
+      },
+      "last_updated": {
+        "type": "date"
+      },
+      "operating_system": {
+        "type": "keyword"
+      },
+      "path": {
+        "type": "keyword"
+      },
+      "pci_vuln": {
+        "type": "keyword"
+      },
+      "plugin_family": {
+        "type": "keyword"
+      },
+      "plugin_id": {
+        "type": "keyword"
+      },
+      "plugin_name": {
+        "type": "keyword"
+      },
+      "plugin_output": {
+        "fields": {
+          "keyword": {
+            "ignore_above": 256,
+            "type": "keyword"
+          }
+        },
+        "norms": false,
+        "type": "text"
+      },
+      "port": {
+        "type": "integer"
+      },
+      "protocol": {
+        "type": "keyword"
+      },
+      "results": {
+        "type": "text"
+      },
+      "risk_number": {
+        "type": "integer"
+      },
+      "risk_score_name": {
+        "type": "keyword"
+      },
+      "risk_score": {
+        "type": "float"
+      },
+      "risk": {
+        "type": "keyword"
+      },
+      "scan_id": {
+        "type": "keyword"
+      },
+      "scan_name": {
+        "type": "keyword"
+      },
+      "scan_reference": {
+        "type": "keyword"
+      },
+      "see_also": {
+        "type": "keyword"
+      },
+      "solution": {
+        "type": "keyword"
+      },
+      "source": {
+        "type": "keyword"
+      },
+      "ssl": {
+        "type": "keyword"
+      },
+      "synopsis": {
+        "type": "keyword"
+      },
+      "system_type": {
+        "type": "keyword"
+      },
+      "tags": {
+        "type": "keyword"
+      },
+      "threat": {
+        "type": "text"
+      },
+      "type": {
+        "type": "keyword"
+      },
+      "vendor_reference": {
+        "type": "keyword"
+      },
+      "vulnerability_state": {
+        "type": "keyword"
+      }
+    }
+  }
+}

resources/elk6/logstash.yml

@@ -0,0 +1,9 @@
+node.name: logstash
+path.config: /usr/share/logstash/pipeline/
+path.data: /tmp
+queue.drain: true
+queue.type: persisted
+xpack.monitoring.elasticsearch.password: changeme
+xpack.monitoring.elasticsearch.url: elasticsearch:9200
+xpack.monitoring.elasticsearch.username: elastic
+xpack.monitoring.enabled: false

resources/elk6/pipeline/1000_nessus_process_file.conf

@@ -0,0 +1,182 @@
+# Author: Austin Taylor and Justin Henderson
+# Email: email@austintaylor.io
+# Last Update: 12/20/2017
+# Version 0.3
+# Description: Take in nessus reports from vulnWhisperer and pumps into logstash
+
+
+input {
+  file {
+    path => "/opt/VulnWhisperer/data/nessus/**/*"
+    mode => "read"
+    start_position => "beginning"
+    file_completed_action => "delete"
+    tags => "nessus"
+  }
+  file {
+    path => "/opt/VulnWhisperer/data/tenable/*.csv"
+    mode => "read"
+    start_position => "beginning"
+    file_completed_action => "delete"
+    tags => "tenable"
+  }
+}
+
+filter {
+  if "nessus" in [tags] or "tenable" in [tags] {
+    # Drop the header column
+    if [message] =~ "^Plugin ID" { drop {} }
+
+    csv {
+      # columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output"]
+      columns => ["plugin_id", "cve", "cvss", "risk", "asset", "protocol", "port", "plugin_name", "synopsis", "description", "solution", "see_also", "plugin_output", "asset_uuid", "vulnerability_state", "ip", "fqdn", "netbios", "operating_system", "mac_address", "plugin_family", "cvss_base", "cvss_temporal", "cvss_temporal_vector", "cvss_vector", "cvss3_base", "cvss3_temporal", "cvss3_temporal_vector", "cvss3_vector", "system_type", "host_start", "host_end"]
+      separator => ","
+      source => "message"
+    }
+
+    ruby {
+        code => "if event.get('description')
+                    event.set('description', event.get('description').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
+                end
+                 if event.get('synopsis')
+                    event.set('synopsis', event.get('synopsis').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
+                end
+                 if event.get('solution')
+                    event.set('solution', event.get('solution').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
+                end
+                 if event.get('see_also')
+                    event.set('see_also', event.get('see_also').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
+                end
+                 if event.get('plugin_output')
+                    event.set('plugin_output', event.get('plugin_output').gsub(92.chr + 'n', 10.chr).gsub(92.chr + 'r', 13.chr))
+                end"
+    }
+
+    #If using filebeats as your source, you will need to replace the "path" field to "source"
+    # Remove when scan name is included in event (current method is error prone)
+    grok {
+      match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.csv$" }
+      tag_on_failure => []
+    }
+
+    # TODO remove when @timestamp is included in event
+    date {
+      match => [ "last_updated", "UNIX" ]
+      target => "@timestamp"
+      remove_field => ["last_updated"]
+    }
+
+    if [risk] == "None" {
+      mutate { add_field => { "risk_number" => 0 }}
+    }
+    if [risk] == "Low" {
+      mutate { add_field => { "risk_number" => 1 }}
+    }
+    if [risk] == "Medium" {
+      mutate { add_field => { "risk_number" => 2 }}
+    }
+    if [risk] == "High" {
+      mutate { add_field => { "risk_number" => 3 }}
+    }
+    if [risk] == "Critical" {
+      mutate { add_field => { "risk_number" => 4 }}
+    }
+
+    if ![cve] or [cve] == "nan" {
+      mutate { remove_field => [ "cve" ] }
+    }
+    if ![cvss] or [cvss] == "nan" {
+      mutate { remove_field => [ "cvss" ] }
+    }
+    if ![cvss_base] or [cvss_base] == "nan" {
+      mutate { remove_field => [ "cvss_base" ] }
+    }
+    if ![cvss_temporal] or [cvss_temporal] == "nan" {
+      mutate { remove_field => [ "cvss_temporal" ] }
+    }
+    if ![cvss_temporal_vector] or [cvss_temporal_vector] == "nan" {
+      mutate { remove_field => [ "cvss_temporal_vector" ] }
+    }
+    if ![cvss_vector] or [cvss_vector] == "nan" {
+      mutate { remove_field => [ "cvss_vector" ] }
+    }
+    if ![cvss3_base] or [cvss3_base] == "nan" {
+      mutate { remove_field => [ "cvss3_base" ] }
+    }
+    if ![cvss3_temporal] or [cvss3_temporal] == "nan" {
+      mutate { remove_field => [ "cvss3_temporal" ] }
+    }
+    if ![cvss3_temporal_vector] or [cvss3_temporal_vector] == "nan" {
+      mutate { remove_field => [ "cvss3_temporal_vector" ] }
+    }
+    if ![description] or [description] == "nan" {
+      mutate { remove_field => [ "description" ] }
+    }
+    if ![mac_address] or [mac_address] == "nan" {
+      mutate { remove_field => [ "mac_address" ] }
+    }
+    if ![netbios] or [netbios] == "nan" {
+      mutate { remove_field => [ "netbios" ] }
+    }
+    if ![operating_system] or [operating_system] == "nan" {
+      mutate { remove_field => [ "operating_system" ] }
+    }
+    if ![plugin_output] or [plugin_output] == "nan" {
+      mutate { remove_field => [ "plugin_output" ] }
+    }
+    if ![see_also] or [see_also] == "nan" {
+      mutate { remove_field => [ "see_also" ] }
+    }
+    if ![synopsis] or [synopsis] == "nan" {
+      mutate { remove_field => [ "synopsis" ] }
+    }
+    if ![system_type] or [system_type] == "nan" {
+      mutate { remove_field => [ "system_type" ] }
+    }
+
+    mutate {
+      remove_field => [ "message" ]
+      add_field => { "risk_score" => "%{cvss}" }
+    }
+    mutate {
+      convert => { "risk_score" => "float" }
+    }
+    if [risk_score] == 0 {
+      mutate {
+        add_field => { "risk_score_name" => "info" }
+      }
+    }
+    if [risk_score] > 0 and [risk_score] < 3 {
+      mutate {
+        add_field => { "risk_score_name" => "low" }
+      }
+    }
+    if [risk_score] >= 3 and [risk_score] < 6 {
+      mutate {
+        add_field => { "risk_score_name" => "medium" }
+      }
+    }
+    if [risk_score] >=6 and [risk_score] < 9 {
+      mutate {
+        add_field => { "risk_score_name" => "high" }
+      }
+    }
+    if [risk_score] >= 9 {
+      mutate {
+        add_field => { "risk_score_name" => "critical" }
+      }
+    }
+  }
+}
+
+output {
+  if "nessus" in [tags] or "tenable" in [tags]{
+    stdout {
+      codec => dots
+    }
+    elasticsearch {
+      hosts => [ "elasticsearch:9200" ]
+      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+    }
+  }
+}

resources/elk6/pipeline/2000_qualys_web_scans.conf

@@ -0,0 +1,160 @@
+# Author: Austin Taylor and Justin Henderson
+# Email: austin@hasecuritysolutions.com
+# Last Update: 12/30/2017
+# Version 0.3
+# Description: Take in qualys web scan reports from vulnWhisperer and pumps into logstash
+
+input {
+  file {
+    path => [ "/opt/VulnWhisperer/data/qualys/*.json" , "/opt/VulnWhisperer/data/qualys_web/*.json", "/opt/VulnWhisperer/data/qualys_vuln/*.json"] 
+    type => json
+    codec => json
+    start_position => "beginning"
+    tags => [ "qualys" ]
+    mode => "read"
+    start_position => "beginning"
+    file_completed_action => "delete"
+  }
+}
+
+filter {
+  if "qualys" in [tags] {
+    grok {
+      match => { "path" => [ "(?<tags>qualys_vuln)_scan_%{DATA}_%{INT:last_updated}.json$", "(?<tags>qualys_web)_%{INT:app_id}_%{INT:last_updated}.json$" ] }
+      tag_on_failure => []
+    }
+    
+    mutate {
+      replace => [ "message", "%{message}" ]
+      #gsub => [
+      #  "message", "\|\|\|", " ",
+      #  "message", "\t\t", " ",
+      #  "message", "    ", " ",
+      #  "message", "   ", " ",
+      #  "message", "  ", " ",
+      #  "message", "nan", " ",
+      #  "message",'\n',''
+      #]
+    }
+
+    if "qualys_web" in [tags] {
+    mutate {
+      add_field => { "asset" => "%{web_application_name}" }
+      add_field => { "risk_score" => "%{cvss}" }
+      }
+    } else if "qualys_vuln" in [tags] {
+      mutate {
+        add_field => { "asset" => "%{ip}" }
+        add_field => { "risk_score" => "%{cvss}" }
+      }
+    }
+
+    if [risk] == "1" {
+      mutate { add_field => { "risk_number" => 0 }}
+      mutate { replace => { "risk" => "info" }}
+    }
+    if [risk] == "2" {
+      mutate { add_field => { "risk_number" => 1 }}
+      mutate { replace => { "risk" => "low" }}
+    }
+    if [risk] == "3" {
+      mutate { add_field => { "risk_number" => 2 }}
+      mutate { replace => { "risk" => "medium" }}
+    }
+    if [risk] == "4" {
+      mutate { add_field => { "risk_number" => 3 }}
+      mutate { replace => { "risk" => "high" }}
+    }
+    if [risk] == "5" {
+      mutate { add_field => { "risk_number" => 4 }}
+      mutate { replace => { "risk" => "critical" }}
+    }
+
+    mutate {
+      remove_field => "message"
+    }
+
+    if [first_time_detected] {
+      date {
+        match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "first_time_detected"
+      }
+    }
+    if [first_time_tested] {
+      date {
+        match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "first_time_tested"
+      }
+    }
+    if [last_time_detected] {
+      date {
+        match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "last_time_detected"
+      }
+    }
+    if [last_time_tested] {
+      date {
+        match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "last_time_tested"
+      }
+    }
+
+    # TODO remove when @timestamp is included in event
+    date {
+      match => [ "last_updated", "UNIX" ]
+      target => "@timestamp"
+      remove_field => "last_updated"
+    }
+    mutate {
+      convert => { "plugin_id" => "integer"}
+      convert => { "id" => "integer"}
+      convert => { "risk_number" => "integer"}
+      convert => { "risk_score" => "float"}
+      convert => { "total_times_detected" => "integer"}
+      convert => { "cvss_temporal" => "float"}
+      convert => { "cvss" => "float"}
+    }
+    if [risk_score] == 0 {
+      mutate {
+        add_field => { "risk_score_name" => "info" }
+      }
+    }
+    if [risk_score] > 0 and [risk_score] < 3 {
+      mutate {
+        add_field => { "risk_score_name" => "low" }
+      }
+    }
+    if [risk_score] >= 3 and [risk_score] < 6 {
+      mutate {
+        add_field => { "risk_score_name" => "medium" }
+      }
+    }
+    if [risk_score] >=6 and [risk_score] < 9 {
+      mutate {
+        add_field => { "risk_score_name" => "high" }
+      }
+    }
+    if [risk_score] >= 9 {
+      mutate {
+        add_field => { "risk_score_name" => "critical" }
+      }
+    }
+
+    if [asset] =~ "\.yourdomain\.(com|net)$" {
+      mutate {
+        add_tag => [ "critical_asset" ]
+      }
+    }
+  }
+}
+output {
+  if "qualys" in [tags] {
+    stdout {
+      codec => dots
+    }
+    elasticsearch {
+      hosts => [ "elasticsearch:9200" ]
+      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+    }
+  }
+}

resources/elk6/pipeline/3000_openvas.conf

@@ -0,0 +1,154 @@
+# Author: Austin Taylor and Justin Henderson
+# Email: austin@hasecuritysolutions.com
+# Last Update: 03/04/2018
+# Version 0.3
+# Description: Take in qualys web scan reports from vulnWhisperer and pumps into logstash
+
+input {
+  file {
+    path => "/opt/VulnWhisperer/data/openvas/*.json"
+    type => json
+    codec => json
+    start_position => "beginning"
+    tags => [ "openvas_scan", "openvas" ]
+    mode => "read"
+    start_position => "beginning"
+    file_completed_action => "delete"
+
+  }
+}
+
+filter {
+  if "openvas_scan" in [tags] {
+    mutate {
+      replace => [ "message", "%{message}" ]
+      gsub => [
+        "message", "\|\|\|", " ",
+        "message", "\t\t", " ",
+        "message", "    ", " ",
+        "message", "   ", " ",
+        "message", "  ", " ",
+        "message", "nan", " ",
+        "message",'\n',''
+      ]
+    }
+
+
+    grok {
+        match => { "path" => "openvas_scan_%{DATA:scan_id}_%{INT:last_updated}.json$" }
+        tag_on_failure => []
+    }
+
+    mutate {
+      add_field => { "risk_score" => "%{cvss}" }
+    }
+
+    if [risk] == "1" {
+      mutate { add_field => { "risk_number" => 0 }}
+      mutate { replace => { "risk" => "info" }}
+    }
+    if [risk] == "2" {
+      mutate { add_field => { "risk_number" => 1 }}
+      mutate { replace => { "risk" => "low" }}
+    }
+    if [risk] == "3" {
+      mutate { add_field => { "risk_number" => 2 }}
+      mutate { replace => { "risk" => "medium" }}
+    }
+    if [risk] == "4" {
+      mutate { add_field => { "risk_number" => 3 }}
+      mutate { replace => { "risk" => "high" }}
+    }
+    if [risk] == "5" {
+      mutate { add_field => { "risk_number" => 4 }}
+      mutate { replace => { "risk" => "critical" }}
+    }
+
+    mutate {
+      remove_field => "message"
+    }
+
+    if [first_time_detected] {
+      date {
+        match => [ "first_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "first_time_detected"
+      }
+    }
+    if [first_time_tested] {
+      date {
+        match => [ "first_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "first_time_tested"
+      }
+    }
+    if [last_time_detected] {
+      date {
+        match => [ "last_time_detected", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "last_time_detected"
+      }
+    }
+    if [last_time_tested] {
+      date {
+        match => [ "last_time_tested", "dd MMM yyyy HH:mma 'GMT'ZZ", "dd MMM yyyy HH:mma 'GMT'" ]
+        target => "last_time_tested"
+      }
+    }
+
+    # TODO remove when @timestamp is included in event
+    date {
+      match => [ "last_updated", "UNIX" ]
+      target => "@timestamp"
+      remove_field => "last_updated"
+    }
+    mutate {
+      convert => { "plugin_id" => "integer"}
+      convert => { "id" => "integer"}
+      convert => { "risk_number" => "integer"}
+      convert => { "risk_score" => "float"}
+      convert => { "total_times_detected" => "integer"}
+      convert => { "cvss_temporal" => "float"}
+      convert => { "cvss" => "float"}
+    }
+    if [risk_score] == 0 {
+      mutate {
+        add_field => { "risk_score_name" => "info" }
+      }
+    }
+    if [risk_score] > 0 and [risk_score] < 3 {
+      mutate {
+        add_field => { "risk_score_name" => "low" }
+      }
+    }
+    if [risk_score] >= 3 and [risk_score] < 6 {
+      mutate {
+        add_field => { "risk_score_name" => "medium" }
+      }
+    }
+    if [risk_score] >=6 and [risk_score] < 9 {
+      mutate {
+        add_field => { "risk_score_name" => "high" }
+      }
+    }
+    if [risk_score] >= 9 {
+      mutate {
+        add_field => { "risk_score_name" => "critical" }
+      }
+    }
+    # Add your critical assets by subnet or by hostname. Comment this field out if you don't want to tag any, but the asset panel will break.
+    if [asset] =~ "^10\.0\.100\." {
+      mutate {
+        add_tag => [ "critical_asset" ]
+      }
+    }
+  }
+}
+output {
+  if "openvas" in [tags] {
+    stdout {
+      codec => dots
+    }
+    elasticsearch {
+      hosts => [ "elasticsearch:9200" ]
+      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+    }
+  }
+}

resources/elk6/pipeline/4000_jira.conf

@@ -0,0 +1,25 @@
+# Description: Take in jira tickets from vulnWhisperer and pumps into logstash
+
+input {
+  file {
+    path => "/opt/VulnWhisperer/data/jira/*.json"
+    type => json
+    codec => json
+    start_position => "beginning"
+    mode => "read"
+    start_position => "beginning"
+    file_completed_action => "delete"
+    
+    tags => [ "jira" ]
+  }
+}
+
+output {
+  if "jira" in [tags] {
+    stdout { codec => rubydebug }
+    elasticsearch {
+      hosts => [ "elasticsearch:9200" ]
+      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
+    }
+  }
+}

resources/elk6/vulnwhisperer.ini

@@ -0,0 +1,109 @@
+[nessus]
+enabled=true
+hostname=localhost
+port=8834
+username=nessus_username
+password=nessus_password
+write_path=/opt/VulnWhisperer/data/nessus/
+db_path=/opt/VulnWhisperer/database
+trash=false
+verbose=true
+
+[tenable]
+enabled=true
+hostname=cloud.tenable.com
+port=443
+username=tenable.io_username
+password=tenable.io_password
+write_path=/opt/VulnWhisperer/data/tenable/
+db_path=/opt/VulnWhisperer/data/database
+trash=false
+verbose=true
+
+[qualys_web]
+#Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
+enabled = true
+hostname = qualysapi.qg2.apps.qualys.com
+username = exampleuser
+password = examplepass
+write_path=/opt/VulnWhisperer/data/qualys/
+db_path=/opt/VulnWhisperer/data/database
+verbose=true
+
+# Set the maximum number of retries each connection should attempt.
+#Note, this applies only to failed connections and timeouts, never to requests where the server returns a response.
+max_retries = 10
+# Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID.
+template_id = 126024
+
+[qualys_vuln]
+#Reference https://www.qualys.com/docs/qualys-was-api-user-guide.pdf to find your API
+enabled = true
+hostname = qualysapi.qg2.apps.qualys.com
+username = exampleuser
+password = examplepass
+write_path=/opt/VulnWhisperer/data/qualys/
+db_path=/opt/VulnWhisperer/data/database
+verbose=true
+
+# Set the maximum number of retries each connection should attempt.
+#Note, this applies only to failed connections and timeouts, never to requests where the server returns a response.
+max_retries = 10
+# Template ID will need to be retrieved for each document. Please follow the reference guide above for instructions on how to get your template ID.
+template_id = 126024
+
+[detectify]
+#Reference https://developer.detectify.com/
+enabled = false
+hostname = api.detectify.com
+#username variable used as apiKey
+username = exampleuser
+#password variable used as secretKey
+password = examplepass
+write_path =/opt/VulnWhisperer/data/detectify/
+db_path = /opt/VulnWhisperer/data/database
+verbose = true
+
+[openvas]
+enabled = false
+hostname = localhost
+port = 4000
+username = exampleuser
+password = examplepass
+write_path=/opt/VulnWhisperer/data/openvas/
+db_path=/opt/VulnWhisperer/data/database
+verbose=true
+
+#[proxy]
+; This section is optional. Leave it out if you're not using a proxy.
+; You can use environmental variables as well: http://www.python-requests.org/en/latest/user/advanced/#proxies
+
+; proxy_protocol set to https, if not specified.
+#proxy_url = proxy.mycorp.com
+
+; proxy_port will override any port specified in proxy_url
+#proxy_port = 8080
+
+; proxy authentication
+#proxy_username = proxyuser
+#proxy_password = proxypass
+
+[jira]
+hostname = jira-host
+username = username
+password = password
+write_path = /opt/VulnWhisperer/data/jira/
+db_path = /opt/VulnWhisperer/data/database
+verbose = true
+dns_resolv = False
+
+#Sample jira report scan, will automatically be created for existent scans
+#[jira.qualys_vuln.test_scan]
+#source = qualys_vuln
+#scan_name = Test Scan
+#jira_project = PROJECT
+; if multiple components, separate by "," = None
+#components = 
+; minimum criticality to report (low, medium, high or critical) = None
+#min_critical_to_report = high
+

setup.py

@@ -4,7 +4,7 @@ from setuptools import setup, find_packages
 
 setup(
     name='VulnWhisperer',
-    version='1.0.1',
+    version='1.8',
     packages=find_packages(),
     url='https://github.com/austin-taylor/vulnwhisperer',
     license="""MIT License
@@ -26,7 +26,7 @@ setup(
     SOFTWARE.""",
     author='Austin Taylor',
     author_email='email@austintaylor.io',
-    description='Vulnerability assessment framework aggregator',
+    description='Vulnerability Assessment Framework Aggregator',
     scripts=['bin/vuln_whisperer']
 )
 

tests/test-docker.sh

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+
+NORMAL=$(tput sgr0)
+GREEN=$(tput setaf 2)
+YELLOW=$(tput setaf 3)
+RED=$(tput setaf 1)
+
+function red() {
+    echo -e "$RED$*$NORMAL"
+}
+
+function green() {
+    echo -e "$GREEN$*$NORMAL"
+}
+
+function yellow() {
+    echo -e "$YELLOW$*$NORMAL"
+}
+
+return_code=0
+
+elasticsearch_url="localhost:9200"
+logstash_url="localhost:9600"
+
+until curl -s "$elasticsearch_url/_cluster/health?pretty" | grep '"status"' | grep -qE "green|yellow"; do
+    yellow "Waiting for Elasticsearch..."
+    sleep 5
+done
+green "✅ Elasticsearch status is green..."
+
+count=0
+until [[ $(curl -s "$logstash_url/_node/stats" | jq '.events.out') -ge 1236 ]]; do
+    yellow "Waiting for Logstash load to finish...  $(curl -s "$logstash_url/_node/stats" | jq '.events.out') of 1236 (attempt $count of 60)"
+    ((count++)) && ((count==60)) && break
+    sleep 5
+done
+
+if [[ count -le 60 && $(curl -s "$logstash_url/_node/stats" | jq '.events.out') -ge 1236 ]]; then
+    green "✅ Logstash load finished..."
+else
+    red "❌ Logstash load didn't complete... $(curl -s "$logstash_url/_node/stats" | jq '.events.out')"
+fi
+
+
+count=0
+until [[ $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq '.count') -ge 1232 ]] ; do
+    yellow "Waiting for Elasticsearch index to sync... $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq '.count') of 1232 logs loaded (attempt $count of 150)"
+    ((count++)) && ((count==150)) && break
+    sleep 2
+done
+if [[ count -le 50 && $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq '.count') -ge 1232 ]]; then
+    green "✅ logstash-vulnwhisperer-2019.03 document count >= 1232"
+else
+    red "❌ TIMED OUT waiting for logstash-vulnwhisperer-2019.03 document count: $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq) != 1232"
+fi
+
+# if [[ $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq '.count') == 1232 ]]; then
+#     green "✅ Passed: logstash-vulnwhisperer-2019.03 document count == 1232"
+# else
+#     red "❌ Failed: logstash-vulnwhisperer-2019.03 document count == 1232 was: $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count") instead"
+#     ((return_code = return_code + 1))
+# fi
+
+# Test Nessus plugin_name:Backported Security Patch Detection (FTP)
+nessus_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_search?q=plugin_name:%22Backported%20Security%20Patch%20Detection%20(FTP)%22%20AND%20asset:176.28.50.164%20AND%20tags:nessus" | jq '.hits.hits[]._source')
+if echo $nessus_doc | jq '.risk' | grep -q "None"; then
+    green "✅ Passed: Nessus risk == None"
+else
+    red "❌ Failed: Nessus risk == None was: $(echo $nessus_doc | jq '.risk') instead"
+    ((return_code = return_code + 1))
+fi
+
+# Test Tenable plugin_name:Backported Security Patch Detection (FTP)
+tenable_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_search?q=plugin_name:%22Backported%20Security%20Patch%20Detection%20(FTP)%22%20AND%20asset:176.28.50.164%20AND%20tags:tenable" | jq '.hits.hits[]._source')
+# Test asset
+if echo $tenable_doc | jq .asset | grep -q '176.28.50.164'; then
+    green "✅ Passed: Tenable asset == 176.28.50.164"
+else
+    red "❌ Failed: Tenable asset == 176.28.50.164 was: $(echo $tenable_doc | jq .asset) instead"
+    ((return_code = return_code + 1))
+fi
+
+# Test @timestamp
+if echo $tenable_doc | jq '.["@timestamp"]' | grep -q '2019-03-30T15:45:44.000Z'; then
+    green "✅ Passed: Tenable @timestamp == 2019-03-30T15:45:44.000Z"
+else
+    red "❌ Failed: Tenable @timestamp == 2019-03-30T15:45:44.000Z was: $(echo $tenable_doc | jq '.["@timestamp"]') instead"
+    ((return_code = return_code + 1))
+fi
+
+# Test Qualys plugin_name:OpenSSL Multiple Remote Security Vulnerabilities
+qualys_vuln_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_search?q=tags:qualys_vuln%20AND%20ip:%22176.28.50.164%22%20AND%20plugin_name:%22OpenSSL%20Multiple%20Remote%20Security%20Vulnerabilities%22%20AND%20port:465" | jq '.hits.hits[]._source')
+# Test @timestamp
+if echo $qualys_vuln_doc | jq '.["@timestamp"]' | grep -q '2019-03-30T10:17:41.000Z'; then
+    green "✅ Passed: Qualys VM @timestamp == 2019-03-30T10:17:41.000Z"
+else
+    red "❌ Failed: Qualys VM @timestamp == 2019-03-30T10:17:41.000Z was: $(echo $qualys_vuln_doc | jq '.["@timestamp"]') instead"
+    ((return_code = return_code + 1))
+fi
+
+# Test @XXXX
+if echo $qualys_vuln_doc | jq '.cvss' | grep -q '6.8'; then
+    green "✅ Passed: Qualys VM cvss == 6.8"
+else
+    red "❌ Failed: Qualys VM cvss == 6.8 was: $(echo $qualys_vuln_doc | jq '.cvss') instead"
+    ((return_code = return_code + 1))
+fi
+
+exit $return_code

tests/test-vuln_whisperer.sh

@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+
+NORMAL=$(tput sgr0)
+GREEN=$(tput setaf 2)
+YELLOW=$(tput setaf 3)
+RED=$(tput setaf 1)
+
+function red() {
+    echo -e "$RED$*$NORMAL"
+}
+
+function green() {
+    echo -e "$GREEN$*$NORMAL"
+}
+
+function yellow() {
+    echo -e "$YELLOW$*$NORMAL"
+}
+
+return_code=0
+
+TEST_PATH=${TEST_PATH:-"tests/data"}
+
+yellow "\n*********************************************"
+yellow "* Test successful scan download and parsing *"
+yellow "*********************************************"
+rm -rf /opt/VulnWhisperer/*
+if vuln_whisperer -F -c configs/test.ini --mock --mock_dir "${TEST_PATH}"; then
+    green "\n✅ Passed: Test successful scan download and parsing"
+else
+    red "\n❌ Failed: Test successful scan download and parsing"
+    ((return_code = return_code + 1))
+fi
+
+yellow "\n*********************************************"
+yellow "*    Test run with no scans to import       *"
+yellow "*********************************************"
+if vuln_whisperer -F -c configs/test.ini --mock --mock_dir "${TEST_PATH}"; then
+    green "\n✅ Passed: Test run with no scans to import"
+else
+    red "\n❌ Failed: Test run with no scans to import"
+    ((return_code = return_code + 1))
+fi
+
+yellow "\n*********************************************"
+yellow "*           Test one failed scan            *"
+yellow "*********************************************"
+rm -rf /opt/VulnWhisperer/*
+yellow "Removing ${TEST_PATH}/nessus/GET_scans_exports_164_download"
+mv "${TEST_PATH}/nessus/GET_scans_exports_164_download"{,.bak}
+if vuln_whisperer -F -c configs/test.ini --mock --mock_dir "${TEST_PATH}"; [[ $? -eq 1 ]]; then
+    green "\n✅ Passed: Test one failed scan"
+else
+    red "\n❌ Failed: Test one failed scan"
+    ((return_code = return_code + 1))
+fi
+
+yellow "\n*********************************************"
+yellow "*           Test two failed scans           *"
+yellow "*********************************************"
+rm -rf /opt/VulnWhisperer/*
+yellow "Removing ${TEST_PATH}/qualys_vuln/scan_1553941061.87241"
+mv "${TEST_PATH}/qualys_vuln/scan_1553941061.87241"{,.bak}
+if vuln_whisperer -F -c configs/test.ini --mock --mock_dir "${TEST_PATH}"; [[ $? -eq 2 ]]; then
+    green "\n✅ Passed: Test two failed scans"
+else
+    red "\n❌ Failed: Test two failed scans"
+    ((return_code = return_code + 1))
+fi
+
+yellow "\n*********************************************"
+yellow "*   Test only nessus with one failed scan   *"
+yellow "*********************************************"
+rm -rf /opt/VulnWhisperer/*
+if vuln_whisperer -F -c configs/test.ini -s nessus --mock --mock_dir "${TEST_PATH}"; [[ $? -eq 1 ]]; then
+    green "\n✅ Passed: Test only nessus with one failed scan"
+else
+    red "\n❌ Failed: Test only nessus with one failed scan"
+    ((return_code = return_code + 1))
+fi
+
+yellow "*********************************************"
+yellow "* Test only Qualys VM with one failed scan  *"
+yellow "*********************************************"
+rm -rf /opt/VulnWhisperer/*
+if vuln_whisperer -F -c configs/test.ini -s qualys_vuln --mock --mock_dir "${TEST_PATH}"; [[ $? -eq 1 ]]; then
+    green "\n✅ Passed: Test only Qualys VM with one failed scan"
+else
+    red "\n❌ Failed: Test only Qualys VM with one failed scan"
+    ((return_code = return_code + 1))
+fi
+
+# Restore the removed files
+mv "${TEST_PATH}/qualys_vuln/scan_1553941061.87241.bak" "${TEST_PATH}/qualys_vuln/scan_1553941061.87241"
+mv "${TEST_PATH}/nessus/GET_scans_exports_164_download.bak" "${TEST_PATH}/nessus/GET_scans_exports_164_download"
+
+exit $return_code

vulnwhisp/base/config.py

@@ -1,8 +1,8 @@
-import os
 import sys
+import logging
 
 # Support for python3
-if (sys.version_info > (3, 0)):
+if sys.version_info > (3, 0):
     import configparser as cp
 else:
     import ConfigParser as cp
@@ -14,9 +14,70 @@ class vwConfig(object):
         self.config_in = config_in
         self.config = cp.RawConfigParser()
         self.config.read(self.config_in)
+        self.logger = logging.getLogger('vwConfig')
 
     def get(self, section, option):
+        self.logger.debug('Calling get for {}:{}'.format(section, option))
         return self.config.get(section, option)
 
     def getbool(self, section, option):
-        return self.config.getboolean(section, option)
+        self.logger.debug('Calling getbool for {}:{}'.format(section, option))
+        return self.config.getboolean(section, option)
+
+    def get_sections_with_attribute(self, attribute):
+        sections = []
+        # TODO: does this not also need the "yes" case?
+        check = ["true", "True", "1"]
+        for section in self.config.sections():
+            try:
+                if self.get(section, attribute) in check:
+                    sections.append(section)
+            except:
+                self.logger.warn("Section {} has no option '{}'".format(section, attribute))
+        return sections
+
+    def exists_jira_profiles(self, profiles):
+        # get list of profiles source_scanner.scan_name
+        for profile in profiles:
+            if not self.config.has_section(self.normalize_section(profile)):
+                self.logger.warn("JIRA Scan Profile missing")
+                return False
+        return True
+
+    def update_jira_profiles(self, profiles):
+        # create JIRA profiles in the ini config file
+        self.logger.debug('Updating Jira profiles: {}'.format(str(profiles)))
+
+        for profile in profiles:
+            #IMPORTANT profile scans/results will be normalized to lower and "_" instead of spaces for ini file section
+            section_name = self.normalize_section(profile)
+            try:
+                self.get(section_name, "source")
+                self.logger.info("Skipping creating of section '{}'; already exists".format(section_name))
+            except:
+                self.logger.warn("Creating config section for '{}'".format(section_name))
+                self.config.add_section(section_name)
+                self.config.set(section_name, 'source', profile.split('.')[0])
+                # in case any scan name contains '.' character
+                self.config.set(section_name, 'scan_name', '.'.join(profile.split('.')[1:]))
+                self.config.set(section_name, 'jira_project', '')
+                self.config.set(section_name, '; if multiple components, separate by ","')
+                self.config.set(section_name, 'components', '')
+                self.config.set(section_name, '; minimum criticality to report (low, medium, high or critical)')
+                self.config.set(section_name, 'min_critical_to_report', 'high')
+                self.config.set(section_name, '; automatically report, boolean value ')
+                self.config.set(section_name, 'autoreport', 'false')
+
+        # TODO: try/catch this
+        # writing changes back to file
+        with open(self.config_in, 'w') as configfile:
+            self.config.write(configfile)
+            self.logger.debug('Written configuration to {}'.format(self.config_in))
+
+        # FIXME: this is the same as return None, that is the default return for return-less functions
+        return
+
+    def normalize_section(self, profile):
+        profile = "jira.{}".format(profile.lower().replace(" ", "_"))
+        self.logger.debug('Normalized profile as: {}'.format(profile))
+        return profile

vulnwhisp/frameworks/nessus.py

@@ -1,219 +1,184 @@
-import requests
-from requests.packages.urllib3.exceptions import InsecureRequestWarning
-
-requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
-import pandas as pd
-from pandas.io.json import json_normalize
-import pytz
-from datetime import datetime
-import json
-import sys
-import os
-import time
-import io
-
-
-class NessusAPI(object):
-    SESSION = '/session'
-    FOLDERS = '/folders'
-    SCANS = '/scans'
-    SCAN_ID = SCANS + '/{scan_id}'
-    HOST_VULN = SCAN_ID + '/hosts/{host_id}'
-    PLUGINS = HOST_VULN + '/plugins/{plugin_id}'
-    EXPORT = SCAN_ID + '/export'
-    EXPORT_TOKEN_DOWNLOAD = '/scans/exports/{token_id}/download'
-    EXPORT_FILE_DOWNLOAD = EXPORT + '/{file_id}/download'
-    EXPORT_STATUS = EXPORT + '/{file_id}/status'
-    EXPORT_HISTORY = EXPORT + '?history_id={history_id}'
-
-    def __init__(self, hostname=None, port=None, username=None, password=None, verbose=True):
-        if username is None or password is None:
-            raise Exception('ERROR: Missing username or password.')
-
-        self.user = username
-        self.password = password
-        self.base = 'https://{hostname}:{port}'.format(hostname=hostname, port=port)
-        self.verbose = verbose
-
-        self.headers = {
-            'Origin': self.base,
-            'Accept-Encoding': 'gzip, deflate, br',
-            'Accept-Language': 'en-US,en;q=0.8',
-            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.96 Safari/537.36',
-            'Content-Type': 'application/json',
-            'Accept': 'application/json, text/javascript, */*; q=0.01',
-            'Referer': self.base,
-            'X-Requested-With': 'XMLHttpRequest',
-            'Connection': 'keep-alive',
-            'X-Cookie': None
-        }
-
-        self.login()
-        self.scan_ids = self.get_scan_ids()
-
-    def vprint(self, msg):
-        if self.verbose:
-            print(msg)
-
-    def login(self):
-        resp = self.get_token()
-        if resp.status_code is 200:
-            self.headers['X-Cookie'] = 'token={token}'.format(token=resp.json()['token'])
-        else:
-            raise Exception('[FAIL] Could not login to Nessus')
-
-    def request(self, url, data=None, headers=None, method='POST', download=False, json=False):
-        if headers is None:
-            headers = self.headers
-        timeout = 0
-        success = False
-
-        url = self.base + url
-        methods = {'GET': requests.get,
-                   'POST': requests.post,
-                   'DELETE': requests.delete}
-
-        while (timeout <= 10) and (not success):
-            data = methods[method](url, data=data, headers=self.headers, verify=False)
-            if data.status_code == 401:
-                try:
-                    self.login()
-                    timeout += 1
-                    self.vprint('[INFO] Token refreshed')
-                except Exception as e:
-                    self.vprint('[FAIL] Could not refresh token\nReason: %s' % e)
-            else:
-                success = True
-
-        if json:
-            data = data.json()
-        if download:
-            return data.content
-        return data
-
-    def get_token(self):
-        auth = '{"username":"%s", "password":"%s"}' % (self.user, self.password)
-        token = self.request(self.SESSION, data=auth, json=False)
-        return token
-
-    def logout(self):
-        self.request(self.SESSION, method='DELETE')
-
-    def get_folders(self):
-        folders = self.request(self.FOLDERS, method='GET', json=True)
-        return folders
-
-    def get_scans(self):
-        scans = self.request(self.SCANS, method='GET', json=True)
-        return scans
-
-    def get_scan_ids(self):
-        scans = self.get_scans()
-        scan_ids = [scan_id['id'] for scan_id in scans['scans']]
-        return scan_ids
-
-    def count_scan(self, scans, folder_id):
-        count = 0
-        for scan in scans:
-            if scan['folder_id'] == folder_id: count = count + 1
-        return count
-
-    def print_scans(self, data):
-        for folder in data['folders']:
-            print("\\{0} - ({1})\\".format(folder['name'], self.count_scan(data['scans'], folder['id'])))
-            for scan in data['scans']:
-                if scan['folder_id'] == folder['id']:
-                    print(
-                    "\t\"{0}\" - sid:{1} - uuid: {2}".format(scan['name'].encode('utf-8'), scan['id'], scan['uuid']))
-
-    def get_scan_details(self, scan_id):
-        data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json=True)
-        return data
-
-    def get_scan_history(self, scan_id):
-        data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json=True)
-        return data['history']
-
-    def get_scan_hosts(self, scan_id):
-        data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json=True)
-        return data['hosts']
-
-    def get_host_vulnerabilities(self, scan_id, host_id):
-        query = self.HOST_VULN.format(scan_id=scan_id, host_id=host_id)
-        data = self.request(query, method='GET', json=True)
-        return data
-
-    def get_plugin_info(self, scan_id, host_id, plugin_id):
-        query = self.PLUGINS.format(scan_id=scan_id, host_id=host_id, plugin_id=plugin_id)
-        data = self.request(query, method='GET', json=True)
-        return data
-
-    def export_scan(self, scan_id, history_id):
-        data = {'format': 'csv'}
-        query = self.EXPORT_REPORT.format(scan_id=scan_id, history_id=history_id)
-        req = self.request(query, data=data, method='POST')
-        return req
-
-    def download_scan(self, scan_id=None, history=None, export_format="", chapters="", dbpasswd=""):
-        running = True
-        counter = 0
-
-        data = {'format': export_format}
-        if not history:
-            query = self.EXPORT.format(scan_id=scan_id)
-        else:
-            query = self.EXPORT_HISTORY.format(scan_id=scan_id, history_id=history)
-            scan_id = str(scan_id)
-        req = self.request(query, data=json.dumps(data), method='POST', json=True)
-        try:
-            file_id = req['file']
-            token_id = req['token']
-        except Exception as e:
-            print("[ERROR] %s" % e)
-        print('Download for file id ' + str(file_id) + '.')
-        while running:
-            time.sleep(2)
-            counter += 2
-            report_status = self.request(self.EXPORT_STATUS.format(scan_id=scan_id, file_id=file_id), method='GET',
-                                         json=True)
-            running = report_status['status'] != 'ready'
-            sys.stdout.write(".")
-            sys.stdout.flush()
-            if counter % 60 == 0:
-                print("")
-
-        print("")
-        content = self.request(self.EXPORT_TOKEN_DOWNLOAD.format(token_id=token_id), method='GET', download=True)
-        return content
-
-    @staticmethod
-    def merge_dicts(self, *dict_args):
-        """
-        Given any number of dicts, shallow copy and merge into a new dict,
-        precedence goes to key value pairs in latter dicts.
-        """
-        result = {}
-        for dictionary in dict_args:
-            result.update(dictionary)
-        return result
-
-    def get_utc_from_local(self, date_time, local_tz=None, epoch=True):
-        date_time = datetime.fromtimestamp(date_time)
-        if local_tz is None:
-            local_tz = pytz.timezone('US/Central')
-        else:
-            local_tz = pytz.timezone(local_tz)
-        # print date_time
-        local_time = local_tz.normalize(local_tz.localize(date_time))
-        local_time = local_time.astimezone(pytz.utc)
-        if epoch:
-            naive = local_time.replace(tzinfo=None)
-            local_time = int((naive - datetime(1970, 1, 1)).total_seconds())
-        return local_time
-
-    def tz_conv(self, tz):
-        time_map = {'Eastern Standard Time': 'US/Eastern',
-                    'Central Standard Time': 'US/Central',
-                    'Pacific Standard Time': 'US/Pacific',
-                    'None': 'US/Central'}
-        return time_map.get(tz, None)
+import json
+import logging
+import sys
+import time
+from datetime import datetime
+
+import pytz
+import requests
+from requests.packages.urllib3.exceptions import InsecureRequestWarning
+
+requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
+
+
+class NessusAPI(object):
+    SESSION = '/session'
+    FOLDERS = '/folders'
+    SCANS = '/scans'
+    SCAN_ID = SCANS + '/{scan_id}'
+    HOST_VULN = SCAN_ID + '/hosts/{host_id}'
+    PLUGINS = HOST_VULN + '/plugins/{plugin_id}'
+    EXPORT = SCAN_ID + '/export'
+    EXPORT_TOKEN_DOWNLOAD = '/scans/exports/{token_id}/download'
+    EXPORT_FILE_DOWNLOAD = EXPORT + '/{file_id}/download'
+    EXPORT_STATUS = EXPORT + '/{file_id}/status'
+    EXPORT_HISTORY = EXPORT + '?history_id={history_id}'
+
+    def __init__(self, hostname=None, port=None, username=None, password=None, verbose=True, profile=None, access_key=None, secret_key=None):
+        self.logger = logging.getLogger('NessusAPI')
+        if verbose:
+            self.logger.setLevel(logging.DEBUG)
+        if not all((username, password)) and not all((access_key, secret_key)):
+            raise Exception('ERROR: Missing username, password or API keys.')
+
+        self.profile = profile
+        self.user = username
+        self.password = password
+        self.api_keys = False
+        self.access_key = access_key
+        self.secret_key = secret_key
+        self.base = 'https://{hostname}:{port}'.format(hostname=hostname, port=port)
+        self.verbose = verbose
+
+        self.session = requests.Session()
+        self.session.verify = False
+        self.session.stream = True
+        self.session.headers = {
+            'Origin': self.base,
+            'Accept-Encoding': 'gzip, deflate, br',
+            'Accept-Language': 'en-US,en;q=0.8',
+            'User-Agent': 'VulnWhisperer for Nessus',
+            'Content-Type': 'application/json',
+            'Accept': 'application/json, text/javascript, */*; q=0.01',
+            'Referer': self.base,
+            'X-Requested-With': 'XMLHttpRequest',
+            'Connection': 'keep-alive',
+            'X-Cookie': None
+        }
+
+        if all((self.access_key, self.secret_key)):
+            self.logger.debug('Using {} API keys'.format(self.profile))
+            self.api_keys = True
+            self.session.headers['X-ApiKeys'] = 'accessKey={}; secretKey={}'.format(self.access_key, self.secret_key)
+        else:
+            self.login()
+
+        self.scans = self.get_scans()
+        self.scan_ids = self.get_scan_ids()
+
+    def login(self):
+        auth = '{"username":"%s", "password":"%s"}' % (self.user, self.password)
+        resp = self.request(self.SESSION, data=auth, json_output=False)
+        if resp.status_code == 200:
+            self.session.headers['X-Cookie'] = 'token={token}'.format(token=resp.json()['token'])
+        else:
+            raise Exception('[FAIL] Could not login to Nessus')
+
+    def request(self, url, data=None, headers=None, method='POST', download=False, json_output=False):
+        timeout = 0
+        success = False
+
+        method = method.lower()
+        url = self.base + url
+        self.logger.debug('Requesting to url {}'.format(url))
+
+        while (timeout <= 10) and (not success):
+            response = getattr(self.session, method)(url, data=data)
+            if response.status_code == 401:
+                if url == self.base + self.SESSION:
+                    break
+                try:
+                    timeout += 1
+                    if self.api_keys:
+                        continue
+                    self.login()
+                    self.logger.info('Token refreshed')
+                except Exception as e:
+                    self.logger.error('Could not refresh token\nReason: {}'.format(str(e)))
+            else:
+                success = True
+
+        if json_output:
+            return response.json()
+        if download:
+            self.logger.debug('Returning data.content')
+            response_data = ''
+            count = 0
+            for chunk in response.iter_content(chunk_size=8192):
+                count += 1
+                if chunk:
+                    response_data += chunk
+            self.logger.debug('Processed {} chunks'.format(count))
+            return response_data
+        return response
+
+    def get_scans(self):
+        scans = self.request(self.SCANS, method='GET', json_output=True)
+        return scans
+
+    def get_scan_ids(self):
+        scans = self.scans
+        scan_ids = [scan_id['id'] for scan_id in scans['scans']] if scans['scans'] else []
+        self.logger.debug('Found {} scan_ids'.format(len(scan_ids)))
+        return scan_ids
+
+    def get_scan_history(self, scan_id):
+        data = self.request(self.SCAN_ID.format(scan_id=scan_id), method='GET', json_output=True)
+        return data['history']
+
+    def download_scan(self, scan_id=None, history=None, export_format=""):
+        running = True
+        counter = 0
+
+        data = {'format': export_format}
+        if not history:
+            query = self.EXPORT.format(scan_id=scan_id)
+        else:
+            query = self.EXPORT_HISTORY.format(scan_id=scan_id, history_id=history)
+            scan_id = str(scan_id)
+        req = self.request(query, data=json.dumps(data), method='POST', json_output=True)
+        try:
+            file_id = req['file']
+            if self.profile == 'nessus':
+                token_id = req['token'] if 'token' in req else req['temp_token']
+        except Exception as e:
+            self.logger.error('{}'.format(str(e)))
+        self.logger.info('Download for file id {}'.format(str(file_id)))
+        while running:
+            time.sleep(2)
+            counter += 2
+            report_status = self.request(self.EXPORT_STATUS.format(scan_id=scan_id, file_id=file_id), method='GET',
+                                         json_output=True)
+            running = report_status['status'] != 'ready'
+            sys.stdout.write(".")
+            sys.stdout.flush()
+            # FIXME: why? can this be removed in favour of a counter?
+            if counter % 60 == 0:
+                self.logger.info("Completed: {}".format(counter))
+        self.logger.info("Done: {}".format(counter))
+        if self.profile == 'tenable' or self.api_keys:
+            content = self.request(self.EXPORT_FILE_DOWNLOAD.format(scan_id=scan_id, file_id=file_id), method='GET', download=True)
+        else:
+            content = self.request(self.EXPORT_TOKEN_DOWNLOAD.format(token_id=token_id), method='GET', download=True)
+        return content
+
+    def get_utc_from_local(self, date_time, local_tz=None, epoch=True):
+        date_time = datetime.fromtimestamp(date_time)
+        if local_tz is None:
+            local_tz = pytz.timezone('UTC')
+        else:
+            local_tz = pytz.timezone(local_tz)
+        local_time = local_tz.normalize(local_tz.localize(date_time))
+        local_time = local_time.astimezone(pytz.utc)
+        if epoch:
+            naive = local_time.replace(tzinfo=None)
+            local_time = int((naive - datetime(1970, 1, 1)).total_seconds())
+        self.logger.debug('Converted timestamp {} in datetime {}'.format(date_time, local_time))
+        return local_time
+
+    def tz_conv(self, tz):
+        time_map = {'Eastern Standard Time': 'US/Eastern',
+                    'Central Standard Time': 'US/Central',
+                    'Pacific Standard Time': 'US/Pacific',
+                    'None': 'US/Central'}
+        return time_map.get(tz, None)

vulnwhisp/frameworks/openvas.py

@@ -0,0 +1,192 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+__author__ = 'Austin Taylor'
+
+import datetime as dt
+import io
+import logging
+
+import pandas as pd
+import requests
+from bs4 import BeautifulSoup
+
+
+class OpenVAS_API(object):
+    OMP = '/omp'
+
+    def __init__(self,
+                 hostname=None,
+                 port=None,
+                 username=None,
+                 password=None,
+                 report_format_id=None,
+                 verbose=True):
+        self.logger = logging.getLogger('OpenVAS_API')
+        if verbose:
+            self.logger.setLevel(logging.DEBUG)
+        if username is None or password is None:
+            raise Exception('ERROR: Missing username or password.')
+
+        self.username = username
+        self.password = password
+        self.base = 'https://{hostname}:{port}'.format(hostname=hostname, port=port)
+        self.verbose = verbose
+        self.processed_reports = 0
+        self.report_format_id = report_format_id
+
+        self.headers = {
+            'Origin': self.base,
+            'Accept-Encoding': 'gzip, deflate, br',
+            'Accept-Language': 'en-US,en;q=0.8',
+            'User-Agent': 'VulnWhisperer for OpenVAS',
+            'Content-Type': 'application/x-www-form-urlencoded',
+            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8',
+            'Cache-Control': 'max-age=0',
+            'Referer': self.base,
+            'X-Requested-With': 'XMLHttpRequest',
+            'Connection': 'keep-alive',
+        }
+
+        self.login()
+
+        self.openvas_reports = self.get_reports()
+        self.report_formats = self.get_report_formats()
+
+    def login(self):
+        resp = self.get_token()
+        if resp.status_code is 200:
+            xml_response = BeautifulSoup(resp.content, 'lxml')
+            self.token = xml_response.find(attrs={'id': 'gsa-token'}).text
+
+            self.cookies = resp.cookies.get_dict()
+        else:
+            raise Exception('[FAIL] Could not login to OpenVAS')
+
+    def request(self, url, data=None, params=None, headers=None, cookies=None, method='POST', download=False,
+                json=False):
+        if headers is None:
+            headers = self.headers
+        if cookies is None:
+            cookies = self.cookies
+
+        timeout = 0
+        success = False
+
+        url = self.base + url
+        methods = {'GET': requests.get,
+                   'POST': requests.post,
+                   'DELETE': requests.delete}
+
+        while (timeout <= 10) and (not success):
+            data = methods[method](url,
+                                   data=data,
+                                   headers=self.headers,
+                                   params=params,
+                                   cookies=cookies,
+                                   verify=False)
+
+            if data.status_code == 401:
+                try:
+                    self.login()
+                    timeout += 1
+                    self.logger.info(' Token refreshed')
+                except Exception as e:
+                    self.logger.error('Could not refresh token\nReason: {}'.format(str(e)))
+            else:
+                success = True
+
+        if json:
+            data = data.json()
+        if download:
+            return data.content
+        return data
+
+    def get_token(self):
+        data = [
+            ('cmd', 'login'),
+            ('text', '/omp?r=1'),
+            ('login', self.username),
+            ('password', self.password),
+        ]
+        token = requests.post(self.base + self.OMP, data=data, verify=False)
+        return token
+    def get_report_formats(self):
+        params = (
+            ('cmd', 'get_report_formats'),
+            ('token', self.token)
+        )
+        self.logger.info('Retrieving available report formats')
+        data = self.request(url=self.OMP, method='GET', params=params)
+
+        bs = BeautifulSoup(data.content, "lxml")
+        table_body = bs.find('tbody')
+        rows = table_body.find_all('tr')
+        format_mapping = {}
+        for row in rows:
+            cols = row.find_all('td')
+            for x in cols:
+                for y in x.find_all('a'):
+                    if y.get_text() != '':
+                        format_mapping[y.get_text()] = \
+                        [h.split('=')[1] for h in y['href'].split('&') if 'report_format_id' in h][0]
+        return format_mapping
+
+    def get_reports(self, complete=True):
+        self.logger.info('Retreiving OpenVAS report data...')
+        params = (('cmd', 'get_reports'),
+                  ('token', self.token),
+                  ('max_results', 1),
+                  ('ignore_pagination', 1),
+                  ('filter', 'apply_overrides=1 min_qod=70 autofp=0 first=1 rows=0 levels=hml sort-reverse=severity'),
+                 )
+        reports = self.request(self.OMP, params=params, method='GET')
+        soup = BeautifulSoup(reports.text, 'lxml')
+        data = []
+        links = []
+        table = soup.find('table', attrs={'class': 'gbntable'})
+        table_body = table.find('tbody')
+
+        rows = table_body.find_all('tr')
+        for row in rows:
+            cols = row.find_all('td')
+            links.extend([a['href'] for a in row.find_all('a', href=True) if 'get_report' in str(a)])
+            cols = [ele.text.strip() for ele in cols]
+            data.append([ele for ele in cols if ele])
+            report = pd.DataFrame(data, columns=['date', 'status', 'task', 'scan_severity', 'high', 'medium', 'low', 'log',
+                                                 'false_pos'])
+
+        if report.shape[0] != 0:
+            report['links'] = links
+            report['report_ids'] = report.links.str.extract('.*report_id=([a-z-0-9]*)', expand=False)
+            report['epoch'] = (pd.to_datetime(report['date']) - dt.datetime(1970, 1, 1)).dt.total_seconds().astype(int)
+        else:
+            raise Exception("Could not retrieve OpenVAS Reports - Please check your settings and try again")
+
+        report['links'] = links
+        report['report_ids'] = report.links.str.extract('.*report_id=([a-z-0-9]*)', expand=False)
+        report['epoch'] = (pd.to_datetime(report['date']) - dt.datetime(1970, 1, 1)).dt.total_seconds().astype(int)
+        if complete:
+            report = report[report.status == 'Done']
+        severity_extraction = report.scan_severity.str.extract('([0-9.]*) \(([\w]+)\)', expand=False)
+        severity_extraction.columns = ['scan_highest_severity', 'severity_rate']
+        report_with_severity = pd.concat([report, severity_extraction], axis=1)
+        return report_with_severity
+
+    def process_report(self, report_id):
+
+        params = (
+            ('token', self.token),
+            ('cmd', 'get_report'),
+            ('report_id', report_id),
+            ('filter', 'apply_overrides=0 min_qod=70 autofp=0 levels=hml first=1 rows=0 sort-reverse=severity'),
+            ('ignore_pagination', '1'),
+            ('report_format_id', '{report_format_id}'.format(report_format_id=self.report_formats['CSV Results'])),
+            ('submit', 'Download'),
+        )
+        self.logger.info('Retrieving {}'.format(report_id))
+        req = self.request(self.OMP, params=params, method='GET')
+        report_df = pd.read_csv(io.BytesIO(req.text.encode('utf-8')))
+        report_df['report_ids'] = report_id
+        self.processed_reports += 1
+        merged_df = pd.merge(report_df, self.openvas_reports, on='report_ids').reset_index().drop('index', axis=1)
+        return merged_df

vulnwhisp/frameworks/qualys_vuln.py

@@ -0,0 +1,124 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+__author__ = 'Nathan Young'
+
+import logging
+import sys
+import xml.etree.ElementTree as ET
+
+import dateutil.parser as dp
+import pandas as pd
+import qualysapi
+
+
+class qualysWhisperAPI(object):
+    SCANS = 'api/2.0/fo/scan'
+
+    def __init__(self, config=None):
+        self.logger = logging.getLogger('qualysWhisperAPI')
+        self.config = config
+        try:
+            self.qgc = qualysapi.connect(config, 'qualys_vuln')
+            # Fail early if we can't make a request or auth is incorrect
+            self.qgc.request('about.php')
+            self.logger.info('Connected to Qualys at {}'.format(self.qgc.server))
+        except Exception as e:
+            self.logger.error('Could not connect to Qualys: {}'.format(str(e)))
+            sys.exit(1)
+
+    def scan_xml_parser(self, xml):
+        all_records = []
+        root = ET.XML(xml.encode("utf-8"))
+        for child in root.find('.//SCAN_LIST'):
+            all_records.append({
+                'name': child.find('TITLE').text,
+                'id': child.find('REF').text,
+                'date': child.find('LAUNCH_DATETIME').text,
+                'type': child.find('TYPE').text,
+                'duration': child.find('DURATION').text,
+                'status': child.find('.//STATE').text,
+            })
+        return pd.DataFrame(all_records)
+
+    def get_all_scans(self):
+        parameters = {
+            'action': 'list',
+            'echo_request': 0,
+            'show_op': 0,
+            'launched_after_datetime': '0001-01-01'
+        }
+        scans_xml = self.qgc.request(self.SCANS, parameters)
+        return self.scan_xml_parser(scans_xml)
+
+    def get_scan_details(self, scan_id=None):
+        parameters = {
+            'action': 'fetch',
+            'echo_request': 0,
+            'output_format': 'json_extended',
+            'mode': 'extended',
+            'scan_ref': scan_id
+        }
+        scan_json = self.qgc.request(self.SCANS, parameters)
+
+        # First two columns are metadata we already have
+        # Last column corresponds to "target_distribution_across_scanner_appliances" element
+        # which doesn't follow the schema and breaks the pandas data manipulation
+        return pd.read_json(scan_json).iloc[2:-1]
+
+
+class qualysUtils:
+    def __init__(self):
+        self.logger = logging.getLogger('qualysUtils')
+
+    def iso_to_epoch(self, dt):
+        out = dp.parse(dt).strftime('%s')
+        self.logger.info('Converted {} to {}'.format(dt, out))
+        return out
+
+
+class qualysVulnScan:
+
+    def __init__(
+        self,
+        config=None,
+        file_in=None,
+        file_stream=False,
+        delimiter=',',
+        quotechar='"',
+    ):
+        self.logger = logging.getLogger('qualysVulnScan')
+        self.file_in = file_in
+        self.file_stream = file_stream
+        self.report = None
+        self.utils = qualysUtils()
+
+        if config:
+            try:
+                self.qw = qualysWhisperAPI(config=config)
+            except Exception as e:
+                self.logger.error('Could not load config! Please check settings. Error: {}'.format(str(e)))
+
+        if file_stream:
+            self.open_file = file_in.splitlines()
+        elif file_in:
+            self.open_file = open(file_in, 'rb')
+
+        self.downloaded_file = None
+
+    def process_data(self, scan_id=None):
+        """Downloads a file from Qualys and normalizes it"""
+        self.logger.info('Downloading scan ID: {}'.format(scan_id))
+        scan_report = self.qw.get_scan_details(scan_id=scan_id)
+        if not scan_report.empty:
+            keep_columns = ['category', 'cve_id', 'cvss3_base', 'cvss3_temporal', 'cvss_base',
+                'cvss_temporal', 'dns', 'exploitability', 'fqdn', 'impact', 'ip', 'ip_status',
+                'netbios', 'os', 'pci_vuln', 'port', 'protocol', 'qid', 'results', 'severity',
+                'solution', 'ssl', 'threat', 'title', 'type', 'vendor_reference']
+            scan_report = scan_report.filter(keep_columns)
+            scan_report['severity'] = scan_report['severity'].astype(int).astype(str)
+            scan_report['qid'] = scan_report['qid'].astype(int).astype(str)
+        else:
+            self.logger.warn('Scan ID {} has no vulnerabilities, skipping.'.format(scan_id))
+            return scan_report
+
+        return scan_report

vulnwhisp/frameworks/qualys_web.py

@@ -0,0 +1,465 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+__author__ = 'Austin Taylor'
+
+from lxml import objectify
+from lxml.builder import E
+import xml.etree.ElementTree as ET
+import pandas as pd
+import qualysapi
+import qualysapi.config as qcconf
+import requests
+import sys
+import os
+import csv
+import logging
+import dateutil.parser as dp
+
+
+class qualysWhisperAPI(object):
+    COUNT_WEBAPP = '/count/was/webapp'
+    COUNT_WASSCAN = '/count/was/wasscan'
+    DELETE_REPORT = '/delete/was/report/{report_id}'
+    GET_WEBAPP_DETAILS = '/get/was/webapp/{was_id}'
+    QPS_REST_3 = '/qps/rest/3.0'
+    REPORT_DETAILS = '/get/was/report/{report_id}'
+    REPORT_STATUS = '/status/was/report/{report_id}'
+    REPORT_CREATE = '/create/was/report'
+    REPORT_DOWNLOAD = '/download/was/report/{report_id}'
+    SCAN_DETAILS = '/get/was/wasscan/{scan_id}'
+    SCAN_DOWNLOAD = '/download/was/wasscan/{scan_id}'
+    SEARCH_REPORTS = '/search/was/report'
+    SEARCH_WEB_APPS = '/search/was/webapp'
+    SEARCH_WAS_SCAN = '/search/was/wasscan'
+    VERSION = '/qps/rest/portal/version'
+
+    def __init__(self, config=None):
+        self.logger = logging.getLogger('qualysWhisperAPI')
+        self.config = config
+        try:
+            self.qgc = qualysapi.connect(config, 'qualys_web')
+            self.logger.info('Connected to Qualys at {}'.format(self.qgc.server))
+        except Exception as e:
+            self.logger.error('Could not connect to Qualys: {}'.format(str(e)))
+        self.headers = {
+            #"content-type": "text/xml"}
+            "Accept" : "application/json",
+            "Content-Type": "application/json"}
+        self.config_parse = qcconf.QualysConnectConfig(config, 'qualys_web')
+        try:
+            self.template_id = self.config_parse.get_template_id()
+        except:
+            self.logger.error('Could not retrieve template ID')
+
+    ####
+    #### GET SCANS TO PROCESS
+    ####
+
+    def get_was_scan_count(self, status):
+        """
+        Checks number of scans, used to control the api limits
+        """
+        parameters = (
+            E.ServiceRequest(
+                E.filters(
+                    E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status))))
+        xml_output = self.qgc.request(self.COUNT_WASSCAN, parameters)
+        root = objectify.fromstring(xml_output.encode('utf-8'))
+        return root.count.text
+
+    def generate_scan_result_XML(self, limit=1000, offset=1, status='FINISHED'):
+        report_xml = E.ServiceRequest(
+            E.filters(
+                E.Criteria({'field': 'status', 'operator': 'EQUALS'}, status
+                           ),
+            ),
+            E.preferences(
+                E.startFromOffset(str(offset)),
+                E.limitResults(str(limit))
+            ),
+        )
+        return report_xml
+
+    def get_scan_info(self, limit=1000, offset=1, status='FINISHED'):
+        """ Returns XML of ALL WAS Scans"""
+        data = self.generate_scan_result_XML(limit=limit, offset=offset, status=status)
+        return self.qgc.request(self.SEARCH_WAS_SCAN, data)
+
+    def xml_parser(self, xml, dupfield=None):
+        all_records = []
+        root = ET.XML(xml)
+        for i, child in enumerate(root):
+            for subchild in child:
+                record = {}
+                dup_tracker = 0
+                for p in subchild:
+                    record[p.tag] = p.text
+                    for o in p:
+                        if o.tag in record:
+                            dup_tracker += 1
+                            record[o.tag + '_%s' % dup_tracker] = o.text
+                        else:
+                            record[o.tag] = o.text
+                all_records.append(record)
+        return pd.DataFrame(all_records)
+
+    def get_all_scans(self, limit=1000, offset=1, status='FINISHED'):
+        qualys_api_limit = limit
+        dataframes = []
+        _records = []
+        try:
+            total = int(self.get_was_scan_count(status=status))
+            self.logger.error('Already have WAS scan count')
+            self.logger.info('Retrieving information for {} scans'.format(total))
+            for i in range(0, total):
+                if i % limit == 0:
+                    if (total - i) < limit:
+                        qualys_api_limit = total - i
+                    self.logger.info('Making a request with a limit of {} at offset {}'.format((str(qualys_api_limit)), str(i + 1)))
+                    scan_info = self.get_scan_info(limit=qualys_api_limit, offset=i + 1, status=status)
+                    _records.append(scan_info)
+            self.logger.debug('Converting XML to DataFrame')
+            dataframes = [self.xml_parser(xml) for xml in _records]
+        except Exception as e:
+            self.logger.error("Couldn't process all scans: {}".format(e))
+
+        return pd.concat(dataframes, axis=0).reset_index().drop('index', axis=1)
+
+    ####
+    #### CREATE VULNERABILITY REPORT AND DOWNLOAD IT
+    ####
+
+    def get_report_status(self, report_id):
+        return self.qgc.request(self.REPORT_STATUS.format(report_id=report_id))
+
+    def download_report(self, report_id):
+        return self.qgc.request(self.REPORT_DOWNLOAD.format(report_id=report_id))
+
+    def generate_scan_report_XML(self, scan_id):
+        """Generates a CSV report for an asset based on template defined in .ini file"""
+        report_xml = E.ServiceRequest(
+            E.data(
+                E.Report(
+                    E.name('<![CDATA[API Scan Report generated by VulnWhisperer]]>'),
+                    E.description('<![CDATA[CSV Scanning report for VulnWhisperer]]>'),
+                    E.format('CSV'),
+                    #type is not needed, as the template already has it
+                    E.type('WAS_SCAN_REPORT'),
+                    E.template(
+                        E.id(self.template_id)
+                    ),
+                    E.config(
+                        E.scanReport(
+                            E.target(
+                                E.scans(
+                                    E.WasScan(
+                                        E.id(scan_id)
+                                    )
+                                ),
+                            ),
+                        ),
+                    )
+                )
+            )
+        )
+        return report_xml
+
+    def create_report(self, report_id, kind='scan'):
+        mapper = {'scan': self.generate_scan_report_XML}
+        try:
+            data = mapper[kind](report_id)
+        except Exception as e:
+            self.logger.error('Error creating report: {}'.format(str(e)))
+        return self.qgc.request(self.REPORT_CREATE, data).encode('utf-8')
+
+    def delete_report(self, report_id):
+        return self.qgc.request(self.DELETE_REPORT.format(report_id=report_id))
+
+
+class qualysReportFields:
+    CATEGORIES = ['VULNERABILITY',
+                  'SENSITIVECONTENT',
+                  'INFORMATION_GATHERED']
+
+    # URL Vulnerability Information
+
+    VULN_BLOCK = [
+        CATEGORIES[0],
+        'ID',
+        'QID',
+        'Url',
+        'Param',
+        'Function',
+        'Form Entry Point',
+        'Access Path',
+        'Authentication',
+        'Ajax Request',
+        'Ajax Request ID',
+        'Ignored',
+        'Ignore Reason',
+        'Ignore Date',
+        'Ignore User',
+        'Ignore Comments',
+        'First Time Detected',
+        'Last Time Detected',
+        'Last Time Tested',
+        'Times Detected',
+        'Payload #1',
+        'Request Method #1',
+        'Request URL #1',
+        'Request Headers #1',
+        'Response #1',
+        'Evidence #1',
+    ]
+
+    INFO_HEADER = [
+        'Vulnerability Category',
+        'ID',
+        'QID',
+        'Response #1',
+        'Last Time Detected',
+    ]
+    INFO_BLOCK = [
+        CATEGORIES[2],
+        'ID',
+        'QID',
+        'Results',
+        'Detection Date',
+    ]
+
+    QID_HEADER = [
+        'QID',
+        'Id',
+        'Title',
+        'Category',
+        'Severity Level',
+        'Groups',
+        'OWASP',
+        'WASC',
+        'CWE',
+        'CVSS Base',
+        'CVSS Temporal',
+        'Description',
+        'Impact',
+        'Solution',
+    ]
+    GROUP_HEADER = ['GROUP', 'Name', 'Category']
+    OWASP_HEADER = ['OWASP', 'Code', 'Name']
+    WASC_HEADER = ['WASC', 'Code', 'Name']
+    SCAN_META = ['Web Application Name', 'URL', 'Owner', 'Scope', 'Operating System']
+    CATEGORY_HEADER = ['Category', 'Severity', 'Level', 'Description']
+
+
+class qualysUtils:
+    def __init__(self):
+        self.logger = logging.getLogger('qualysUtils')
+
+    def grab_section(
+            self,
+            report,
+            section,
+            end=[],
+            pop_last=False,
+    ):
+        temp_list = []
+        max_col_count = 0
+        with open(report, 'rb') as csvfile:
+            q_report = csv.reader(csvfile, delimiter=',', quotechar='"')
+            for line in q_report:
+                if set(line) == set(section):
+                    break
+
+            # Reads text until the end of the block:
+            for line in q_report:  # This keeps reading the file
+                temp_list.append(line)
+
+                if line in end:
+                    break
+            if pop_last and len(temp_list) > 1:
+                temp_list.pop(-1)
+        return temp_list
+
+    def iso_to_epoch(self, dt):
+        return dp.parse(dt).strftime('%s')
+
+    def cleanser(self, _data):
+        repls = (('\n', '|||'), ('\r', '|||'), (',', ';'), ('\t', '|||'))
+        if _data:
+            _data = reduce(lambda a, kv: a.replace(*kv), repls, str(_data))
+        return _data
+
+class qualysScanReport:
+    # URL Vulnerability Information
+    WEB_SCAN_VULN_BLOCK = list(qualysReportFields.VULN_BLOCK)
+    WEB_SCAN_VULN_BLOCK.insert(WEB_SCAN_VULN_BLOCK.index('QID'), 'Detection ID')
+
+    WEB_SCAN_VULN_HEADER = list(WEB_SCAN_VULN_BLOCK)
+    WEB_SCAN_VULN_HEADER[WEB_SCAN_VULN_BLOCK.index(qualysReportFields.CATEGORIES[0])] = \
+        'Vulnerability Category'
+
+    WEB_SCAN_SENSITIVE_HEADER = list(WEB_SCAN_VULN_HEADER)
+    WEB_SCAN_SENSITIVE_HEADER.insert(WEB_SCAN_SENSITIVE_HEADER.index('Url'
+                                                                     ), 'Content')
+
+    WEB_SCAN_SENSITIVE_BLOCK = list(WEB_SCAN_SENSITIVE_HEADER)
+    WEB_SCAN_SENSITIVE_BLOCK.insert(WEB_SCAN_SENSITIVE_BLOCK.index('QID'), 'Detection ID')
+    WEB_SCAN_SENSITIVE_BLOCK[WEB_SCAN_SENSITIVE_BLOCK.index('Vulnerability Category'
+                                                            )] = qualysReportFields.CATEGORIES[1]
+
+    WEB_SCAN_INFO_HEADER = list(qualysReportFields.INFO_HEADER)
+    WEB_SCAN_INFO_HEADER.insert(WEB_SCAN_INFO_HEADER.index('QID'), 'Detection ID')
+
+    WEB_SCAN_INFO_BLOCK = list(qualysReportFields.INFO_BLOCK)
+    WEB_SCAN_INFO_BLOCK.insert(WEB_SCAN_INFO_BLOCK.index('QID'), 'Detection ID')
+
+    QID_HEADER = list(qualysReportFields.QID_HEADER)
+    GROUP_HEADER = list(qualysReportFields.GROUP_HEADER)
+    OWASP_HEADER = list(qualysReportFields.OWASP_HEADER)
+    WASC_HEADER = list(qualysReportFields.WASC_HEADER)
+    SCAN_META = list(qualysReportFields.SCAN_META)
+    CATEGORY_HEADER = list(qualysReportFields.CATEGORY_HEADER)
+
+    def __init__(
+            self,
+            config=None,
+            file_in=None,
+            file_stream=False,
+            delimiter=',',
+            quotechar='"',
+    ):
+        self.logger = logging.getLogger('qualysScanReport')
+        self.file_in = file_in
+        self.file_stream = file_stream
+        self.report = None
+        self.utils = qualysUtils()
+
+        if config:
+            try:
+                self.qw = qualysWhisperAPI(config=config)
+            except Exception as e:
+                self.logger.error('Could not load config! Please check settings. Error: {}'.format(str(e)))
+
+        if file_stream:
+            self.open_file = file_in.splitlines()
+        elif file_in:
+
+            self.open_file = open(file_in, 'rb')
+
+        self.downloaded_file = None
+
+    def grab_sections(self, report):
+        all_dataframes = []
+        dict_tracker = {}
+        with open(report, 'rb') as csvfile:
+            dict_tracker['WEB_SCAN_VULN_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
+                                                                                       self.WEB_SCAN_VULN_BLOCK,
+                                                                                       end=[
+                                                                                           self.WEB_SCAN_SENSITIVE_BLOCK,
+                                                                                           self.WEB_SCAN_INFO_BLOCK],
+                                                                                       pop_last=True),
+                                                               columns=self.WEB_SCAN_VULN_HEADER)
+            dict_tracker['WEB_SCAN_SENSITIVE_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
+                                                                                            self.WEB_SCAN_SENSITIVE_BLOCK,
+                                                                                            end=[
+                                                                                                self.WEB_SCAN_INFO_BLOCK,
+                                                                                                self.WEB_SCAN_SENSITIVE_BLOCK],
+                                                                                            pop_last=True),
+                                                                columns=self.WEB_SCAN_SENSITIVE_HEADER)
+            dict_tracker['WEB_SCAN_INFO_BLOCK'] = pd.DataFrame(self.utils.grab_section(report,
+                                                                                       self.WEB_SCAN_INFO_BLOCK,
+                                                                                       end=[self.QID_HEADER],
+                                                                                       pop_last=True),
+                                                                columns=self.WEB_SCAN_INFO_HEADER)
+            dict_tracker['QID_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
+                                                                              self.QID_HEADER,
+                                                                              end=[self.GROUP_HEADER],
+                                                                              pop_last=True),
+                                                                columns=self.QID_HEADER)
+            dict_tracker['GROUP_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
+                                                                                self.GROUP_HEADER,
+                                                                                end=[self.OWASP_HEADER],
+                                                                                pop_last=True),
+                                                                columns=self.GROUP_HEADER)
+            dict_tracker['OWASP_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
+                                                                                self.OWASP_HEADER,
+                                                                                end=[self.WASC_HEADER],
+                                                                                pop_last=True),
+                                                                columns=self.OWASP_HEADER)
+            dict_tracker['WASC_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
+                                                                               self.WASC_HEADER, end=[['APPENDIX']],
+                                                                               pop_last=True),
+                                                                columns=self.WASC_HEADER)
+
+            dict_tracker['SCAN_META'] = pd.DataFrame(self.utils.grab_section(report,
+                                                                             self.SCAN_META,
+                                                                             end=[self.CATEGORY_HEADER],
+                                                                             pop_last=True),
+                                                                columns=self.SCAN_META)
+
+            dict_tracker['CATEGORY_HEADER'] = pd.DataFrame(self.utils.grab_section(report,
+                                                                                   self.CATEGORY_HEADER),
+                                                                columns=self.CATEGORY_HEADER)
+            all_dataframes.append(dict_tracker)
+
+        return all_dataframes
+
+    def data_normalizer(self, dataframes):
+        """
+        Merge and clean data
+        :param dataframes:
+        :return:
+        """
+        df_dict = dataframes[0]
+        merged_df = pd.concat([df_dict['WEB_SCAN_VULN_BLOCK'], df_dict['WEB_SCAN_SENSITIVE_BLOCK'],
+                               df_dict['WEB_SCAN_INFO_BLOCK']], axis=0,
+                              ignore_index=False)
+        merged_df = pd.merge(merged_df, df_dict['QID_HEADER'], left_on='QID',
+                             right_on='Id')
+
+        if 'Content' not in merged_df:
+            merged_df['Content'] = ''
+
+        columns_to_cleanse = ['Payload #1', 'Request Method #1', 'Request URL #1',
+                              'Request Headers #1', 'Response #1', 'Evidence #1',
+                              'Description', 'Impact', 'Solution', 'Url', 'Content']
+
+        for col in columns_to_cleanse:
+            merged_df[col] = merged_df[col].apply(self.utils.cleanser)
+
+        merged_df = merged_df.drop(['QID_y', 'QID_x'], axis=1)
+        merged_df = merged_df.rename(columns={'Id': 'QID'})
+
+        merged_df = merged_df.assign(**df_dict['SCAN_META'].to_dict(orient='records')[0])
+
+        merged_df = pd.merge(merged_df, df_dict['CATEGORY_HEADER'], how='left', left_on=['Category', 'Severity Level'],
+                             right_on=['Category', 'Severity'], suffixes=('Severity', 'CatSev'))
+
+        merged_df = merged_df.replace('N/A', '').fillna('')
+
+        try:
+            merged_df = \
+                merged_df[~merged_df.Title.str.contains('Links Crawled|External Links Discovered')]
+        except Exception as e:
+            self.logger.error('Error normalizing: {}'.format(str(e)))
+        return merged_df
+
+    def download_file(self, path='', file_id=None):
+        report = self.qw.download_report(file_id)
+        filename = path + str(file_id) + '.csv'
+        file_out = open(filename, 'w')
+        for line in report.splitlines():
+            file_out.write(line + '\n')
+        file_out.close()
+        self.logger.info('File written to {}'.format(filename))
+        return filename
+
+    def process_data(self, path='', file_id=None, cleanup=True):
+        """Downloads a file from qualys and normalizes it"""
+
+        download_file = self.download_file(path=path, file_id=file_id)
+        self.logger.info('Downloading file ID: {}'.format(file_id))
+        report_data = self.grab_sections(download_file)
+        merged_data = self.data_normalizer(report_data)
+        merged_data.sort_index(axis=1, inplace=True)
+
+        return merged_data

vulnwhisp/reporting/jira_api.py

@@ -0,0 +1,669 @@
+import json
+import os
+from datetime import datetime, date, timedelta
+
+from jira import JIRA
+import requests
+import logging
+from bottle import template
+import re
+
+class JiraAPI(object):
+    def __init__(self, hostname=None, username=None, password=None, path="", debug=False, clean_obsolete=True, max_time_window=12, decommission_time_window=3):
+        self.logger = logging.getLogger('JiraAPI')
+        if debug:
+            self.logger.setLevel(logging.DEBUG)
+
+        if "https://" not in hostname:
+            hostname = "https://{}".format(hostname)
+        self.username = username
+        self.password = password
+        self.jira = JIRA(options={'server': hostname}, basic_auth=(self.username, self.password))
+        self.logger.info("Created vjira service for {}".format(hostname))
+        self.all_tickets = []
+        self.excluded_tickets = []
+        self.JIRA_REOPEN_ISSUE = "Reopen Issue"
+        self.JIRA_CLOSE_ISSUE = "Close Issue"
+        self.JIRA_RESOLUTION_OBSOLETE = "Obsolete"
+        self.JIRA_RESOLUTION_FIXED = "Fixed"
+        self.template_path = 'vulnwhisp/reporting/resources/ticket.tpl'
+        self.max_ips_ticket = 30
+        self.attachment_filename = "vulnerable_assets.txt"
+        self.max_time_tracking = max_time_window #in months
+        if path:
+            self.download_tickets(path)
+        else:
+            self.logger.warn("No local path specified, skipping Jira ticket download.")
+        self.max_decommission_time = decommission_time_window #in months
+        # [HIGIENE] close tickets older than 12 months as obsolete (max_time_window defined)
+        if clean_obsolete:
+            self.close_obsolete_tickets()
+        # deletes the tag "server_decommission" from those tickets closed <=3 months ago
+        self.decommission_cleanup()
+        
+        self.jira_still_vulnerable_comment = '''This ticket has been reopened due to the vulnerability not having been fixed (if multiple assets are affected, all need to be fixed; if the server is down, lastest known vulnerability might be the one reported).
+        - In the case of the team accepting the risk and wanting to close the ticket, please add the label "*risk_accepted*" to the ticket before closing it.
+        - If server has been decommissioned, please add the label "*server_decommission*" to the ticket before closing it.
+        - If when checking the vulnerability it looks like a false positive, _+please elaborate in a comment+_ and add the label "*false_positive*" before closing it; we will review it and report it to the vendor.
+        
+        If you have further doubts, please contact the Security Team.'''
+
+    def create_ticket(self, title, desc, project="IS", components=[], tags=[], attachment_contents = []):
+        labels = ['vulnerability_management']
+        for tag in tags:
+            labels.append(str(tag))
+
+        self.logger.info("Creating ticket for project {} title: {}".format(project, title[:20]))
+        self.logger.debug("project {} has a component requirement: {}".format(project, components))
+        project_obj = self.jira.project(project)
+        components_ticket = []
+        for component in components:
+            exists = False
+            for c in project_obj.components:
+                if component == c.name:
+                    self.logger.debug("resolved component name {} to id {}".format(c.name, c.id))
+                    components_ticket.append({ "id": c.id })
+                    exists=True
+            if not exists:
+                self.logger.error("Error creating Ticket: component {} not found".format(component))
+                return 0
+        
+        try:
+            new_issue = self.jira.create_issue(project=project,
+                                               summary=title,
+                                               description=desc,
+                                               issuetype={'name': 'Bug'},
+                                               labels=labels,
+                                               components=components_ticket)
+            
+            self.logger.info("Ticket {} created successfully".format(new_issue))
+            
+            if attachment_contents:
+                self.add_content_as_attachment(new_issue, attachment_contents)
+        
+        except Exception as e:
+            self.logger.error("Failed to create ticket on Jira Project '{}'. Error: {}".format(project, e))
+            new_issue = False
+        
+        return new_issue
+    
+    #Basic JIRA Metrics
+    def metrics_open_tickets(self, project=None):
+        jql = "labels= vulnerability_management and resolution = Unresolved" 
+        if project:
+            jql += " and (project='{}')".format(project)
+        self.logger.debug('Executing: {}'.format(jql)) 
+        return len(self.jira.search_issues(jql, maxResults=0))
+
+    def metrics_closed_tickets(self, project=None):
+        jql = "labels= vulnerability_management and NOT resolution = Unresolved AND created >=startOfMonth(-{})".format(self.max_time_tracking) 
+        if project:
+            jql += " and (project='{}')".format(project)
+        return len(self.jira.search_issues(jql, maxResults=0))
+
+    def sync(self, vulnerabilities, project, components=[]):
+        #JIRA structure of each vulnerability: [source, scan_name, title, diagnosis, consequence, solution, ips, risk, references]
+        self.logger.info("JIRA Sync started")
+
+        for vuln in vulnerabilities:
+            # JIRA doesn't allow labels with spaces, so making sure that the scan_name doesn't have spaces
+            # if it has, they will be replaced by "_"
+            if " " in  vuln['scan_name']:
+                vuln['scan_name'] = "_".join(vuln['scan_name'].split(" "))
+            
+            # we exclude from the vulnerabilities to report those assets that already exist with *risk_accepted*/*server_decommission*
+            vuln = self.exclude_accepted_assets(vuln)
+            
+            # make sure after exclusion of risk_accepted assets there are still assets
+            if vuln['ips']:
+                exists = False
+                to_update = False
+                ticketid = ""
+                ticket_assets = []
+                exists, to_update, ticketid, ticket_assets = self.check_vuln_already_exists(vuln)
+
+                if exists:
+                    # If ticket "resolved" -> reopen, as vulnerability is still existent
+                    self.reopen_ticket(ticketid=ticketid, comment=self.jira_still_vulnerable_comment)
+                    self.add_label(ticketid, vuln['risk'])
+                    continue
+                elif to_update:
+                    self.ticket_update_assets(vuln, ticketid, ticket_assets)
+                    self.add_label(ticketid, vuln['risk'])
+                    continue
+                attachment_contents = []
+                # if assets >30, add as attachment
+                # create local text file with assets, attach it to ticket
+                if len(vuln['ips']) > self.max_ips_ticket:
+                    attachment_contents = vuln['ips']
+                    vuln['ips'] = ["Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(assets = len(attachment_contents))]
+                try:
+                    tpl = template(self.template_path, vuln)
+                except Exception as e:
+                    self.logger.error('Exception templating: {}'.format(str(e)))
+                    return 0
+                self.create_ticket(title=vuln['title'], desc=tpl, project=project, components=components, tags=[vuln['source'], vuln['scan_name'], 'vulnerability', vuln['risk']], attachment_contents = attachment_contents)
+            else:
+                self.logger.info("Ignoring vulnerability as all assets are already reported in a risk_accepted ticket")
+        
+        self.close_fixed_tickets(vulnerabilities)
+        # we reinitialize so the next sync redoes the query with their specific variables
+        self.all_tickets = []
+        self.excluded_tickets = []
+        return True
+    
+    def exclude_accepted_assets(self, vuln):
+        # we want to check JIRA tickets with risk_accepted/server_decommission or false_positive labels sharing the same source
+        # will exclude tickets older than 12 months, old tickets will get closed for higiene and recreated if still vulnerable
+        labels = [vuln['source'], vuln['scan_name'], 'vulnerability_management', 'vulnerability'] 
+        
+        if not self.excluded_tickets:
+            jql = "{} AND labels in (risk_accepted,server_decommission, false_positive) AND NOT labels=advisory AND created >=startOfMonth(-{})".format(" AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
+            self.excluded_tickets = self.jira.search_issues(jql, maxResults=0)
+
+        title = vuln['title']
+        #WARNING: function IGNORES DUPLICATES, after finding a "duplicate" will just return it exists
+        #it wont iterate over the rest of tickets looking for other possible duplicates/similar issues
+        self.logger.info("Comparing vulnerability to risk_accepted tickets")
+        assets_to_exclude = []
+        tickets_excluded_assets = []
+        for index in range(len(self.excluded_tickets)):
+            checking_ticketid, checking_title, checking_assets = self.ticket_get_unique_fields(self.excluded_tickets[index])
+            if title.encode('ascii') == checking_title.encode('ascii'):
+                if checking_assets:
+                    #checking_assets is a list, we add to our full list for later delete all assets
+                    assets_to_exclude+=checking_assets
+                    tickets_excluded_assets.append(checking_ticketid)
+       
+        if assets_to_exclude:
+            assets_to_remove = []
+            self.logger.warn("Vulnerable Assets seen on an already existing risk_accepted Jira ticket: {}".format(', '.join(tickets_excluded_assets)))
+            self.logger.debug("Original assets: {}".format(vuln['ips']))
+            #assets in vulnerability have the structure "ip - hostname - port", so we need to match by partial 
+            for exclusion in assets_to_exclude:
+                # for efficiency, we walk the backwards the array of ips from the scanners, as we will be popping out the matches 
+                # and we don't want it to affect the rest of the processing (otherwise, it would miss the asset right after the removed one)
+                for index in range(len(vuln['ips']))[::-1]:
+                    if exclusion == vuln['ips'][index].split(" - ")[0]:
+                        self.logger.debug("Deleting asset {} from vulnerability {}, seen in risk_accepted.".format(vuln['ips'][index], title))
+                        vuln['ips'].pop(index)
+            self.logger.debug("Modified assets: {}".format(vuln['ips']))
+
+        return vuln
+
+    def check_vuln_already_exists(self, vuln):
+        '''
+        This function compares a vulnerability with a collection of tickets.
+        Returns [exists (bool), is equal (bool), ticketid (str), assets (array)]
+        '''
+        # we need to return if the vulnerability has already been reported and the ID of the ticket for further processing
+        #function returns array [duplicated(bool), update(bool), ticketid, ticket_assets]
+        title = vuln['title']
+        labels = [vuln['source'], vuln['scan_name'], 'vulnerability_management', 'vulnerability'] 
+        #list(set()) to remove duplicates
+        assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", ",".join(vuln['ips']))))
+        
+        if not self.all_tickets:
+            self.logger.info("Retrieving all JIRA tickets with the following tags {}".format(labels))
+            # we want to check all JIRA tickets, to include tickets moved to other queues
+            # will exclude tickets older than 12 months, old tickets will get closed for higiene and recreated if still vulnerable
+            jql = "{} AND NOT labels=advisory AND created >=startOfMonth(-{})".format(" AND ".join(["labels={}".format(label) for label in labels]), self.max_time_tracking)
+            
+            self.all_tickets = self.jira.search_issues(jql, maxResults=0)
+        
+        #WARNING: function IGNORES DUPLICATES, after finding a "duplicate" will just return it exists
+        #it wont iterate over the rest of tickets looking for other possible duplicates/similar issues
+        self.logger.info("Comparing Vulnerabilities to created tickets")
+        for index in range(len(self.all_tickets)):
+            checking_ticketid, checking_title, checking_assets = self.ticket_get_unique_fields(self.all_tickets[index])
+            # added "not risk_accepted", as if it is risk_accepted, we will create a new ticket excluding the accepted assets
+            if title.encode('ascii') == checking_title.encode('ascii') and not self.is_risk_accepted(self.jira.issue(checking_ticketid)): 
+                difference = list(set(assets).symmetric_difference(checking_assets))
+                #to check intersection - set(assets) & set(checking_assets)
+                if difference: 
+                    self.logger.info("Asset mismatch, ticket to update. Ticket ID: {}".format(checking_ticketid))
+                    return False, True, checking_ticketid, checking_assets #this will automatically validate
+                else:
+                    self.logger.info("Confirmed duplicated. TickedID: {}".format(checking_ticketid))
+                    return True, False, checking_ticketid, [] #this will automatically validate
+        return False, False, "", []
+
+    def ticket_get_unique_fields(self, ticket):
+        title = ticket.raw.get('fields', {}).get('summary').encode("ascii").strip()
+        ticketid = ticket.key.encode("ascii")
+
+        assets = self.get_assets_from_description(ticket)
+        if not assets:
+            #check if attachment, if so, get assets from attachment
+            assets = self.get_assets_from_attachment(ticket)
+                
+        return ticketid, title, assets
+
+    def get_assets_from_description(self, ticket, _raw = False):
+        # Get the assets as a string "host - protocol/port - hostname" separated by "\n"
+        # structure the text to have the same structure as the assets from the attachment
+        affected_assets = ""
+        try:
+            affected_assets = ticket.raw.get('fields', {}).get('description').encode("ascii").split("{panel:title=Affected Assets}")[1].split("{panel}")[0].replace('\n','').replace(' * ','\n').replace('\n', '', 1)
+        except Exception as e:
+            self.logger.error("Unable to process the Ticket's 'Affected Assets'. Ticket ID: {}. Reason: {}".format(ticket, e))
+
+        if affected_assets:
+            if _raw:
+                # from line 406 check if the text in the panel corresponds to having added an attachment
+                if "added as an attachment" in affected_assets:
+                    return False
+                return affected_assets
+
+            try:
+                # if _raw is not true, we return only the IPs of the affected assets
+                return list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", affected_assets)))
+            except Exception as e:
+                self.logger.error("Ticket IPs regex failed. Ticket ID: {}. Reason: {}".format(ticket, e))
+        return False
+
+    def get_assets_from_attachment(self, ticket, _raw = False):
+        # Get the assets as a string "host - protocol/port - hostname" separated by "\n"
+        affected_assets = []
+        try:
+            fields = self.jira.issue(ticket.key).raw.get('fields', {})
+            attachments = fields.get('attachment', {})
+            affected_assets = ""
+            #we will make sure we get the latest version of the file
+            latest = ''
+            attachment_id = ''
+            if attachments:
+                for item in attachments:
+                    if item.get('filename') == self.attachment_filename:
+                        if not latest:
+                            latest = item.get('created')
+                            attachment_id = item.get('id') 
+                        else:
+                            if latest < item.get('created'):
+                                latest = item.get('created')         
+                                attachment_id = item.get('id') 
+            affected_assets = self.jira.attachment(attachment_id).get()
+
+        except Exception as e:
+            self.logger.error("Failed to get assets from ticket attachment. Ticket ID: {}. Reason: {}".format(ticket, e))
+
+        if affected_assets:
+            if _raw:
+                return affected_assets
+
+            try:
+                # if _raw is not true, we return only the IPs of the affected assets
+                affected_assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", affected_assets)))
+                return affected_assets
+            except Exception as e:
+                self.logger.error("Ticket IPs Attachment regex failed. Ticket ID: {}. Reason: {}".format(ticket, e))
+
+        return False
+
+    def parse_asset_to_json(self, asset):
+        hostname, protocol, port = "", "", ""
+        asset_info = asset.split(" - ")
+        ip = asset_info[0]
+        proto_port = asset_info[1]
+        # in case there is some case where hostname is not reported at all
+        if len(asset_info) == 3:
+            hostname = asset_info[2]
+        if proto_port != "N/A/N/A":
+            protocol, port = proto_port.split("/")
+            port = int(float(port))
+
+        asset_dict = {
+            "host": ip,
+            "protocol": protocol,
+            "port": port,
+            "hostname": hostname
+        }
+
+        return asset_dict
+
+    def clean_old_attachments(self, ticket):
+        fields = ticket.raw.get('fields')
+        attachments = fields.get('attachment')
+        if attachments:
+            for item in attachments:
+                if item.get('filename') == self.attachment_filename:
+                    self.jira.delete_attachment(item.get('id'))
+
+    def add_content_as_attachment(self, issue, contents):
+        try:
+            #Create the file locally with the data
+            attachment_file = open(self.attachment_filename, "w")
+            attachment_file.write("\n".join(contents))
+            attachment_file.close()
+            #Push the created file to the ticket
+            attachment_file = open(self.attachment_filename, "rb")
+            self.jira.add_attachment(issue, attachment_file, self.attachment_filename)
+            attachment_file.close()
+            #remove the temp file
+            os.remove(self.attachment_filename)
+            self.logger.info("Added attachment successfully.")
+        except:
+            self.logger.error("Error while attaching file to ticket.")
+            return False
+
+        return True
+
+    def get_ticket_reported_assets(self, ticket):
+        #[METRICS] return a list with all the affected assets for that vulnerability (including already resolved ones) 
+        return list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b",str(self.jira.issue(ticket).raw))))
+
+    def get_resolution_time(self, ticket):
+        #get time a ticket took to be resolved
+        ticket_obj = self.jira.issue(ticket)
+        if self.is_ticket_resolved(ticket_obj):
+            ticket_data = ticket_obj.raw.get('fields')
+            #dates follow format '2018-11-06T10:36:13.849+0100'
+            created = [int(x) for x in ticket_data['created'].split('.')[0].replace('T', '-').replace(':','-').split('-')]
+            resolved =[int(x) for x in ticket_data['resolutiondate'].split('.')[0].replace('T', '-').replace(':','-').split('-')]
+            
+            start = datetime(created[0],created[1],created[2],created[3],created[4],created[5])
+            end = datetime(resolved[0],resolved[1],resolved[2],resolved[3],resolved[4],resolved[5])
+            return (end-start).days
+        else:
+            self.logger.error("Ticket {ticket} is not resolved, can't calculate resolution time".format(ticket=ticket))
+
+        return False
+
+    def ticket_update_assets(self, vuln, ticketid, ticket_assets):
+        # correct description will always be in the vulnerability to report, only needed to update description to new one
+        self.logger.info("Ticket {} exists, UPDATE requested".format(ticketid))
+        
+        #for now, if a vulnerability has been accepted ('accepted_risk'), ticket is completely ignored and not updated (no new assets)
+
+        #TODO when vulnerability accepted, create a new ticket with only the non-accepted vulnerable assets
+        #this would require go through the downloaded tickets, check duplicates/accepted ones, and if so,
+        #check on their assets to exclude them from the new ticket
+        risk_accepted = False
+        ticket_obj = self.jira.issue(ticketid)
+        if self.is_ticket_resolved(ticket_obj):
+            if self.is_risk_accepted(ticket_obj):
+                return 0
+            self.reopen_ticket(ticketid=ticketid, comment=self.jira_still_vulnerable_comment)
+        
+        #First will do the comparison of assets
+        ticket_obj.update()
+        assets = list(set(re.findall(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", ",".join(vuln['ips']))))
+        difference = list(set(assets).symmetric_difference(ticket_assets))
+        
+        comment = ''
+        added = ''
+        removed = ''
+        #put a comment with the assets that have been added/removed
+        for asset in difference:
+            if asset in assets:
+                if not added:
+                    added = '\nThe following assets *have been newly detected*:\n'
+                added += '* {}\n'.format(asset)
+            elif asset in ticket_assets:
+                if not removed:
+                    removed= '\nThe following assets *have been resolved*:\n'
+                removed += '* {}\n'.format(asset)
+
+        comment = added + removed
+        
+        #then will check if assets are too many that need to be added as an attachment
+        attachment_contents = []
+        if len(vuln['ips']) > self.max_ips_ticket:
+            attachment_contents = vuln['ips']
+            vuln['ips'] = ["Affected hosts ({assets}) exceed Jira's allowed character limit, added as an attachment.".format(assets = len(attachment_contents))]
+        
+        #fill the ticket description template
+        try:
+            tpl = template(self.template_path, vuln)
+        except Exception as e:
+            self.logger.error('Exception updating assets: {}'.format(str(e)))
+            return 0
+
+        #proceed checking if it requires adding as an attachment
+        try:
+            #update attachment with hosts and delete the old versions
+            if attachment_contents:
+                self.clean_old_attachments(ticket_obj)
+                self.add_content_as_attachment(ticket_obj, attachment_contents)
+                
+            ticket_obj.update(description=tpl, comment=comment, fields={"labels":ticket_obj.fields.labels})
+            self.logger.info("Ticket {} updated successfully".format(ticketid))
+            self.add_label(ticketid, 'updated')
+        except Exception as e:
+            self.logger.error("Error while trying up update ticket {ticketid}.\nReason: {e}".format(ticketid = ticketid, e=e))
+        return 0
+
+    def add_label(self, ticketid, label):
+        ticket_obj = self.jira.issue(ticketid)
+        
+        if label not in [x.encode('utf8') for x in ticket_obj.fields.labels]:
+            ticket_obj.fields.labels.append(label)
+        
+            try:
+                ticket_obj.update(fields={"labels":ticket_obj.fields.labels})
+                self.logger.info("Added label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
+            except:
+                self.logger.error("Error while trying to add label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
+        
+        return 0
+
+    def remove_label(self, ticketid, label):
+        ticket_obj = self.jira.issue(ticketid)
+        
+        if label in [x.encode('utf8') for x in ticket_obj.fields.labels]:
+            ticket_obj.fields.labels.remove(label)
+        
+            try:
+                ticket_obj.update(fields={"labels":ticket_obj.fields.labels})
+                self.logger.info("Removed label {label} from ticket {ticket}".format(label=label, ticket=ticketid))
+            except:
+                self.logger.error("Error while trying to remove label {label} to ticket {ticket}".format(label=label, ticket=ticketid))
+        else:
+            self.logger.error("Error: label {label} not in ticket {ticket}".format(label=label, ticket=ticketid))
+        
+        return 0
+
+    def close_fixed_tickets(self, vulnerabilities):
+        '''
+        Close tickets which vulnerabilities have been resolved and are still open.
+        Higiene clean up affects to all tickets created by the module, filters by label 'vulnerability_management'
+        '''
+        found_vulns = []
+        for vuln in vulnerabilities:
+            found_vulns.append(vuln['title'])
+
+        comment = '''This ticket is being closed as it appears that the vulnerability no longer exists.
+        If the vulnerability reappears, a new ticket will be opened.'''
+
+        for ticket in self.all_tickets:
+            if ticket.raw['fields']['summary'].strip() in found_vulns:
+                self.logger.info("Ticket {} is still vulnerable".format(ticket))
+                continue
+            self.logger.info("Ticket {} is no longer vulnerable".format(ticket))
+            self.close_ticket(ticket, self.JIRA_RESOLUTION_FIXED, comment) 
+        return 0
+
+
+    def is_ticket_reopenable(self, ticket_obj):
+        transitions = self.jira.transitions(ticket_obj)
+        for transition in transitions:
+            if transition.get('name') == self.JIRA_REOPEN_ISSUE:
+                self.logger.debug("Ticket is reopenable")
+                return True
+        self.logger.error("Ticket {} can't be opened. Check Jira transitions.".format(ticket_obj))
+        return False
+
+    def is_ticket_closeable(self, ticket_obj):
+        transitions = self.jira.transitions(ticket_obj)
+        for transition in transitions:
+            if transition.get('name') == self.JIRA_CLOSE_ISSUE:
+                return True
+        self.logger.error("Ticket {} can't closed. Check Jira transitions.".format(ticket_obj))
+        return False
+
+    def is_ticket_resolved(self, ticket_obj):
+        #Checks if a ticket is resolved or not
+        if ticket_obj is not None:
+            if ticket_obj.raw['fields'].get('resolution') is not None:
+                if ticket_obj.raw['fields'].get('resolution').get('name') != 'Unresolved':
+                    self.logger.debug("Checked ticket {} is already closed".format(ticket_obj))
+                    self.logger.info("Ticket {} is closed".format(ticket_obj))
+                    return True
+        self.logger.debug("Checked ticket {} is already open".format(ticket_obj))
+        return False
+
+
+    def is_risk_accepted(self, ticket_obj):
+        if ticket_obj is not None:
+            if ticket_obj.raw['fields'].get('labels') is not None:
+                labels = ticket_obj.raw['fields'].get('labels')
+                if "risk_accepted" in labels:
+                    self.logger.warn("Ticket {} accepted risk, will be ignored".format(ticket_obj))
+                    return True
+                elif "server_decommission" in labels:
+                    self.logger.warn("Ticket {} server decommissioned, will be ignored".format(ticket_obj))
+                    return True
+                elif "false_positive" in labels:
+                    self.logger.warn("Ticket {} flagged false positive, will be ignored".format(ticket_obj))
+                    return True
+        self.logger.info("Ticket {} risk has not been accepted".format(ticket_obj))
+        return False
+
+    def reopen_ticket(self, ticketid, ignore_labels=False, comment=""):
+        self.logger.debug("Ticket {} exists, REOPEN requested".format(ticketid))
+        # this will reopen a ticket by ticketid
+        ticket_obj = self.jira.issue(ticketid)
+        
+        if self.is_ticket_resolved(ticket_obj):
+            if (not self.is_risk_accepted(ticket_obj) or ignore_labels):
+                try:
+                    if self.is_ticket_reopenable(ticket_obj):
+                        error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_REOPEN_ISSUE, comment = comment)
+                        self.logger.info("Ticket {} reopened successfully".format(ticketid))
+                        if not ignore_labels:
+                            self.add_label(ticketid, 'reopened')
+                        return 1
+                except Exception as e:
+                    # continue with ticket data so that a new ticket is created in place of the "lost" one
+                    self.logger.error("error reopening ticket {}: {}".format(ticketid, e))
+                    return 0
+        return 0
+
+    def close_ticket(self, ticketid, resolution, comment):
+        # this will close a ticket by ticketid
+        self.logger.debug("Ticket {} exists, CLOSE requested".format(ticketid))
+        ticket_obj = self.jira.issue(ticketid)
+        if not self.is_ticket_resolved(ticket_obj):
+            try:
+                if self.is_ticket_closeable(ticket_obj):
+                    #need to add the label before closing the ticket
+                    self.add_label(ticketid, 'closed')
+                    error = self.jira.transition_issue(issue=ticketid, transition=self.JIRA_CLOSE_ISSUE, comment = comment, resolution = {"name": resolution })
+                    self.logger.info("Ticket {} closed successfully".format(ticketid))
+                    return 1
+            except Exception as e:
+                # continue with ticket data so that a new ticket is created in place of the "lost" one
+                self.logger.error("error closing ticket {}: {}".format(ticketid, e))
+                return 0
+                
+        return 0
+
+    def close_obsolete_tickets(self):
+        # Close tickets older than 12 months, vulnerabilities not solved will get created a new ticket 
+        self.logger.info("Closing obsolete tickets older than {} months".format(self.max_time_tracking))
+        jql = "labels=vulnerability_management AND NOT labels=advisory AND created <startOfMonth(-{}) and resolution=Unresolved".format(self.max_time_tracking)
+        tickets_to_close = self.jira.search_issues(jql, maxResults=0)
+        
+        comment = '''This ticket is being closed for hygiene, as it is more than {} months old.
+        If the vulnerability still exists, a new ticket will be opened.'''.format(self.max_time_tracking)
+        
+        for ticket in tickets_to_close:
+                self.close_ticket(ticket, self.JIRA_RESOLUTION_OBSOLETE, comment)
+        
+        return 0
+
+    def project_exists(self, project):
+        try:
+            self.jira.project(project)
+            return True
+        except:
+            return False
+        return False
+
+    def download_tickets(self, path):
+        '''
+        saves all tickets locally, local snapshot of vulnerability_management ticktes
+        '''
+        #check if file already exists
+        check_date = str(date.today())
+        fname = '{}jira_{}.json'.format(path, check_date) 
+        if os.path.isfile(fname):
+            self.logger.info("File {} already exists, skipping ticket download".format(fname))
+            return True
+        try:
+            self.logger.info("Saving locally tickets from the last {} months".format(self.max_time_tracking))
+            jql = "labels=vulnerability_management AND NOT labels=advisory AND created >=startOfMonth(-{})".format(self.max_time_tracking)
+            tickets_data = self.jira.search_issues(jql, maxResults=0)
+
+            #TODO process tickets, creating a new field called "_metadata" with all the affected assets well structured
+            # for future processing in ELK/Splunk; this includes downloading attachments with assets and processing them
+
+            processed_tickets = []
+
+            for ticket in tickets_data:
+                assets = self.get_assets_from_description(ticket, _raw=True)
+                if not assets:
+                    # check if attachment, if so, get assets from attachment
+                    assets = self.get_assets_from_attachment(ticket, _raw=True)
+                # process the affected assets to save them as json structure on a new field from the JSON
+                _metadata = {"affected_hosts": []}
+                if assets:
+                    if "\n" in assets:
+                        for asset in assets.split("\n"):
+                            assets_json = self.parse_asset_to_json(asset)
+                            _metadata["affected_hosts"].append(assets_json)
+                    else:
+                        assets_json = self.parse_asset_to_json(assets)
+                        _metadata["affected_hosts"].append(assets_json)
+
+
+                temp_ticket = ticket.raw.get('fields')
+                temp_ticket['_metadata'] = _metadata
+
+                processed_tickets.append(temp_ticket)
+            
+            #end of line needed, as writelines() doesn't add it automatically, otherwise one big line
+            to_save = [json.dumps(ticket.raw.get('fields'))+"\n" for ticket in tickets_data]
+            with open(fname, 'w') as outfile:
+                outfile.writelines(to_save)
+                self.logger.info("Tickets saved succesfully.")
+            
+            return True
+
+        except Exception as e:
+            self.logger.error("Tickets could not be saved locally: {}.".format(e))
+
+        return False 
+
+    def decommission_cleanup(self):
+        '''
+        deletes the server_decomission tag from those tickets that have been 
+        closed already for more than x months (default is 3 months) in order to clean solved issues
+        for statistics purposes
+        '''
+        self.logger.info("Deleting 'server_decommission' tag from tickets closed more than {} months ago".format(self.max_decommission_time))
+
+        jql = "labels=vulnerability_management AND labels=server_decommission and resolutiondate <=startOfMonth(-{})".format(self.max_decommission_time)
+        decommissioned_tickets = self.jira.search_issues(jql, maxResults=0)
+        
+        comment = '''This ticket is having deleted the *server_decommission* tag, as it is more than {} months old and is expected to already have been decommissioned.
+        If that is not the case and the vulnerability still exists, the vulnerability will be opened again.'''.format(self.max_decommission_time)
+        
+        for ticket in decommissioned_tickets:
+            #we open first the ticket, as we want to make sure the process is not blocked due to 
+            #an unexisting jira workflow or unallowed edit from closed tickets
+            self.reopen_ticket(ticketid=ticket, ignore_labels=True)
+            self.remove_label(ticket, 'server_decommission')
+            self.close_ticket(ticket, self.JIRA_RESOLUTION_FIXED, comment)
+        
+        return 0

vulnwhisp/reporting/resources/ticket.tpl

@@ -0,0 +1,34 @@
+{panel:title=Description}
+{{ !diagnosis}}
+{panel}
+
+
+{panel:title=Consequence}
+{{ !consequence}}
+{panel}
+
+{panel:title=Solution}
+{{ !solution}}
+{panel}
+
+{panel:title=Affected Assets}
+% for ip in ips:
+ * {{ip}}
+% end
+{panel}
+
+{panel:title=References}
+% for ref in references:
+ * {{ref}}
+% end
+{panel}
+
+
+
+Please do not delete or modify the ticket assigned tags or title, as they are used to be synced. If the ticket ceases to be recognised, a new ticket will raise.
+
+In the case of the team accepting the risk and wanting to close the ticket, please add the label "*risk_accepted*" to the ticket before closing it.
+
+If server has been decommissioned, please add the label "*server_decommission*" to the ticket before closing it.
+
+If when checking the vulnerability it looks like a false positive, _+please elaborate in a comment+_ and add the label "*false_positive*" before closing it; we will review it and report it to the vendor.

vulnwhisp/test/mock.py

@@ -0,0 +1,76 @@
+import os
+import logging
+import httpretty
+
+
+class mockAPI(object):
+    def __init__(self, mock_dir=None, debug=False):
+        self.mock_dir = mock_dir
+
+        if not self.mock_dir:
+            # Try to guess the mock_dir if python setup.py develop was used
+            self.mock_dir = '/'.join(__file__.split('/')[:-3]) + '/tests/data'
+
+        self.logger = logging.getLogger('mockAPI')
+        if debug:
+            self.logger.setLevel(logging.DEBUG)
+
+        self.logger.info('mockAPI initialised, API requests will be mocked')
+        self.logger.debug('Test path resolved as {}'.format(self.mock_dir))
+
+    def get_directories(self, path):
+        dir, subdirs, files = next(os.walk(path))
+        return subdirs
+
+    def get_files(self, path):
+        dir, subdirs, files = next(os.walk(path))
+        return files
+
+    def qualys_vuln_callback(self, request, uri, response_headers):
+        self.logger.debug('Simulating response for {} ({})'.format(uri, request.body))
+        if 'list' in request.parsed_body['action']:
+            return [200,
+                    response_headers,
+                    open('{}/{}'.format(self.qualys_vuln_path, 'scans')).read()]
+        elif 'fetch' in request.parsed_body['action']:
+            try:
+                response_body = open('{}/{}'.format(
+                                      self.qualys_vuln_path,
+                                      request.parsed_body['scan_ref'][0].replace('/', '_'))
+                                    ).read()
+            except:
+                # Can't find the file, just send an empty response
+                response_body = ''
+        return [200, response_headers, response_body]
+
+    def create_nessus_resource(self, framework):
+        for filename in self.get_files('{}/{}'.format(self.mock_dir, framework)):
+            method, resource = filename.split('_', 1)
+            resource = resource.replace('_', '/')
+            self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, method, resource))
+            httpretty.register_uri(
+                getattr(httpretty, method), 'https://{}:443/{}'.format(framework, resource),
+                body=open('{}/{}/{}'.format(self.mock_dir, framework, filename)).read()
+            )
+
+    def create_qualys_vuln_resource(self, framework):
+        # Create health check endpoint
+        self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, 'GET', 'msp/about.php'))
+        httpretty.register_uri(
+                httpretty.GET,
+                'https://{}:443/{}'.format(framework, 'msp/about.php'),
+                body='')
+
+        self.logger.debug('Adding mocked {} endpoint {} {}'.format(framework, 'POST', 'api/2.0/fo/scan'))
+        httpretty.register_uri(
+            httpretty.POST, 'https://{}:443/{}'.format(framework, 'api/2.0/fo/scan/'),
+            body=self.qualys_vuln_callback)
+
+    def mock_endpoints(self):
+        for framework in self.get_directories(self.mock_dir):
+            if framework in ['nessus', 'tenable']:
+                self.create_nessus_resource(framework)
+            elif framework == 'qualys_vuln':
+                self.qualys_vuln_path = self.mock_dir + '/' + framework
+                self.create_qualys_vuln_resource(framework)
+        httpretty.enable()

vulnwhisp/utils/cli.py

@@ -1,16 +0,0 @@
-class bcolors:
-    """
-    Utility to add colors to shell for scripts
-    """
-    HEADERS = '\033[95m'
-    OKBLUE = '\033[94m'
-    OKGREEN = '\033[92m'
-    WARNING = '\033[93m'
-    FAIL = '\033[91m'
-    ENDC = '\033[0m'
-    BOLD = '\033[1m'
-    UNDERLINE = '\033[4m'
-
-    INFO = '{info}[INFO]{endc}'.format(info=OKBLUE, endc=ENDC)
-    SUCCESS = '{green}[SUCCESS]{endc}'.format(green=OKGREEN, endc=ENDC)
-    FAIL = '{red}[FAIL]{endc}'.format(red=FAIL, endc=ENDC)