Merge branch 'docker-fixes' into feature-nessus-json-latest

* docker-fixes: (33 commits)
  fix test output
  set default path and fix restore
  Set limit to bail out on
  increase timeout for ES sync
  restore deleted files
  Test updates
  Move vulnwhisperer tests to a script
  standardise /tmp to /opt
  Update test
  Add docker test script
  Expose Logstash API port
  sudo chown
  fix
  Fix permissions for ES
  Fix build command
  Test travis docker
  update kibana objects to match template
  update index template
  Retry template installation a few times
  Add initial ELK6 index template
  ...

# Conflicts:
#	.travis.yml
#	configs/test.ini
#	docker-compose-test.yml
#	docker-compose.v6.yml
#	resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf
#	resources/elk6/init_kibana.sh
#	resources/elk6/pipeline/1000_nessus_process_file.conf
#	resources/elk6/pipeline/2000_qualys_web_scans.conf
#	resources/elk6/pipeline/3000_openvas.conf

.travis.yml

@@ -5,11 +5,20 @@ python:
     - 2.7
 env:
     - TEST_PATH=tests/data
+
+services:
+  - docker
 #    - 3.6
 #matrix:
 #    allow_failures:
 #        - python: 3.6 - Commenting out testing for Python 3.6 until ready
 
+before_install:
+    - mkdir -p ./data/esdata1
+    - mkdir -p ./data/es_snapshots
+    - sudo chown -R 1000:1000 ./data/es*
+    - docker build -t vulnwhisperer-local .
+    - docker-compose -f docker-compose-test.yml up -d
 install:
     - pip install -r requirements.txt
     - pip install flake8  # pytest  # add another testing frameworks later
@@ -20,26 +29,8 @@ before_script:
     - flake8 . --count --exit-zero --exclude=deps/qualysapi --max-complexity=10 --max-line-length=127 --statistics
 script:
     - python setup.py install
-    # Test successful scan download and parsing
+    - bash tests/test-vuln_whisperer.sh
-    - rm -rf /tmp/VulnWhisperer
+    - bash tests/test-docker.sh
-    - vuln_whisperer -F -c configs/test.ini --mock --mock_dir ${TEST_PATH}
-    # Run a second time with no scans to import
-    - rm -rf /tmp/VulnWhisperer/data/database
-    - vuln_whisperer -F -c configs/test.ini --mock --mock_dir ${TEST_PATH}
-    # Test one failed scan
-    - rm -rf /tmp/VulnWhisperer
-    - rm -f ${TEST_PATH}/nessus/GET_scans_exports_164_download
-    - vuln_whisperer -F -c configs/test.ini --mock --mock_dir ${TEST_PATH}; [[ $? -eq 1 ]]
-    # Test two failed scans
-    - rm -rf /tmp/VulnWhisperer
-    - rm -f ${TEST_PATH}/qualys_vuln/scan_1553941061.87241
-    - vuln_whisperer -F -c configs/test.ini --mock --mock_dir ${TEST_PATH}; [[ $? -eq 2 ]]
-    # Test only nessus
-    - rm -rf /tmp/VulnWhisperer
-    - vuln_whisperer -F -c configs/test.ini -s nessus --mock --mock_dir ${TEST_PATH}; [[ $? -eq 1 ]]
-    # Test only qualy_vuln
-    - rm -rf /tmp/VulnWhisperer
-    - vuln_whisperer -F -c configs/test.ini -s qualys_vuln --mock --mock_dir ${TEST_PATH}; [[ $? -eq 1 ]]
 notifications:
     on_success: change
     on_failure: change  # `always` will be the setting once code changes slow down

README.md

@@ -77,6 +77,12 @@ cd /path/to/VulnWhisperer
 python setup.py install
 ```
 
+**(Optional) If using a proxy, add proxy URL as environment variable to PATH**
+```shell
+export HTTP_PROXY=http://example.com:8080
+export HTTPS_PROXY=http://example.com:8080
+```
+
 Now you're ready to pull down scans. (see <a href="#run">run section</a>)
 
 Configuration

configs/test.ini

@@ -4,8 +4,8 @@ hostname=nessus
 port=443
 username=nessus_username
 password=nessus_password
-write_path=/tmp/VulnWhisperer/data/nessus/
+write_path=/opt/VulnWhisperer/data/nessus/
-db_path=/tmp/VulnWhisperer/data/database
+db_path=/opt/VulnWhisperer/data/database
 trash=false
 verbose=true
 
@@ -15,8 +15,8 @@ hostname=tenable
 port=443
 username=tenable.io_username
 password=tenable.io_password
-write_path=/tmp/VulnWhisperer/data/tenable/
+write_path=/opt/VulnWhisperer/data/tenable/
-db_path=/tmp/VulnWhisperer/data/database
+db_path=/opt/VulnWhisperer/data/database
 trash=false
 verbose=true
 
@@ -26,8 +26,8 @@ enabled = false
 hostname = qualys_web
 username = exampleuser
 password = examplepass
-write_path=/tmp/VulnWhisperer/data/qualys_web/
+write_path=/opt/VulnWhisperer/data/qualys_web/
-db_path=/tmp/VulnWhisperer/data/database
+db_path=/opt/VulnWhisperer/data/database
 verbose=true
 
 # Set the maximum number of retries each connection should attempt.
@@ -42,8 +42,8 @@ enabled = true
 hostname = qualys_vuln
 username = exampleuser
 password = examplepass
-write_path=/tmp/VulnWhisperer/data/qualys_vuln/
+write_path=/opt/VulnWhisperer/data/qualys_vuln/
-db_path=/tmp/VulnWhisperer/data/database
+db_path=/opt/VulnWhisperer/data/database
 verbose=true
 
 [detectify]
@@ -54,8 +54,8 @@ hostname = detectify
 username = exampleuser
 #password variable used as secretKey
 password = examplepass
-write_path =/tmp/VulnWhisperer/data/detectify/
+write_path =/opt/VulnWhisperer/data/detectify/
-db_path = /tmp/VulnWhisperer/data/database
+db_path = /opt/VulnWhisperer/data/database
 verbose = true
 
 [openvas]
@@ -64,8 +64,8 @@ hostname = openvas
 port = 4000
 username = exampleuser
 password = examplepass
-write_path=/tmp/VulnWhisperer/data/openvas/
+write_path=/opt/VulnWhisperer/data/openvas/
-db_path=/tmp/VulnWhisperer/data/database
+db_path=/opt/VulnWhisperer/data/database
 verbose=true
 
 [jira]
@@ -73,8 +73,8 @@ enabled = false
 hostname = jira-host
 username = username
 password = password
-write_path = /tmp/VulnWhisperer/data/jira/
+write_path = /opt/VulnWhisperer/data/jira/
-db_path = /tmp/VulnWhisperer/data/database
+db_path = /opt/VulnWhisperer/data/database
 verbose = true
 dns_resolv = False
 

docker-compose-test.yml

@@ -19,8 +19,8 @@ services:
         hard: 65536
     mem_limit: 8g
     volumes:
-      - ./docker_data/esdata1:/usr/share/elasticsearch/data
+      - ./data/esdata1:/usr/share/elasticsearch/data
-      - ./docker_data/es_snapshots:/snapshots
+      - ./data/es_snapshots:/snapshots
     ports:
       - 9200:9200
     #restart: always
@@ -39,8 +39,6 @@ services:
       - 5601:5601
     depends_on:
       - elasticsearch
-    volumes:
-      - ./docker_data/kibana_optimize:/usr/share/kibana/optimize
     networks:
       esnet:
         aliases:
@@ -64,20 +62,21 @@ services:
     container_name: logstash
     volumes:
       - ./resources/elk6/pipeline/:/usr/share/logstash/pipeline
-      - ./docker_data/data/:/opt/VulnWhisperer/data
+      - ./data/vulnwhisperer/:/opt/VulnWhisperer/data
-      - ./resources/elk6/logstash.yml:/usr/share/logstash/config/logstash.yml
+      # - ./resources/elk6/logstash.yml:/usr/share/logstash/config/logstash.yml
     environment:
       - xpack.monitoring.enabled=false
     depends_on:
       - elasticsearch
+    ports:
+      - 9600:9600
     networks:
       esnet:
         aliases:
           - logstash.local
-
   vulnwhisperer:
-    # image: vulnwhisperer-1.8
+    # image: hasecuritysolutions/vulnwhisperer:latest
-    image: vulnwhisperer-pemontto
+    image: vulnwhisperer-local
     container_name: vulnwhisperer
     entrypoint: [
                   "vuln_whisperer",
@@ -89,8 +88,8 @@ services:
                   "/tests/data"
     ]    
     volumes:
-      # - /opt/VulnWhisperer/data/:/opt/VulnWhisperer/data
+      - ./data/vulnwhisperer/:/opt/VulnWhisperer/data
-      - ./docker_data/data/:/opt/VulnWhisperer/data
+      # - ./resources/elk6/vulnwhisperer.ini:/opt/VulnWhisperer/vulnwhisperer.ini
       - ./configs/test.ini:/opt/VulnWhisperer/vulnwhisperer.ini
       - ./tests/data/:/tests/data
     network_mode: host

docker-compose.v6.yml

@@ -8,7 +8,6 @@ services:
       - bootstrap.memory_lock=true
       - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
       - xpack.security.enabled=false
-
     ulimits:
       memlock:
         soft: -1
@@ -46,6 +45,7 @@ services:
     volumes:
       - ./resources/elk6/init_kibana.sh:/opt/init_kibana.sh
       - ./resources/elk6/kibana_APIonly.json:/opt/kibana_APIonly.json
+      - ./resources/elk6/logstash-vulnwhisperer-template.json:/opt/index-template.json
     command: sh -c "apk add --no-cache curl bash && chmod +x /opt/init_kibana.sh && chmod +r /opt/kibana_APIonly.json && cd /opt/ && /bin/bash /opt/init_kibana.sh" # /opt/kibana_APIonly.json"
     networks:
       esnet:
@@ -75,7 +75,6 @@ services:
                   "/opt/VulnWhisperer/vulnwhisperer.ini"
     ]    
     volumes:
-      - /opt/VulnWhisperer/data/:/opt/VulnWhisperer/data
       - ./data/:/opt/VulnWhisperer/data
       - ./resources/elk6/vulnwhisperer.ini:/opt/VulnWhisperer/vulnwhisperer.ini
     network_mode: host

resources/elk5-old_compatibility/elasticsearch/logstash-vulnwhisperer-template.json

@@ -53,7 +53,7 @@
       ],
       "properties": {
         "plugin_id": {
-          "type": "integer"
+          "type": "float"
         },
         "last_updated": {
           "type": "date"

resources/elk6/init_kibana.sh

@@ -3,7 +3,7 @@
 #kibana_url="localhost:5601"
 kibana_url="kibana.local:5601"
 elasticsearch_url="elasticsearch.local:9200"
-add_saved_objects="curl -u elastic:changeme -k -XPOST 'http://"$kibana_url"/api/saved_objects/_bulk_create' -H 'Content-Type: application/json' -H \"kbn-xsrf: true\" -d @"
+add_saved_objects="curl -s -u elastic:changeme -k -XPOST 'http://"$kibana_url"/api/saved_objects/_bulk_create' -H 'Content-Type: application/json' -H \"kbn-xsrf: true\" -d @"
 
 #Create all saved objects - including index pattern
 saved_objects_file="kibana_APIonly.json"
@@ -12,16 +12,26 @@ saved_objects_file="kibana_APIonly.json"
 
 until curl -s "$elasticsearch_url/_cluster/health?pretty" | grep '"status"' | grep -qE "green|yellow"; do
     curl -s "$elasticsearch_url/_cluster/health?pretty"
-    echo "Waiting for Elasticsearch"
+    echo "Waiting for Elasticsearch..."
     sleep 5
 done
 
-echo "Loading VulnWhisperer index template"
+count=0
-curl -XPUT "http://$elasticsearch_url/_template/vulnwhisperer" -H 'Content-Type: application/json' -d '@/opt/index-template.json'
+until  curl -s --fail -XPUT "http://$elasticsearch_url/_template/vulnwhisperer" -H 'Content-Type: application/json' -d '@/opt/index-template.json'; do
+    echo "Loading VulnWhisperer index template..."
+    ((count++)) && ((count==60)) && break
+    sleep 1
+done
 
-until [ "`curl -I "$kibana_url"/status | head -n1 |cut -d$' ' -f2`" == "200" ]; do
+if [[ count -le 60 && $(curl -s -I http://$elasticsearch_url/_template/vulnwhisperer | head -n1 |cut -d$' ' -f2) == "200" ]]; then
-    curl -I "$kibana_url"/status
+    echo -e "\n✅ VulnWhisperer index template loaded"
-    echo "Waiting for Kibana"
+else
+    echo -e "\n❌ VulnWhisperer index template failed to load"
+fi
+
+until [ "`curl -s -I "$kibana_url"/status | head -n1 |cut -d$' ' -f2`" == "200" ]; do
+    curl -s -I "$kibana_url"/status
+    echo "Waiting for Kibana..."
     sleep 5
 done
 
@@ -39,5 +49,4 @@ eval $(echo $add_saved_objects$saved_objects_file)
 
 #Create jira index pattern, separated for not fill of crap variables the Discover tab by default
 #index_name = "logstash-jira-*"
-#os.system(add_index+index_name+"' '-d{\"attributes\":{\"title\":\""+index_name+"\",\"timeFieldName\":\"@timestamp\"}}'")
+#os.system(add_index+index_name+"' '-d{\"attributes\":{\"title\":\""+index_name+"\",\"timeFieldName\":\"@timestamp\"}}'")
-

resources/elk6/logstash-vulnwhisperer-template.json

@@ -0,0 +1,233 @@
+{
+  "index_patterns": "logstash-vulnwhisperer-*",
+  "mappings": {
+    "doc": {
+      "properties": {
+        "@timestamp": {
+          "type": "date"
+        },
+        "@version": {
+          "type": "keyword"
+        },
+        "asset": {
+          "type": "text",
+          "norms": false,
+          "fields": {
+            "keyword": {
+              "type": "keyword",
+              "ignore_above": 256
+            }
+          }
+        },
+        "asset_uuid": {
+          "type": "keyword"
+        },
+        "assign_ip": {
+          "type": "ip"
+        },
+        "category": {
+          "type": "keyword"
+        },
+        "cve": {
+          "type": "keyword"
+        },
+        "cvss_base": {
+          "type": "float"
+        },
+        "cvss_temporal_vector": {
+          "type": "keyword"
+        },
+        "cvss_temporal": {
+          "type": "float"
+        },
+        "cvss_vector": {
+          "type": "keyword"
+        },
+        "cvss": {
+          "type": "float"
+        },
+        "cvss3_base": {
+          "type": "float"
+        },
+        "cvss3_temporal_vector": {
+          "type": "keyword"
+        },
+        "cvss3_temporal": {
+          "type": "float"
+        },
+        "cvss3_vector": {
+          "type": "keyword"
+        },
+        "cvss3": {
+          "type": "float"
+        },
+        "description": {
+          "fields": {
+            "keyword": {
+              "ignore_above": 256,
+              "type": "keyword"
+            }
+          },
+          "norms": false,
+          "type": "text"
+        },
+        "dns": {
+          "type": "keyword"
+        },
+        "exploitability": {
+          "fields": {
+            "keyword": {
+              "ignore_above": 256,
+              "type": "keyword"
+            }
+          },
+          "norms": false,
+          "type": "text"
+        },
+        "fqdn": {
+          "type": "keyword"
+        },
+        "geoip": {
+          "dynamic": true,
+          "type": "object",
+          "properties": {
+            "ip": {
+              "type": "ip"
+            },
+            "latitude": {
+              "type": "float"
+            },
+            "location": {
+              "type": "geo_point"
+            },
+            "longitude": {
+              "type": "float"
+            }
+          }
+        },
+        "history_id": {
+          "type": "keyword"
+        },
+        "host": {
+          "type": "keyword"
+        },
+        "host_end": {
+          "type": "date"
+        },
+        "host_start": {
+          "type": "date"
+        },
+        "impact": {
+          "fields": {
+            "keyword": {
+              "ignore_above": 256,
+              "type": "keyword"
+            }
+          },
+          "norms": false,
+          "type": "text"
+        },
+        "ip_status": {
+          "type": "keyword"
+        },
+        "ip": {
+          "type": "ip"
+        },
+        "last_updated": {
+          "type": "date"
+        },
+        "operating_system": {
+          "type": "keyword"
+        },
+        "path": {
+          "type": "keyword"
+        },
+        "pci_vuln": {
+          "type": "keyword"
+        },
+        "plugin_family": {
+          "type": "keyword"
+        },
+        "plugin_id": {
+          "type": "keyword"
+        },
+        "plugin_name": {
+          "type": "keyword"
+        },
+        "plugin_output": {
+          "fields": {
+            "keyword": {
+              "ignore_above": 256,
+              "type": "keyword"
+            }
+          },
+          "norms": false,
+          "type": "text"
+        },
+        "port": {
+          "type": "integer"
+        },
+        "protocol": {
+          "type": "keyword"
+        },
+        "results": {
+          "type": "text"
+        },
+        "risk_number": {
+          "type": "integer"
+        },
+        "risk_score_name": {
+          "type": "keyword"
+        },
+        "risk_score": {
+          "type": "float"
+        },
+        "risk": {
+          "type": "keyword"
+        },
+        "scan_id": {
+          "type": "keyword"
+        },
+        "scan_name": {
+          "type": "keyword"
+        },
+        "scan_reference": {
+          "type": "keyword"
+        },
+        "see_also": {
+          "type": "keyword"
+        },
+        "solution": {
+          "type": "keyword"
+        },
+        "source": {
+          "type": "keyword"
+        },
+        "ssl": {
+          "type": "keyword"
+        },
+        "synopsis": {
+          "type": "keyword"
+        },
+        "system_type": {
+          "type": "keyword"
+        },
+        "tags": {
+          "type": "keyword"
+        },
+        "threat": {
+          "type": "text"
+        },
+        "type": {
+          "type": "keyword"
+        },
+        "vendor_reference": {
+          "type": "keyword"
+        },
+        "vulnerability_state": {
+          "type": "keyword"
+        }
+      }
+    }
+  }
+}

tests/test-docker.sh

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+
+NORMAL=$(tput sgr0)
+GREEN=$(tput setaf 2)
+YELLOW=$(tput setaf 3)
+RED=$(tput setaf 1)
+
+function red() {
+    echo -e "$RED$*$NORMAL"
+}
+
+function green() {
+    echo -e "$GREEN$*$NORMAL"
+}
+
+function yellow() {
+    echo -e "$YELLOW$*$NORMAL"
+}
+
+return_code=0
+
+elasticsearch_url="localhost:9200"
+logstash_url="localhost:9600"
+
+until curl -s "$elasticsearch_url/_cluster/health?pretty" | grep '"status"' | grep -qE "green|yellow"; do
+    yellow "Waiting for Elasticsearch..."
+    sleep 5
+done
+green "✅ Elasticsearch status is green..."
+
+count=0
+until [[ $(curl -s "$logstash_url/_node/stats" | jq '.events.out') -ge 1236 ]]; do
+    yellow "Waiting for Logstash load to finish...  $(curl -s "$logstash_url/_node/stats" | jq '.events.out') of 1236 (attempt $count of 60)"
+    ((count++)) && ((count==60)) && break
+    sleep 5
+done
+
+if [[ count -le 60 && $(curl -s "$logstash_url/_node/stats" | jq '.events.out') -ge 1236 ]]; then
+    green "✅ Logstash load finished..."
+else
+    red "❌ Logstash load didn't complete... $(curl -s "$logstash_url/_node/stats" | jq '.events.out')"
+fi
+
+
+count=0
+until [[ $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq '.count') -ge 1232 ]] ; do
+    yellow "Waiting for Elasticsearch index to sync... $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq '.count') of 1232 logs loaded (attempt $count of 150)"
+    ((count++)) && ((count==150)) && break
+    sleep 2
+done
+if [[ count -le 50 && $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq '.count') -ge 1232 ]]; then
+    green "✅ logstash-vulnwhisperer-2019.03 document count >= 1232"
+else
+    red "❌ TIMED OUT waiting for logstash-vulnwhisperer-2019.03 document count: $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq) != 1232"
+fi
+
+# if [[ $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count" | jq '.count') == 1232 ]]; then
+#     green "✅ Passed: logstash-vulnwhisperer-2019.03 document count == 1232"
+# else
+#     red "❌ Failed: logstash-vulnwhisperer-2019.03 document count == 1232 was: $(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_count") instead"
+#     ((return_code = return_code + 1))
+# fi
+
+# Test Nessus plugin_name:Backported Security Patch Detection (FTP)
+nessus_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_search?q=plugin_name:%22Backported%20Security%20Patch%20Detection%20(FTP)%22%20AND%20asset:176.28.50.164%20AND%20tags:nessus" | jq '.hits.hits[]._source')
+if echo $nessus_doc | jq '.risk' | grep -q "None"; then
+    green "✅ Passed: Nessus risk == None"
+else
+    red "❌ Failed: Nessus risk == None was: $(echo $nessus_doc | jq '.risk') instead"
+    ((return_code = return_code + 1))
+fi
+
+# Test Tenable plugin_name:Backported Security Patch Detection (FTP)
+tenable_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_search?q=plugin_name:%22Backported%20Security%20Patch%20Detection%20(FTP)%22%20AND%20asset:176.28.50.164%20AND%20tags:tenable" | jq '.hits.hits[]._source')
+# Test asset
+if echo $tenable_doc | jq .asset | grep -q '176.28.50.164'; then
+    green "✅ Passed: Tenable asset == 176.28.50.164"
+else
+    red "❌ Failed: Tenable asset == 176.28.50.164 was: $(echo $tenable_doc | jq .asset) instead"
+    ((return_code = return_code + 1))
+fi
+
+# Test @timestamp
+if echo $tenable_doc | jq '.["@timestamp"]' | grep -q '2019-03-30T15:45:44.000Z'; then
+    green "✅ Passed: Tenable @timestamp == 2019-03-30T15:45:44.000Z"
+else
+    red "❌ Failed: Tenable @timestamp == 2019-03-30T15:45:44.000Z was: $(echo $tenable_doc | jq '.["@timestamp"]') instead"
+    ((return_code = return_code + 1))
+fi
+
+# Test Qualys plugin_name:OpenSSL Multiple Remote Security Vulnerabilities
+qualys_vuln_doc=$(curl -s "$elasticsearch_url/logstash-vulnwhisperer-2019.03/_search?q=tags:qualys_vuln%20AND%20ip:%22176.28.50.164%22%20AND%20plugin_name:%22OpenSSL%20Multiple%20Remote%20Security%20Vulnerabilities%22%20AND%20port:465" | jq '.hits.hits[]._source')
+# Test @timestamp
+if echo $qualys_vuln_doc | jq '.["@timestamp"]' | grep -q '2019-03-30T10:17:41.000Z'; then
+    green "✅ Passed: Qualys VM @timestamp == 2019-03-30T10:17:41.000Z"
+else
+    red "❌ Failed: Qualys VM @timestamp == 2019-03-30T10:17:41.000Z was: $(echo $qualys_vuln_doc | jq '.["@timestamp"]') instead"
+    ((return_code = return_code + 1))
+fi
+
+# Test @XXXX
+if echo $qualys_vuln_doc | jq '.cvss' | grep -q '6.8'; then
+    green "✅ Passed: Qualys VM cvss == 6.8"
+else
+    red "❌ Failed: Qualys VM cvss == 6.8 was: $(echo $qualys_vuln_doc | jq '.cvss') instead"
+    ((return_code = return_code + 1))
+fi
+
+exit $return_code

tests/test-vuln_whisperer.sh

@@ -0,0 +1,97 @@
+#!/usr/bin/env bash
+
+NORMAL=$(tput sgr0)
+GREEN=$(tput setaf 2)
+YELLOW=$(tput setaf 3)
+RED=$(tput setaf 1)
+
+function red() {
+    echo -e "$RED$*$NORMAL"
+}
+
+function green() {
+    echo -e "$GREEN$*$NORMAL"
+}
+
+function yellow() {
+    echo -e "$YELLOW$*$NORMAL"
+}
+
+return_code=0
+
+TEST_PATH=${TEST_PATH:-"tests/data"}
+
+yellow "\n*********************************************"
+yellow "* Test successful scan download and parsing *"
+yellow "*********************************************"
+rm -rf /opt/VulnWhisperer/*
+if vuln_whisperer -F -c configs/test.ini --mock --mock_dir "${TEST_PATH}"; then
+    green "\n✅ Passed: Test successful scan download and parsing"
+else
+    red "\n❌ Failed: Test successful scan download and parsing"
+    ((return_code = return_code + 1))
+fi
+
+yellow "\n*********************************************"
+yellow "*    Test run with no scans to import       *"
+yellow "*********************************************"
+if vuln_whisperer -F -c configs/test.ini --mock --mock_dir "${TEST_PATH}"; then
+    green "\n✅ Passed: Test run with no scans to import"
+else
+    red "\n❌ Failed: Test run with no scans to import"
+    ((return_code = return_code + 1))
+fi
+
+yellow "\n*********************************************"
+yellow "*           Test one failed scan            *"
+yellow "*********************************************"
+rm -rf /opt/VulnWhisperer/*
+yellow "Removing ${TEST_PATH}/nessus/GET_scans_exports_164_download"
+mv "${TEST_PATH}/nessus/GET_scans_exports_164_download"{,.bak}
+if vuln_whisperer -F -c configs/test.ini --mock --mock_dir "${TEST_PATH}"; [[ $? -eq 1 ]]; then
+    green "\n✅ Passed: Test one failed scan"
+else
+    red "\n❌ Failed: Test one failed scan"
+    ((return_code = return_code + 1))
+fi
+
+yellow "\n*********************************************"
+yellow "*           Test two failed scans           *"
+yellow "*********************************************"
+rm -rf /opt/VulnWhisperer/*
+yellow "Removing ${TEST_PATH}/qualys_vuln/scan_1553941061.87241"
+mv "${TEST_PATH}/qualys_vuln/scan_1553941061.87241"{,.bak}
+if vuln_whisperer -F -c configs/test.ini --mock --mock_dir "${TEST_PATH}"; [[ $? -eq 2 ]]; then
+    green "\n✅ Passed: Test two failed scans"
+else
+    red "\n❌ Failed: Test two failed scans"
+    ((return_code = return_code + 1))
+fi
+
+yellow "\n*********************************************"
+yellow "*   Test only nessus with one failed scan   *"
+yellow "*********************************************"
+rm -rf /opt/VulnWhisperer/*
+if vuln_whisperer -F -c configs/test.ini -s nessus --mock --mock_dir "${TEST_PATH}"; [[ $? -eq 1 ]]; then
+    green "\n✅ Passed: Test only nessus with one failed scan"
+else
+    red "\n❌ Failed: Test only nessus with one failed scan"
+    ((return_code = return_code + 1))
+fi
+
+yellow "*********************************************"
+yellow "* Test only Qualys VM with one failed scan  *"
+yellow "*********************************************"
+rm -rf /opt/VulnWhisperer/*
+if vuln_whisperer -F -c configs/test.ini -s qualys_vuln --mock --mock_dir "${TEST_PATH}"; [[ $? -eq 1 ]]; then
+    green "\n✅ Passed: Test only Qualys VM with one failed scan"
+else
+    red "\n❌ Failed: Test only Qualys VM with one failed scan"
+    ((return_code = return_code + 1))
+fi
+
+# Restore the removed files
+mv "${TEST_PATH}/qualys_vuln/scan_1553941061.87241.bak" "${TEST_PATH}/qualys_vuln/scan_1553941061.87241"
+mv "${TEST_PATH}/nessus/GET_scans_exports_164_download.bak" "${TEST_PATH}/nessus/GET_scans_exports_164_download"
+
+exit $return_code