Merge pull request #172 from pemontto/feature-fixes

Feature fixes

.travis.yml

@@ -23,6 +23,8 @@ script:
     # Test successful scan download and parsing
     - rm -rf /tmp/VulnWhisperer
     - vuln_whisperer -c configs/test.ini --mock --mock_dir ${TEST_PATH}
+    # Run a second time with no scans to import
+    - vuln_whisperer -c configs/test.ini --mock --mock_dir ${TEST_PATH}
     # Test one failed scan
     - rm -rf /tmp/VulnWhisperer
     - rm -f ${TEST_PATH}/nessus/GET_scans_exports_164_download

Dockerfile

@@ -2,10 +2,10 @@ FROM centos:latest
 
 MAINTAINER Justin Henderson justin@hasecuritysolutions.com
 
-RUN yum update -y
+RUN yum update -y && \
-RUN yum install -y python python-devel git gcc
+    yum install -y python python-devel git gcc && \
-RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+    curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
-RUN python get-pip.py
+    python get-pip.py
 
 WORKDIR /opt/VulnWhisperer
 
@@ -13,17 +13,15 @@ COPY requirements.txt requirements.txt
 COPY setup.py setup.py
 COPY vulnwhisp/ vulnwhisp/
 COPY bin/ bin/
-COPY deps/ deps/
 COPY configs/frameworks_example.ini frameworks_example.ini
 
-RUN python setup.py clean --all
+RUN python setup.py clean --all && \
-RUN pip install -r requirements.txt
+    pip install -r requirements.txt
 
-WORKDIR /opt/VulnWhisperer/deps/qualysapi
-RUN python setup.py install
 
 WORKDIR /opt/VulnWhisperer
-RUN python setup.py install
+RUN python setup.py install && \   
+    ln -s /opt/VulnWhisperer /tmp/VulnWhisperer
 
 
 CMD  vuln_whisperer -c /opt/VulnWhisperer/frameworks_example.ini

docker-compose-test.yml

@@ -0,0 +1,94 @@
+version: '2'
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.6.0
+    container_name: elasticsearch
+    environment:
+      - cluster.name=vulnwhisperer
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
+      - xpack.security.enabled=false
+      - cluster.routing.allocation.disk.threshold_enabled=false
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    mem_limit: 8g
+    volumes:
+      - ./docker_data/esdata1:/usr/share/elasticsearch/data
+    ports:
+      - 9200:9200
+    #restart: always
+    networks:
+      esnet:
+        aliases:
+          - elasticsearch.local
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:6.6.0
+    container_name: kibana
+    environment:
+      SERVER_NAME: kibana
+      ELASTICSEARCH_URL: http://elasticsearch:9200
+    ports:
+      - 5601:5601
+    depends_on:
+      - elasticsearch
+    # volumes:
+    #   - ./kibana-data:
+    networks:
+      esnet:
+        aliases:
+          - kibana.local
+
+  kibana-config:
+    image: alpine
+    container_name: kibana-config
+    volumes:
+      - ./resources/elk6/init_kibana.sh:/opt/init_kibana.sh
+      - ./resources/elk6/kibana_APIonly.json:/opt/kibana_APIonly.json
+      - ./docker_data/kibana_optimize:/usr/share/kibana/optimize
+    command: sh -c "apk add --no-cache curl bash && chmod +x /opt/init_kibana.sh && chmod +r /opt/kibana_APIonly.json && cd /opt/ && /bin/bash /opt/init_kibana.sh" # /opt/kibana_APIonly.json"
+    networks:
+      esnet:
+        aliases:
+          - kibana-config.local
+
+  logstash:
+    image: docker.elastic.co/logstash/logstash:6.6.0
+    container_name: logstash
+    volumes:
+      - ./resources/elk6/pipeline/:/usr/share/logstash/pipeline
+      - ./docker_data/data/:/opt/VulnWhisperer/data
+      - ./resources/elk6/logstash.yml:/usr/share/logstash/config/logstash.yml
+    environment:
+      - xpack.monitoring.enabled=false
+    depends_on:
+      - elasticsearch
+    networks:
+      esnet:
+        aliases:
+          - logstash.local
+  vulnwhisperer:
+    image: hasecuritysolutions/vulnwhisperer:latest
+    container_name: vulnwhisperer
+    entrypoint: [
+                  "vuln_whisperer",
+                  "-c",
+                  "/opt/VulnWhisperer/vulnwhisperer.ini",
+                  "--mock",
+                  "--mock_dir",
+                  "/tests/data"
+    ]    
+    volumes:
+      # - /opt/VulnWhisperer/data/:/opt/VulnWhisperer/data
+      - ./docker_data/data/:/opt/VulnWhisperer/data
+      - ./configs/test.ini:/opt/VulnWhisperer/vulnwhisperer.ini
+      - ./tests/data/:/tests/data
+    network_mode: host
+
+networks:
+  esnet:

docker-compose.v6.yml

@@ -56,7 +56,7 @@ services:
     container_name: logstash
     volumes:
       - ./resources/elk6/pipeline/:/usr/share/logstash/pipeline
-      - ./data/:/opt/vulnwhisperer/data
+      - ./data/:/opt/VulnWhisperer/data
       #- ./resources/elk6/logstash.yml:/usr/share/logstash/config/logstash.yml
     environment:
       - xpack.monitoring.enabled=false
@@ -72,12 +72,12 @@ services:
     entrypoint: [
                   "vuln_whisperer",
                   "-c",
-                  "/opt/vulnwhisperer/vulnwhisperer.ini"
+                  "/opt/VulnWhisperer/vulnwhisperer.ini"
     ]    
     volumes:
-      - /opt/vulnwhisperer/data/:/opt/vulnwhisperer/data
+      - /opt/VulnWhisperer/data/:/opt/VulnWhisperer/data
-      - ./data/:/opt/vulnwhisperer/data
+      - ./data/:/opt/VulnWhisperer/data
-      - ./resources/elk6/vulnwhisperer.ini:/opt/vulnwhisperer/vulnwhisperer.ini
+      - ./resources/elk6/vulnwhisperer.ini:/opt/VulnWhisperer/vulnwhisperer.ini
     network_mode: host
 volumes:
   esdata1:

resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf

@@ -7,13 +7,13 @@
 
 input {
   file {
-    path => "/opt/vulnwhisperer/nessus/**/*"
+    path => "/opt/VulnWhisperer/nessus/**/*"
     start_position => "beginning"
     tags => "nessus"
     type => "nessus"
   }
   file {
-    path => "/opt/vulnwhisperer/tenable/*.csv"
+    path => "/opt/VulnWhisperer/tenable/*.csv"
     start_position => "beginning"
     tags => "tenable"
     type => "tenable"

resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf

@@ -6,7 +6,7 @@
 
 input {
   file {
-    path => [ "/opt/vulnwhisperer/data/qualys/*.json" , "/opt/vulnwhisperer/data/qualys_web/*.json", "/opt/vulnwhisperer/data/qualys_vuln/*.json" ]
+    path => [ "/opt/VulnWhisperer/data/qualys/*.json" , "/opt/VulnWhisperer/data/qualys_web/*.json", "/opt/VulnWhisperer/data/qualys_vuln/*.json" ]
     type => json
     codec => json
     start_position => "beginning"

resources/elk5-old_compatibility/logstash/3000_openvas.conf

@@ -6,7 +6,7 @@
 
 input {
   file {
-    path => "/opt/vulnwhisperer/openvas/*.json"
+    path => "/opt/VulnWhisperer/openvas/*.json"
     type => json
     codec => json
     start_position => "beginning"

resources/elk5-old_compatibility/logstash/4000_jira.conf

@@ -2,7 +2,7 @@
 
 input {
   file {
-    path => "/opt/vulnwhisperer/jira/*.json"
+    path => "/opt/VulnWhisperer/jira/*.json"
     type => json
     codec => json
     start_position => "beginning"

resources/elk6/pipeline/1000_nessus_process_file.conf

@@ -7,14 +7,14 @@
 
 input {
   file {
-    path => "/opt/vulnwhisperer/data/nessus/**/*"
+    path => "/opt/VulnWhisperer/data/nessus/**/*"
     mode => "read"
     start_position => "beginning"
     file_completed_action => "delete"
     tags => "nessus"
   }
   file {
-    path => "/opt/vulnwhisperer/data/tenable/*.csv"
+    path => "/opt/VulnWhisperer/data/tenable/*.csv"
     mode => "read"
     start_position => "beginning"
     file_completed_action => "delete"
@@ -53,11 +53,13 @@ filter {
     }
 
     #If using filebeats as your source, you will need to replace the "path" field to "source"
+    # Remove when scan name is included in event (current method is error prone)
     grok {
       match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.csv$" }
       tag_on_failure => []
     }
 
+    # TODO remove when @timestamp is included in event
     date {
       match => [ "last_updated", "UNIX" ]
       target => "@timestamp"
@@ -169,6 +171,9 @@ filter {
 
 output {
   if "nessus" in [tags] or "tenable" in [tags]{
+    stdout {
+      codec => dots
+    }
     elasticsearch {
       hosts => [ "elasticsearch:9200" ]
       index => "logstash-vulnwhisperer-%{+YYYY.MM}"

resources/elk6/pipeline/2000_qualys_web_scans.conf

@@ -6,7 +6,7 @@
 
 input {
   file {
-    path => [ "/opt/vulnwhisperer/data/qualys/*.json" , "/opt/vulnwhisperer/data/qualys_web/*.json", "/opt/vulnwhisperer/data/qualys_vuln/*.json"] 
+    path => [ "/opt/VulnWhisperer/data/qualys/*.json" , "/opt/VulnWhisperer/data/qualys_web/*.json", "/opt/VulnWhisperer/data/qualys_vuln/*.json"] 
     type => json
     codec => json
     start_position => "beginning"
@@ -98,6 +98,8 @@ filter {
         target => "last_time_tested"
       }
     }
+
+    # TODO remove when @timestamp is included in event
     date {
       match => [ "last_updated", "UNIX" ]
       target => "@timestamp"
@@ -147,6 +149,9 @@ filter {
 }
 output {
   if "qualys" in [tags] {
+    stdout {
+      codec => dots
+    }
     elasticsearch {
       hosts => [ "elasticsearch:9200" ]
       index => "logstash-vulnwhisperer-%{+YYYY.MM}"

resources/elk6/pipeline/3000_openvas.conf

@@ -6,7 +6,7 @@
 
 input {
   file {
-    path => "/opt/vulnwhisperer/data/openvas/*.json"
+    path => "/opt/VulnWhisperer/data/openvas/*.json"
     type => json
     codec => json
     start_position => "beginning"
@@ -92,6 +92,8 @@ filter {
         target => "last_time_tested"
       }
     }
+
+    # TODO remove when @timestamp is included in event
     date {
       match => [ "last_updated", "UNIX" ]
       target => "@timestamp"
@@ -141,6 +143,9 @@ filter {
 }
 output {
   if "openvas" in [tags] {
+    stdout {
+      codec => dots
+    }
     elasticsearch {
       hosts => [ "elasticsearch:9200" ]
       index => "logstash-vulnwhisperer-%{+YYYY.MM}"

resources/elk6/pipeline/4000_jira.conf

@@ -2,7 +2,7 @@
 
 input {
   file {
-    path => "/opt/vulnwhisperer/data/jira/*.json"
+    path => "/opt/VulnWhisperer/data/jira/*.json"
     type => json
     codec => json
     start_position => "beginning"

resources/elk6/vulnwhisperer.ini

@@ -4,8 +4,8 @@ hostname=localhost
 port=8834
 username=nessus_username
 password=nessus_password
-write_path=/opt/vulnwhisperer/data/nessus/
+write_path=/opt/VulnWhisperer/data/nessus/
-db_path=/opt/vulnwhisperer/database
+db_path=/opt/VulnWhisperer/database
 trash=false
 verbose=true
 
@@ -15,7 +15,7 @@ hostname=cloud.tenable.com
 port=443
 username=tenable.io_username
 password=tenable.io_password
-write_path=/opt/vulnwhisperer/data/tenable/
+write_path=/opt/VulnWhisperer/data/tenable/
 db_path=/opt/VulnWhisperer/data/database
 trash=false
 verbose=true
@@ -26,8 +26,8 @@ enabled = true
 hostname = qualysapi.qg2.apps.qualys.com
 username = exampleuser
 password = examplepass
-write_path=/opt/vulnwhisperer/data/qualys/
+write_path=/opt/VulnWhisperer/data/qualys/
-db_path=/opt/vulnwhisperer/data/database
+db_path=/opt/VulnWhisperer/data/database
 verbose=true
 
 # Set the maximum number of retries each connection should attempt.
@@ -42,8 +42,8 @@ enabled = true
 hostname = qualysapi.qg2.apps.qualys.com
 username = exampleuser
 password = examplepass
-write_path=/opt/vulnwhisperer/data/qualys/
+write_path=/opt/VulnWhisperer/data/qualys/
-db_path=/opt/vulnwhisperer/data/database
+db_path=/opt/VulnWhisperer/data/database
 verbose=true
 
 # Set the maximum number of retries each connection should attempt.
@@ -60,8 +60,8 @@ hostname = api.detectify.com
 username = exampleuser
 #password variable used as secretKey
 password = examplepass
-write_path =/opt/vulnwhisperer/data/detectify/
+write_path =/opt/VulnWhisperer/data/detectify/
-db_path = /opt/vulnwhisperer/data/database
+db_path = /opt/VulnWhisperer/data/database
 verbose = true
 
 [openvas]
@@ -70,8 +70,8 @@ hostname = localhost
 port = 4000
 username = exampleuser
 password = examplepass
-write_path=/opt/vulnwhisperer/data/openvas/
+write_path=/opt/VulnWhisperer/data/openvas/
-db_path=/opt/vulnwhisperer/data/database
+db_path=/opt/VulnWhisperer/data/database
 verbose=true
 
 #[proxy]
@@ -92,8 +92,8 @@ verbose=true
 hostname = jira-host
 username = username
 password = password
-write_path = /opt/vulnwhisperer/data/jira/
+write_path = /opt/VulnWhisperer/data/jira/
-db_path = /opt/vulnwhisperer/data/database
+db_path = /opt/VulnWhisperer/data/database
 verbose = true
 dns_resolv = False
 

vulnwhisp/vulnwhisp.py

@@ -412,7 +412,7 @@ class vulnWhispererNessus(vulnWhispererBase):
                                                     history_id, norm_time, 'csv')
                     repls = (('\\', '_'), ('/', '_'), (' ', '_'))
                     file_name = reduce(lambda a, kv: a.replace(*kv), repls, file_name)
-                    relative_path_name = self.path_check(folder_name + '/' + file_name)
+                    relative_path_name = self.path_check(folder_name + '/' + file_name).encode('utf8')
 
                     if os.path.isfile(relative_path_name):
                         if self.develop:
@@ -581,7 +581,7 @@ class vulnWhispererQualys(vulnWhispererBase):
                           + '_{last_updated}'.format(last_updated=launched_date) \
                           + '.{extension}'.format(extension=output_format)
 
-            relative_path_name = self.path_check(report_name)
+            relative_path_name = self.path_check(report_name).encode('utf8')
 
             if os.path.isfile(relative_path_name):
                 #TODO Possibly make this optional to sync directories
@@ -743,7 +743,7 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
             report_name = 'openvas_scan_{scan_name}_{last_updated}.{extension}'.format(scan_name=scan_name,
                                                                                        last_updated=launched_date,
                                                                                        extension=output_format)
-            relative_path_name = self.path_check(report_name)
+            relative_path_name = self.path_check(report_name).encode('utf8')
             scan_reference = report_id
 
             if os.path.isfile(relative_path_name):
@@ -861,7 +861,7 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
                           + '_{last_updated}'.format(last_updated=launched_date) \
                           + '.json'
 
-            relative_path_name = self.path_check(report_name)
+            relative_path_name = self.path_check(report_name).encode('utf8')
 
             if os.path.isfile(relative_path_name):
                 #TODO Possibly make this optional to sync directories