Merge pull request #172 from pemontto/feature-fixes

Feature fixes
2019-04-15 11:47:02 +02:00
parent f23dd0bc83 bf5070f361
commit 4c9fa9d241
14 changed files with 151 additions and 42 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -23,6 +23,8 @@ script:
    # Test successful scan download and parsing
    - rm -rf /tmp/VulnWhisperer
    - vuln_whisperer -c configs/test.ini --mock --mock_dir ${TEST_PATH}
+    # Run a second time with no scans to import
+    - vuln_whisperer -c configs/test.ini --mock --mock_dir ${TEST_PATH}
    # Test one failed scan
    - rm -rf /tmp/VulnWhisperer
    - rm -f ${TEST_PATH}/nessus/GET_scans_exports_164_download
--- a/18
+++ b/18
@ -2,10 +2,10 @@ FROM centos:latest

 MAINTAINER Justin Henderson justin@hasecuritysolutions.com

-RUN yum update -y
-RUN yum install -y python python-devel git gcc
-RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
-RUN python get-pip.py
+RUN yum update -y && \
+    yum install -y python python-devel git gcc && \
+    curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+    python get-pip.py

 WORKDIR /opt/VulnWhisperer

@ -13,17 +13,15 @@ COPY requirements.txt requirements.txt
 COPY setup.py setup.py
 COPY vulnwhisp/ vulnwhisp/
 COPY bin/ bin/
-COPY deps/ deps/
 COPY configs/frameworks_example.ini frameworks_example.ini

-RUN python setup.py clean --all
-RUN pip install -r requirements.txt
+RUN python setup.py clean --all && \
+    pip install -r requirements.txt

-WORKDIR /opt/VulnWhisperer/deps/qualysapi
-RUN python setup.py install

 WORKDIR /opt/VulnWhisperer
-RUN python setup.py install
+RUN python setup.py install && \   
+    ln -s /opt/VulnWhisperer /tmp/VulnWhisperer


 CMD  vuln_whisperer -c /opt/VulnWhisperer/frameworks_example.ini
--- a/docker-compose-test.yml
+++ b/docker-compose-test.yml
@ -0,0 +1,94 @@
+version: '2'
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:6.6.0
+    container_name: elasticsearch
+    environment:
+      - cluster.name=vulnwhisperer
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms1g -Xmx1g"
+      - xpack.security.enabled=false
+      - cluster.routing.allocation.disk.threshold_enabled=false
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    mem_limit: 8g
+    volumes:
+      - ./docker_data/esdata1:/usr/share/elasticsearch/data
+    ports:
+      - 9200:9200
+    #restart: always
+    networks:
+      esnet:
+        aliases:
+          - elasticsearch.local
+
+  kibana:
+    image: docker.elastic.co/kibana/kibana:6.6.0
+    container_name: kibana
+    environment:
+      SERVER_NAME: kibana
+      ELASTICSEARCH_URL: http://elasticsearch:9200
+    ports:
+      - 5601:5601
+    depends_on:
+      - elasticsearch
+    # volumes:
+    #   - ./kibana-data:
+    networks:
+      esnet:
+        aliases:
+          - kibana.local
+
+  kibana-config:
+    image: alpine
+    container_name: kibana-config
+    volumes:
+      - ./resources/elk6/init_kibana.sh:/opt/init_kibana.sh
+      - ./resources/elk6/kibana_APIonly.json:/opt/kibana_APIonly.json
+      - ./docker_data/kibana_optimize:/usr/share/kibana/optimize
+    command: sh -c "apk add --no-cache curl bash && chmod +x /opt/init_kibana.sh && chmod +r /opt/kibana_APIonly.json && cd /opt/ && /bin/bash /opt/init_kibana.sh" # /opt/kibana_APIonly.json"
+    networks:
+      esnet:
+        aliases:
+          - kibana-config.local
+
+  logstash:
+    image: docker.elastic.co/logstash/logstash:6.6.0
+    container_name: logstash
+    volumes:
+      - ./resources/elk6/pipeline/:/usr/share/logstash/pipeline
+      - ./docker_data/data/:/opt/VulnWhisperer/data
+      - ./resources/elk6/logstash.yml:/usr/share/logstash/config/logstash.yml
+    environment:
+      - xpack.monitoring.enabled=false
+    depends_on:
+      - elasticsearch
+    networks:
+      esnet:
+        aliases:
+          - logstash.local
+  vulnwhisperer:
+    image: hasecuritysolutions/vulnwhisperer:latest
+    container_name: vulnwhisperer
+    entrypoint: [
+                  "vuln_whisperer",
+                  "-c",
+                  "/opt/VulnWhisperer/vulnwhisperer.ini",
+                  "--mock",
+                  "--mock_dir",
+                  "/tests/data"
+    ]    
+    volumes:
+      # - /opt/VulnWhisperer/data/:/opt/VulnWhisperer/data
+      - ./docker_data/data/:/opt/VulnWhisperer/data
+      - ./configs/test.ini:/opt/VulnWhisperer/vulnwhisperer.ini
+      - ./tests/data/:/tests/data
+    network_mode: host
+
+networks:
+  esnet:
--- a/docker-compose.v6.yml
+++ b/docker-compose.v6.yml
@ -56,7 +56,7 @@ services:
    container_name: logstash
    volumes:
      - ./resources/elk6/pipeline/:/usr/share/logstash/pipeline
-      - ./data/:/opt/vulnwhisperer/data
+      - ./data/:/opt/VulnWhisperer/data
      #- ./resources/elk6/logstash.yml:/usr/share/logstash/config/logstash.yml
    environment:
      - xpack.monitoring.enabled=false
@ -72,12 +72,12 @@ services:
    entrypoint: [
                  "vuln_whisperer",
                  "-c",
-                  "/opt/vulnwhisperer/vulnwhisperer.ini"
+                  "/opt/VulnWhisperer/vulnwhisperer.ini"
    ]    
    volumes:
-      - /opt/vulnwhisperer/data/:/opt/vulnwhisperer/data
-      - ./data/:/opt/vulnwhisperer/data
-      - ./resources/elk6/vulnwhisperer.ini:/opt/vulnwhisperer/vulnwhisperer.ini
+      - /opt/VulnWhisperer/data/:/opt/VulnWhisperer/data
+      - ./data/:/opt/VulnWhisperer/data
+      - ./resources/elk6/vulnwhisperer.ini:/opt/VulnWhisperer/vulnwhisperer.ini
    network_mode: host
 volumes:
  esdata1:
--- a/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf
+++ b/resources/elk5-old_compatibility/logstash/1000_nessus_process_file.conf
@ -7,13 +7,13 @@

 input {
  file {
-    path => "/opt/vulnwhisperer/nessus/**/*"
+    path => "/opt/VulnWhisperer/nessus/**/*"
    start_position => "beginning"
    tags => "nessus"
    type => "nessus"
  }
  file {
-    path => "/opt/vulnwhisperer/tenable/*.csv"
+    path => "/opt/VulnWhisperer/tenable/*.csv"
    start_position => "beginning"
    tags => "tenable"
    type => "tenable"
--- a/resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf
+++ b/resources/elk5-old_compatibility/logstash/2000_qualys_web_scans.conf
@ -6,7 +6,7 @@

 input {
  file {
-    path => [ "/opt/vulnwhisperer/data/qualys/*.json" , "/opt/vulnwhisperer/data/qualys_web/*.json", "/opt/vulnwhisperer/data/qualys_vuln/*.json" ]
+    path => [ "/opt/VulnWhisperer/data/qualys/*.json" , "/opt/VulnWhisperer/data/qualys_web/*.json", "/opt/VulnWhisperer/data/qualys_vuln/*.json" ]
    type => json
    codec => json
    start_position => "beginning"
--- a/resources/elk5-old_compatibility/logstash/3000_openvas.conf
+++ b/resources/elk5-old_compatibility/logstash/3000_openvas.conf
@ -6,7 +6,7 @@

 input {
  file {
-    path => "/opt/vulnwhisperer/openvas/*.json"
+    path => "/opt/VulnWhisperer/openvas/*.json"
    type => json
    codec => json
    start_position => "beginning"
--- a/resources/elk5-old_compatibility/logstash/4000_jira.conf
+++ b/resources/elk5-old_compatibility/logstash/4000_jira.conf
@ -2,7 +2,7 @@

 input {
  file {
-    path => "/opt/vulnwhisperer/jira/*.json"
+    path => "/opt/VulnWhisperer/jira/*.json"
    type => json
    codec => json
    start_position => "beginning"
--- a/resources/elk6/pipeline/1000_nessus_process_file.conf
+++ b/resources/elk6/pipeline/1000_nessus_process_file.conf
@ -7,14 +7,14 @@

 input {
  file {
-    path => "/opt/vulnwhisperer/data/nessus/**/*"
+    path => "/opt/VulnWhisperer/data/nessus/**/*"
    mode => "read"
    start_position => "beginning"
    file_completed_action => "delete"
    tags => "nessus"
  }
  file {
-    path => "/opt/vulnwhisperer/data/tenable/*.csv"
+    path => "/opt/VulnWhisperer/data/tenable/*.csv"
    mode => "read"
    start_position => "beginning"
    file_completed_action => "delete"
@ -53,11 +53,13 @@ filter {
    }

    #If using filebeats as your source, you will need to replace the "path" field to "source"
+    # Remove when scan name is included in event (current method is error prone)
    grok {
      match => { "path" => "(?<scan_name>[a-zA-Z0-9_.\-]+)_%{INT:scan_id}_%{INT:history_id}_%{INT:last_updated}.csv$" }
      tag_on_failure => []
    }

+    # TODO remove when @timestamp is included in event
    date {
      match => [ "last_updated", "UNIX" ]
      target => "@timestamp"
@ -169,6 +171,9 @@ filter {

 output {
  if "nessus" in [tags] or "tenable" in [tags]{
+    stdout {
+      codec => dots
+    }
    elasticsearch {
      hosts => [ "elasticsearch:9200" ]
      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
--- a/resources/elk6/pipeline/2000_qualys_web_scans.conf
+++ b/resources/elk6/pipeline/2000_qualys_web_scans.conf
@ -6,7 +6,7 @@

 input {
  file {
-    path => [ "/opt/vulnwhisperer/data/qualys/*.json" , "/opt/vulnwhisperer/data/qualys_web/*.json", "/opt/vulnwhisperer/data/qualys_vuln/*.json"] 
+    path => [ "/opt/VulnWhisperer/data/qualys/*.json" , "/opt/VulnWhisperer/data/qualys_web/*.json", "/opt/VulnWhisperer/data/qualys_vuln/*.json"] 
    type => json
    codec => json
    start_position => "beginning"
@ -98,6 +98,8 @@ filter {
        target => "last_time_tested"
      }
    }
+
+    # TODO remove when @timestamp is included in event
    date {
      match => [ "last_updated", "UNIX" ]
      target => "@timestamp"
@ -147,6 +149,9 @@ filter {
 }
 output {
  if "qualys" in [tags] {
+    stdout {
+      codec => dots
+    }
    elasticsearch {
      hosts => [ "elasticsearch:9200" ]
      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
--- a/resources/elk6/pipeline/3000_openvas.conf
+++ b/resources/elk6/pipeline/3000_openvas.conf
@ -6,7 +6,7 @@

 input {
  file {
-    path => "/opt/vulnwhisperer/data/openvas/*.json"
+    path => "/opt/VulnWhisperer/data/openvas/*.json"
    type => json
    codec => json
    start_position => "beginning"
@ -92,6 +92,8 @@ filter {
        target => "last_time_tested"
      }
    }
+
+    # TODO remove when @timestamp is included in event
    date {
      match => [ "last_updated", "UNIX" ]
      target => "@timestamp"
@ -141,6 +143,9 @@ filter {
 }
 output {
  if "openvas" in [tags] {
+    stdout {
+      codec => dots
+    }
    elasticsearch {
      hosts => [ "elasticsearch:9200" ]
      index => "logstash-vulnwhisperer-%{+YYYY.MM}"
--- a/resources/elk6/pipeline/4000_jira.conf
+++ b/resources/elk6/pipeline/4000_jira.conf
@ -2,7 +2,7 @@

 input {
  file {
-    path => "/opt/vulnwhisperer/data/jira/*.json"
+    path => "/opt/VulnWhisperer/data/jira/*.json"
    type => json
    codec => json
    start_position => "beginning"
--- a/resources/elk6/vulnwhisperer.ini
+++ b/resources/elk6/vulnwhisperer.ini
@ -4,8 +4,8 @@ hostname=localhost
 port=8834
 username=nessus_username
 password=nessus_password
-write_path=/opt/vulnwhisperer/data/nessus/
-db_path=/opt/vulnwhisperer/database
+write_path=/opt/VulnWhisperer/data/nessus/
+db_path=/opt/VulnWhisperer/database
 trash=false
 verbose=true

@ -15,7 +15,7 @@ hostname=cloud.tenable.com
 port=443
 username=tenable.io_username
 password=tenable.io_password
-write_path=/opt/vulnwhisperer/data/tenable/
+write_path=/opt/VulnWhisperer/data/tenable/
 db_path=/opt/VulnWhisperer/data/database
 trash=false
 verbose=true
@ -26,8 +26,8 @@ enabled = true
 hostname = qualysapi.qg2.apps.qualys.com
 username = exampleuser
 password = examplepass
-write_path=/opt/vulnwhisperer/data/qualys/
-db_path=/opt/vulnwhisperer/data/database
+write_path=/opt/VulnWhisperer/data/qualys/
+db_path=/opt/VulnWhisperer/data/database
 verbose=true

 # Set the maximum number of retries each connection should attempt.
@ -42,8 +42,8 @@ enabled = true
 hostname = qualysapi.qg2.apps.qualys.com
 username = exampleuser
 password = examplepass
-write_path=/opt/vulnwhisperer/data/qualys/
-db_path=/opt/vulnwhisperer/data/database
+write_path=/opt/VulnWhisperer/data/qualys/
+db_path=/opt/VulnWhisperer/data/database
 verbose=true

 # Set the maximum number of retries each connection should attempt.
@ -60,8 +60,8 @@ hostname = api.detectify.com
 username = exampleuser
 #password variable used as secretKey
 password = examplepass
-write_path =/opt/vulnwhisperer/data/detectify/
-db_path = /opt/vulnwhisperer/data/database
+write_path =/opt/VulnWhisperer/data/detectify/
+db_path = /opt/VulnWhisperer/data/database
 verbose = true

 [openvas]
@ -70,8 +70,8 @@ hostname = localhost
 port = 4000
 username = exampleuser
 password = examplepass
-write_path=/opt/vulnwhisperer/data/openvas/
-db_path=/opt/vulnwhisperer/data/database
+write_path=/opt/VulnWhisperer/data/openvas/
+db_path=/opt/VulnWhisperer/data/database
 verbose=true

 #[proxy]
@ -92,8 +92,8 @@ verbose=true
 hostname = jira-host
 username = username
 password = password
-write_path = /opt/vulnwhisperer/data/jira/
-db_path = /opt/vulnwhisperer/data/database
+write_path = /opt/VulnWhisperer/data/jira/
+db_path = /opt/VulnWhisperer/data/database
 verbose = true
 dns_resolv = False

--- a/vulnwhisp/vulnwhisp.py
+++ b/vulnwhisp/vulnwhisp.py
@ -412,7 +412,7 @@ class vulnWhispererNessus(vulnWhispererBase):
                                                    history_id, norm_time, 'csv')
                    repls = (('\\', '_'), ('/', '_'), (' ', '_'))
                    file_name = reduce(lambda a, kv: a.replace(*kv), repls, file_name)
-                    relative_path_name = self.path_check(folder_name + '/' + file_name)
+                    relative_path_name = self.path_check(folder_name + '/' + file_name).encode('utf8')

                    if os.path.isfile(relative_path_name):
                        if self.develop:
@ -581,7 +581,7 @@ class vulnWhispererQualys(vulnWhispererBase):
                          + '_{last_updated}'.format(last_updated=launched_date) \
                          + '.{extension}'.format(extension=output_format)

-            relative_path_name = self.path_check(report_name)
+            relative_path_name = self.path_check(report_name).encode('utf8')

            if os.path.isfile(relative_path_name):
                #TODO Possibly make this optional to sync directories
@ -743,7 +743,7 @@ class vulnWhispererOpenVAS(vulnWhispererBase):
            report_name = 'openvas_scan_{scan_name}_{last_updated}.{extension}'.format(scan_name=scan_name,
                                                                                       last_updated=launched_date,
                                                                                       extension=output_format)
-            relative_path_name = self.path_check(report_name)
+            relative_path_name = self.path_check(report_name).encode('utf8')
            scan_reference = report_id

            if os.path.isfile(relative_path_name):
@ -861,7 +861,7 @@ class vulnWhispererQualysVuln(vulnWhispererBase):
                          + '_{last_updated}'.format(last_updated=launched_date) \
                          + '.json'

-            relative_path_name = self.path_check(report_name)
+            relative_path_name = self.path_check(report_name).encode('utf8')

            if os.path.isfile(relative_path_name):
                #TODO Possibly make this optional to sync directories