Merge branch 'master' of github.com:MISP/misp-modules

2019-09-16 14:31:01 +02:00 · 2019-09-16 14:31:01 +02:00 · 5ebd0bd4fc
parent 8d33d6c18c f28a642f11
commit 5ebd0bd4fc
7 changed files with 321 additions and 36 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -10,6 +10,9 @@ python:
    - "3.6-dev"
    - "3.7-dev"

+before_install:
+    - docker build -t misp-modules --build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") docker/
+
 install:
    - sudo apt-get install libzbar0 libzbar-dev libpoppler-cpp-dev
    - pip install pipenv
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@ -0,0 +1,130 @@
+FROM python:3.7-buster AS build
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV WORKDIR="/usr/local/src/misp_modules"
+ENV VENV_DIR="/misp_modules"
+
+# Install Packages for build
+RUN set -eu \
+        ;mkdir -p ${WORKDIR} ${VENV_DIR} \
+        ;apt-get update  \
+        ;apt-get install -y \
+                git \
+                libpq5 \
+                libjpeg-dev \
+                tesseract-ocr \
+                libpoppler-cpp-dev \
+                imagemagick \
+                virtualenv \
+                libopencv-dev \
+                zbar-tools \
+                libzbar0 \
+                libzbar-dev \
+                libfuzzy-dev \
+        ;apt-get -y autoremove \
+        ;apt-get -y clean \
+        ;rm -rf /var/lib/apt/lists/* \
+        ;
+
+# Create MISP Modules
+RUN set -eu \
+        ;git clone https://github.com/MISP/misp-modules.git ${WORKDIR} \
+        ;virtualenv -p python3 ${VENV_DIR}/venv \
+        ;cd ${WORKDIR} \
+        ;${VENV_DIR}/venv/bin/pip3 install -I -r REQUIREMENTS --no-cache-dir \
+        ;${VENV_DIR}/venv/bin/pip3 install . --no-cache-dir \
+        ;
+
+#########################################
+# Start Final Docker Image
+#
+FROM python:3.7-slim-buster AS final
+
+ENV DEBIAN_FRONTEND noninteractive 
+ENV VENV_DIR="/misp_modules"
+
+# Copy all builded files from build stage
+COPY --from=build ${VENV_DIR} ${VENV_DIR}
+
+# Install Packages to run it
+RUN set -eu \
+        ;apt-get update  \
+        ;apt-get install -y \
+                curl \
+                libpq5 \
+                # libjpeg-dev \
+                tesseract-ocr \
+                libpoppler-cpp-dev \
+                imagemagick \
+                # virtualenv \
+                # libopencv-dev \
+                zbar-tools \
+                libzbar0 \
+                # libzbar-dev \
+                # libfuzzy-dev \
+        ;apt-get -y autoremove \
+        ;apt-get -y clean \
+        ;rm -rf /var/lib/apt/lists/* \
+        ;chown -R nobody ${VENV_DIR} \
+        ;
+
+# Entrypoint
+        COPY files/entrypoint.sh /entrypoint.sh
+        ENTRYPOINT [ "/entrypoint.sh" ]
+
+# Add Healthcheck Config
+        COPY files/healthcheck.sh /healthcheck.sh
+        HEALTHCHECK --interval=1m --timeout=45s --retries=3 CMD ["/healthcheck.sh"]
+
+# Change Workdir
+        WORKDIR ${VENV_DIR}
+
+# Change from root to www-data
+        USER nobody
+
+# Expose Port
+        EXPOSE 6666
+
+# Shortterm ARG Variables:
+        ARG VENDOR="MISP"
+        ARG COMPONENT="misp-modules"
+        ARG BUILD_DATE
+        ARG GIT_REPO="https://github.com/MISP/misp-modules"
+        ARG VCS_REF
+        ARG RELEASE_DATE
+        ARG NAME="MISP-dockerized-misp-modules"
+        ARG DESCRIPTION="This docker container contains MISP modules in an Debian Container."
+        ARG DOCUMENTATION="https://misp.github.io/misp-modules/"
+        ARG AUTHOR="MISP"
+        ARG LICENSE="BSD-3-Clause"
+
+# Longterm Environment Variables
+ENV \
+        BUILD_DATE=${BUILD_DATE} \
+        NAME=${NAME} \
+        PATH=$PATH:${VENV_DIR}/venv/bin
+
+# Labels
+LABEL org.label-schema.build-date="${BUILD_DATE}" \
+        org.label-schema.name="${NAME}" \
+        org.label-schema.description="${DESCRIPTION}" \
+        org.label-schema.vcs-ref="${VCS_REF}" \
+        org.label-schema.vcs-url="${GIT_REPO}" \
+        org.label-schema.url="${GIT_REPO}" \
+        org.label-schema.vendor="${VENDOR}" \
+        org.label-schema.version="${VERSION}" \
+        org.label-schema.usage="${DOCUMENTATION}" \
+        org.label-schema.schema-version="1.0.0-rc1"
+
+LABEL   org.opencontainers.image.created="${BUILD_DATE}" \
+        org.opencontainers.image.url="${GIT_REPO}" \
+        org.opencontainers.image.source="${GIT_REPO}" \
+        org.opencontainers.image.version="${VERSION}" \
+        org.opencontainers.image.revision="${VCS_REF}" \
+        org.opencontainers.image.vendor="${VENDOR}" \
+        org.opencontainers.image.title="${NAME}" \
+        org.opencontainers.image.description="${DESCRIPTION}" \
+        org.opencontainers.image.documentation="${DOCUMENTATION}" \
+        org.opencontainers.image.authors="${AUTHOR}" \
+        org.opencontainers.image.licenses="${LICENSE}"
+
--- a/docker/files/entrypoint.sh
+++ b/docker/files/entrypoint.sh
@ -0,0 +1,37 @@
+#!/bin/sh
+set -eu
+
+# Variables
+NC='\033[0m' # No Color
+Light_Green='\033[1;32m' 
+STARTMSG="${Light_Green}[ENTRYPOINT_MISP_MODULES]${NC}"
+VENV_DIR=${VENV_DIR:-"/misp-modules"}
+MISP_MODULES_BINARY="${VENV_DIR}/venv/bin/misp-modules"
+DEBUG=""
+
+# Functions
+echo (){
+    command echo "$STARTMSG $*"
+}
+
+# Environment Variables
+MISP_MODULES_DEBUG=${MISP_MODULES_DEBUG:-"false"}
+
+#
+#   MAIN
+#
+
+
+# Check if debugging mode should be enabled
+[ "$MISP_MODULES_DEBUG" = "true" ] && DEBUG="-d"
+
+# check if a command parameter exists and start misp-modules
+if [ $# = 0 ]
+then
+    # If no cmd parameter is set
+    echo "Start MISP Modules" && $MISP_MODULES_BINARY $DEBUG -l 0.0.0.0  > /dev/stdout 2> /dev/stderr
+else
+    # If cmd parameter is set
+    echo "Start MISP Modules" && $MISP_MODULES_BINARY $DEBUG -l 0.0.0.0 > /dev/stdout 2> /dev/stderr &
+    exec "$@"
+fi
--- a/docker/files/healthcheck.sh
+++ b/docker/files/healthcheck.sh
@ -0,0 +1,4 @@
+#!/bin/sh
+
+# If no contain is there or curl get an error back: exit 1. Docker restart then the container.
+curl -fk http://0.0.0.0:6666/modules || exit 1
--- a/docs/install.md
+++ b/docs/install.md
@ -1,41 +1,114 @@
-## How to install and start MISP modules in a Python virtualenv?
+## How to install and start MISP modules (in a Python virtualenv)?

 ~~~~bash
-sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick
-sudo -u www-data virtualenv -p python3 /var/www/MISP/venv
+SUDO_WWW="sudo -u www-data"
+
+sudo apt-get install -y \
+  git \
+  libpq5 \
+  libjpeg-dev \
+  tesseract-ocr \
+  libpoppler-cpp-dev \
+  imagemagick virtualenv \
+  libopencv-dev \
+  zbar-tools \
+  libzbar0 \
+  libzbar-dev \
+  libfuzzy-dev
+
+# BEGIN with virtualenv:   
+$SUDO_WWW virtualenv -p python3 /var/www/MISP/venv
+# END with virtualenv
+
 cd /usr/local/src/
 sudo git clone https://github.com/MISP/misp-modules.git
 cd misp-modules
-sudo -u www-data /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS
-sudo -u www-data /var/www/MISP/venv/bin/pip install .
-sudo apt install ruby-pygments.rb -y
-sudo gem install asciidoctor-pdf --pre
-sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local
+
+# BEGIN with virtualenv: 
+$SUDO_WWW  /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS
+$SUDO_WWW  /var/www/MISP/venv/bin/pip install .
+# END with virtualenv
+
+# BEGIN without virtualenv: 
+sudo pip install -I -r REQUIREMENTS
+sudo pip install .
+# END without virtualenv
+
+# Start misp-modules as a service
+sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl enable --now misp-modules
 /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules
 ~~~~

-## How to install and start MISP modules?
+## How to install and start MISP modules on RHEL-based distributions ?
+
+As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the SCL repository.

 ~~~~bash
-sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick
+SUDO_WWW="sudo -u apache"
+sudo yum install \
+  rh-ruby22 \
+  openjpeg-devel \
+  rubygem-rouge \
+  rubygem-asciidoctor \
+  zbar-devel \
+  opencv-devel \
+  gcc-c++ \
+  pkgconfig \
+  poppler-cpp-devel \
+  python-devel \
+  redhat-rpm-config
 cd /usr/local/src/
 sudo git clone https://github.com/MISP/misp-modules.git
 cd misp-modules
-sudo pip3 install -I -r REQUIREMENTS
-sudo pip3 install -I .
-sudo apt install ruby-pygments.rb -y
-sudo gem install asciidoctor-pdf --pre
-sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local
-/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules
+$SUDO_WWW /usr/bin/scl enable rh-python36 "virtualenv -p python3 /var/www/MISP/venv"
+$SUDO_WWW /var/www/MISP/venv/bin/pip install -U -I -r REQUIREMENTS
+$SUDO_WWW /var/www/MISP/venv/bin/pip install -U .
+~~~~
+
+Create the service file /etc/systemd/system/misp-modules.service :
+
+~~~~bash
+echo "[Unit]
+Description=MISP's modules
+After=misp-workers.service
+
+[Service]
+Type=simple
+User=apache
+Group=apache
+ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22  '/var/www/MISP/venv/bin/misp-modules –l 127.0.0.1 –s'
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target" | sudo tee /etc/systemd/system/misp-modules.service
+~~~~
+
+The After=misp-workers.service must be changed or removed if you have not created a misp-workers service. Then, enable the misp-modules service and start it:
+
+~~~~bash
+systemctl daemon-reload
+systemctl enable --now misp-modules
 ~~~~

 ## How to use an MISP modules Docker container

+### Docker build
+
+~~~~bash
+docker build -t misp-modules \
+    --build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") \
+  docker/
+~~~~
+
 ### Docker run

 ~~~~bash
 # Start Redis
 docker run --rm -d --name=misp-redis redis:alpine
+# Start MISP-modules
 docker run \
    --rm -d --name=misp-modules \
    -e REDIS_BACKEND=misp-redis \
@ -43,7 +116,7 @@ docker run \
    -e REDIS_PW="" \
    -e REDIS_DATABASE="245" \
    -e MISP_MODULES_DEBUG="false" \
-    dcso/misp-dockerized-redis
+    dcso/misp-dockerized-misp-modules
 ~~~~

 ### Docker-compose
@ -53,6 +126,12 @@ services:
  misp-modules:
    # https://hub.docker.com/r/dcso/misp-dockerized-misp-modules
    image: dcso/misp-dockerized-misp-modules:3
+    
+    # Local image:
+    #image: misp-modules
+    #build:
+    #  context: docker/
+    
    environment:
      # Redis
      REDIS_BACKEND: misp-redis
--- a/misp_modules/modules/import_mod/cuckooimport.py
+++ b/misp_modules/modules/import_mod/cuckooimport.py
@ -1,9 +1,11 @@
 import json
 import base64
-import tarfile
+import io
 import logging
 import posixpath
-from io import BytesIO, BufferedReader
+import stat
+import tarfile
+import zipfile
 from pymisp import MISPEvent, MISPObject, MISPAttribute
 from pymisp.tools import make_binary_objects
 from collections import OrderedDict
@ -12,10 +14,14 @@ log = logging.getLogger(__name__)

 misperrors = {'error': 'Error'}

-moduleinfo = {'version': '1.0',
-              'author': 'Pierre-Jean Grenier',
-              'description': 'Cuckoo archive import',
-              'module-type': ['import']}
+moduleinfo = {
+    'version': '1.1',
+    'author': 'Pierre-Jean Grenier',
+    'description': "Import a Cuckoo archive (zipfile or bzip2 tarball), "
+                   "either downloaded manually or exported from the "
+                   "API (/tasks/report/{task_id}/all).",
+    'module-type': ['import'],
+}

 moduleconfig = []

@ -202,13 +208,21 @@ class CuckooParser():
        self.files = None
        self.malware_binary = None
        self.report = None
-        self.config = {key: int(on) for key, on in config.items()}
+        self.config = {
+            # if an option is missing (we receive None as a value),
+            # fall back to the default specified in the options
+            key: int(
+                on if on is not None
+                else self.options[key]["userConfig"]["checked"] == 'true'
+            )
+            for key, on in config.items()
+        }

    def get_file(self, relative_filepath):
-        """Return a BufferedReader for the corresponding relative_filepath
-        in the Cuckoo archive. If not found, return an empty BufferedReader
+        """Return an io.BufferedIOBase for the corresponding relative_filepath
+        in the Cuckoo archive. If not found, return an empty io.BufferedReader
        to avoid fatal errors."""
-        blackhole = BufferedReader(open('/dev/null', 'rb'))
+        blackhole = io.BufferedReader(open('/dev/null', 'rb'))
        res = self.files.get(relative_filepath, blackhole)
        if res == blackhole:
            log.debug(f"Did not find file {relative_filepath}, "
@ -220,12 +234,30 @@ class CuckooParser():
        # archive_encoded is base 64 encoded content
        # we extract the info about each file but do not retrieve
        # it automatically, as it may take too much space in memory
-        buf_io = BytesIO(base64.b64decode(archive_encoded))
-        f = tarfile.open(fileobj=buf_io, mode='r:bz2')
-        self.files = {
-            info.name: f.extractfile(info)
-            for info in f.getmembers()
-        }
+        buf_io = io.BytesIO(base64.b64decode(archive_encoded))
+        if zipfile.is_zipfile(buf_io):
+            # the archive was probably downloaded from the WebUI
+            buf_io.seek(0)  # don't forget this not to read an empty buffer
+            z = zipfile.ZipFile(buf_io, 'r')
+            self.files = {
+                info.filename: z.open(info)
+                for info in z.filelist
+                # only extract the regular files and dirs, we don't
+                # want any symbolic link
+                if stat.S_ISREG(info.external_attr >> 16)
+                or stat.S_ISDIR(info.external_attr >> 16)
+            }
+        else:
+            # the archive was probably downloaded from the API
+            buf_io.seek(0)  # don't forget this not to read an empty buffer
+            f = tarfile.open(fileobj=buf_io, mode='r:bz2')
+            self.files = {
+                info.name: f.extractfile(info)
+                for info in f.getmembers()
+                # only extract the regular files and dirs, we don't
+                # want any symbolic link
+                if info.isreg() or info.isdir()
+            }

        # We want to keep the order of the keys of sub-dicts in the report,
        # eg. the signatures have marks with unknown keys such as
@ -280,7 +312,7 @@ class CuckooParser():
            log.debug("Sample is a file, uploading it")
            self.read_malware()
            file_o, bin_type_o, bin_section_li = make_binary_objects(
-                pseudofile=BytesIO(self.malware_binary),
+                pseudofile=io.BytesIO(self.malware_binary),
                filename=target["file"]["name"],
            )

@ -548,7 +580,7 @@ class CuckooParser():
            filename = posixpath.basename(path)

        dropped_file = self.get_file(path)
-        dropped_binary = BytesIO(dropped_file.read())
+        dropped_binary = io.BytesIO(dropped_file.read())
        # create ad hoc objects
        file_o, bin_type_o, bin_section_li = make_binary_objects(
            pseudofile=dropped_binary, filename=filename,
--- a/tests/test_expansions.py
+++ b/tests/test_expansions.py
@ -43,7 +43,7 @@ class TestExpansions(unittest.TestCase):
        query = {"module": "hibp", "email-src": "info@circl.lu"}
        response = self.misp_modules_post(query)
        to_check = self.get_values(response)
-        if to_check == "haveibeenpwned.com API not accessible (HTTP 403)":
+        if to_check == "haveibeenpwned.com API not accessible (HTTP 401)":
            self.skipTest(f"haveibeenpwned blocks travis IPs: {response}")
        self.assertEqual(to_check, 'OK (Not Found)', response)