Merge branch 'master' of github.com:MISP/misp-modules

.travis.yml

@@ -10,6 +10,9 @@ python:
     - "3.6-dev"
     - "3.7-dev"
 
+before_install:
+    - docker build -t misp-modules --build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") docker/
+
 install:
     - sudo apt-get install libzbar0 libzbar-dev libpoppler-cpp-dev
     - pip install pipenv

docker/Dockerfile

@@ -0,0 +1,130 @@
+FROM python:3.7-buster AS build
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV WORKDIR="/usr/local/src/misp_modules"
+ENV VENV_DIR="/misp_modules"
+
+# Install Packages for build
+RUN set -eu \
+        ;mkdir -p ${WORKDIR} ${VENV_DIR} \
+        ;apt-get update  \
+        ;apt-get install -y \
+                git \
+                libpq5 \
+                libjpeg-dev \
+                tesseract-ocr \
+                libpoppler-cpp-dev \
+                imagemagick \
+                virtualenv \
+                libopencv-dev \
+                zbar-tools \
+                libzbar0 \
+                libzbar-dev \
+                libfuzzy-dev \
+        ;apt-get -y autoremove \
+        ;apt-get -y clean \
+        ;rm -rf /var/lib/apt/lists/* \
+        ;
+
+# Create MISP Modules
+RUN set -eu \
+        ;git clone https://github.com/MISP/misp-modules.git ${WORKDIR} \
+        ;virtualenv -p python3 ${VENV_DIR}/venv \
+        ;cd ${WORKDIR} \
+        ;${VENV_DIR}/venv/bin/pip3 install -I -r REQUIREMENTS --no-cache-dir \
+        ;${VENV_DIR}/venv/bin/pip3 install . --no-cache-dir \
+        ;
+
+#########################################
+# Start Final Docker Image
+#
+FROM python:3.7-slim-buster AS final
+
+ENV DEBIAN_FRONTEND noninteractive 
+ENV VENV_DIR="/misp_modules"
+
+# Copy all builded files from build stage
+COPY --from=build ${VENV_DIR} ${VENV_DIR}
+
+# Install Packages to run it
+RUN set -eu \
+        ;apt-get update  \
+        ;apt-get install -y \
+                curl \
+                libpq5 \
+                # libjpeg-dev \
+                tesseract-ocr \
+                libpoppler-cpp-dev \
+                imagemagick \
+                # virtualenv \
+                # libopencv-dev \
+                zbar-tools \
+                libzbar0 \
+                # libzbar-dev \
+                # libfuzzy-dev \
+        ;apt-get -y autoremove \
+        ;apt-get -y clean \
+        ;rm -rf /var/lib/apt/lists/* \
+        ;chown -R nobody ${VENV_DIR} \
+        ;
+
+# Entrypoint
+        COPY files/entrypoint.sh /entrypoint.sh
+        ENTRYPOINT [ "/entrypoint.sh" ]
+
+# Add Healthcheck Config
+        COPY files/healthcheck.sh /healthcheck.sh
+        HEALTHCHECK --interval=1m --timeout=45s --retries=3 CMD ["/healthcheck.sh"]
+
+# Change Workdir
+        WORKDIR ${VENV_DIR}
+
+# Change from root to www-data
+        USER nobody
+
+# Expose Port
+        EXPOSE 6666
+
+# Shortterm ARG Variables:
+        ARG VENDOR="MISP"
+        ARG COMPONENT="misp-modules"
+        ARG BUILD_DATE
+        ARG GIT_REPO="https://github.com/MISP/misp-modules"
+        ARG VCS_REF
+        ARG RELEASE_DATE
+        ARG NAME="MISP-dockerized-misp-modules"
+        ARG DESCRIPTION="This docker container contains MISP modules in an Debian Container."
+        ARG DOCUMENTATION="https://misp.github.io/misp-modules/"
+        ARG AUTHOR="MISP"
+        ARG LICENSE="BSD-3-Clause"
+
+# Longterm Environment Variables
+ENV \
+        BUILD_DATE=${BUILD_DATE} \
+        NAME=${NAME} \
+        PATH=$PATH:${VENV_DIR}/venv/bin
+
+# Labels
+LABEL org.label-schema.build-date="${BUILD_DATE}" \
+        org.label-schema.name="${NAME}" \
+        org.label-schema.description="${DESCRIPTION}" \
+        org.label-schema.vcs-ref="${VCS_REF}" \
+        org.label-schema.vcs-url="${GIT_REPO}" \
+        org.label-schema.url="${GIT_REPO}" \
+        org.label-schema.vendor="${VENDOR}" \
+        org.label-schema.version="${VERSION}" \
+        org.label-schema.usage="${DOCUMENTATION}" \
+        org.label-schema.schema-version="1.0.0-rc1"
+
+LABEL   org.opencontainers.image.created="${BUILD_DATE}" \
+        org.opencontainers.image.url="${GIT_REPO}" \
+        org.opencontainers.image.source="${GIT_REPO}" \
+        org.opencontainers.image.version="${VERSION}" \
+        org.opencontainers.image.revision="${VCS_REF}" \
+        org.opencontainers.image.vendor="${VENDOR}" \
+        org.opencontainers.image.title="${NAME}" \
+        org.opencontainers.image.description="${DESCRIPTION}" \
+        org.opencontainers.image.documentation="${DOCUMENTATION}" \
+        org.opencontainers.image.authors="${AUTHOR}" \
+        org.opencontainers.image.licenses="${LICENSE}"
+

docker/files/entrypoint.sh

@@ -0,0 +1,37 @@
+#!/bin/sh
+set -eu
+
+# Variables
+NC='\033[0m' # No Color
+Light_Green='\033[1;32m' 
+STARTMSG="${Light_Green}[ENTRYPOINT_MISP_MODULES]${NC}"
+VENV_DIR=${VENV_DIR:-"/misp-modules"}
+MISP_MODULES_BINARY="${VENV_DIR}/venv/bin/misp-modules"
+DEBUG=""
+
+# Functions
+echo (){
+    command echo "$STARTMSG $*"
+}
+
+# Environment Variables
+MISP_MODULES_DEBUG=${MISP_MODULES_DEBUG:-"false"}
+
+#
+#   MAIN
+#
+
+
+# Check if debugging mode should be enabled
+[ "$MISP_MODULES_DEBUG" = "true" ] && DEBUG="-d"
+
+# check if a command parameter exists and start misp-modules
+if [ $# = 0 ]
+then
+    # If no cmd parameter is set
+    echo "Start MISP Modules" && $MISP_MODULES_BINARY $DEBUG -l 0.0.0.0  > /dev/stdout 2> /dev/stderr
+else
+    # If cmd parameter is set
+    echo "Start MISP Modules" && $MISP_MODULES_BINARY $DEBUG -l 0.0.0.0 > /dev/stdout 2> /dev/stderr &
+    exec "$@"
+fi

docker/files/healthcheck.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+# If no contain is there or curl get an error back: exit 1. Docker restart then the container.
+curl -fk http://0.0.0.0:6666/modules || exit 1

docs/install.md

@@ -1,41 +1,114 @@
-## How to install and start MISP modules in a Python virtualenv?
+## How to install and start MISP modules (in a Python virtualenv)?
 
 ~~~~bash
-sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick
+SUDO_WWW="sudo -u www-data"
-sudo -u www-data virtualenv -p python3 /var/www/MISP/venv
+
+sudo apt-get install -y \
+  git \
+  libpq5 \
+  libjpeg-dev \
+  tesseract-ocr \
+  libpoppler-cpp-dev \
+  imagemagick virtualenv \
+  libopencv-dev \
+  zbar-tools \
+  libzbar0 \
+  libzbar-dev \
+  libfuzzy-dev
+
+# BEGIN with virtualenv:   
+$SUDO_WWW virtualenv -p python3 /var/www/MISP/venv
+# END with virtualenv
+
 cd /usr/local/src/
 sudo git clone https://github.com/MISP/misp-modules.git
 cd misp-modules
-sudo -u www-data /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS
+
-sudo -u www-data /var/www/MISP/venv/bin/pip install .
+# BEGIN with virtualenv: 
-sudo apt install ruby-pygments.rb -y
+$SUDO_WWW  /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS
-sudo gem install asciidoctor-pdf --pre
+$SUDO_WWW  /var/www/MISP/venv/bin/pip install .
-sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local
+# END with virtualenv
+
+# BEGIN without virtualenv: 
+sudo pip install -I -r REQUIREMENTS
+sudo pip install .
+# END without virtualenv
+
+# Start misp-modules as a service
+sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl enable --now misp-modules
 /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules
 ~~~~
 
-## How to install and start MISP modules?
+## How to install and start MISP modules on RHEL-based distributions ?
+
+As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the SCL repository.
 
 ~~~~bash
-sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick
+SUDO_WWW="sudo -u apache"
+sudo yum install \
+  rh-ruby22 \
+  openjpeg-devel \
+  rubygem-rouge \
+  rubygem-asciidoctor \
+  zbar-devel \
+  opencv-devel \
+  gcc-c++ \
+  pkgconfig \
+  poppler-cpp-devel \
+  python-devel \
+  redhat-rpm-config
 cd /usr/local/src/
 sudo git clone https://github.com/MISP/misp-modules.git
 cd misp-modules
-sudo pip3 install -I -r REQUIREMENTS
+$SUDO_WWW /usr/bin/scl enable rh-python36 "virtualenv -p python3 /var/www/MISP/venv"
-sudo pip3 install -I .
+$SUDO_WWW /var/www/MISP/venv/bin/pip install -U -I -r REQUIREMENTS
-sudo apt install ruby-pygments.rb -y
+$SUDO_WWW /var/www/MISP/venv/bin/pip install -U .
-sudo gem install asciidoctor-pdf --pre
+~~~~
-sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local
+
-/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules
+Create the service file /etc/systemd/system/misp-modules.service :
+
+~~~~bash
+echo "[Unit]
+Description=MISP's modules
+After=misp-workers.service
+
+[Service]
+Type=simple
+User=apache
+Group=apache
+ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22  '/var/www/MISP/venv/bin/misp-modules –l 127.0.0.1 –s'
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target" | sudo tee /etc/systemd/system/misp-modules.service
+~~~~
+
+The After=misp-workers.service must be changed or removed if you have not created a misp-workers service. Then, enable the misp-modules service and start it:
+
+~~~~bash
+systemctl daemon-reload
+systemctl enable --now misp-modules
 ~~~~
 
 ## How to use an MISP modules Docker container
 
+### Docker build
+
+~~~~bash
+docker build -t misp-modules \
+    --build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") \
+  docker/
+~~~~
+
 ### Docker run
 
 ~~~~bash
 # Start Redis
 docker run --rm -d --name=misp-redis redis:alpine
+# Start MISP-modules
 docker run \
     --rm -d --name=misp-modules \
     -e REDIS_BACKEND=misp-redis \
@@ -43,7 +116,7 @@ docker run \
     -e REDIS_PW="" \
     -e REDIS_DATABASE="245" \
     -e MISP_MODULES_DEBUG="false" \
-    dcso/misp-dockerized-redis
+    dcso/misp-dockerized-misp-modules
 ~~~~
 
 ### Docker-compose
@@ -53,6 +126,12 @@ services:
   misp-modules:
     # https://hub.docker.com/r/dcso/misp-dockerized-misp-modules
     image: dcso/misp-dockerized-misp-modules:3
+    
+    # Local image:
+    #image: misp-modules
+    #build:
+    #  context: docker/
+    
     environment:
       # Redis
       REDIS_BACKEND: misp-redis

misp_modules/modules/import_mod/cuckooimport.py

@@ -1,9 +1,11 @@
 import json
 import base64
-import tarfile
+import io
 import logging
 import posixpath
-from io import BytesIO, BufferedReader
+import stat
+import tarfile
+import zipfile
 from pymisp import MISPEvent, MISPObject, MISPAttribute
 from pymisp.tools import make_binary_objects
 from collections import OrderedDict
@@ -12,10 +14,14 @@ log = logging.getLogger(__name__)
 
 misperrors = {'error': 'Error'}
 
-moduleinfo = {'version': '1.0',
+moduleinfo = {
-              'author': 'Pierre-Jean Grenier',
+    'version': '1.1',
-              'description': 'Cuckoo archive import',
+    'author': 'Pierre-Jean Grenier',
-              'module-type': ['import']}
+    'description': "Import a Cuckoo archive (zipfile or bzip2 tarball), "
+                   "either downloaded manually or exported from the "
+                   "API (/tasks/report/{task_id}/all).",
+    'module-type': ['import'],
+}
 
 moduleconfig = []
 
@@ -202,13 +208,21 @@ class CuckooParser():
         self.files = None
         self.malware_binary = None
         self.report = None
-        self.config = {key: int(on) for key, on in config.items()}
+        self.config = {
+            # if an option is missing (we receive None as a value),
+            # fall back to the default specified in the options
+            key: int(
+                on if on is not None
+                else self.options[key]["userConfig"]["checked"] == 'true'
+            )
+            for key, on in config.items()
+        }
 
     def get_file(self, relative_filepath):
-        """Return a BufferedReader for the corresponding relative_filepath
+        """Return an io.BufferedIOBase for the corresponding relative_filepath
-        in the Cuckoo archive. If not found, return an empty BufferedReader
+        in the Cuckoo archive. If not found, return an empty io.BufferedReader
         to avoid fatal errors."""
-        blackhole = BufferedReader(open('/dev/null', 'rb'))
+        blackhole = io.BufferedReader(open('/dev/null', 'rb'))
         res = self.files.get(relative_filepath, blackhole)
         if res == blackhole:
             log.debug(f"Did not find file {relative_filepath}, "
@@ -220,12 +234,30 @@ class CuckooParser():
         # archive_encoded is base 64 encoded content
         # we extract the info about each file but do not retrieve
         # it automatically, as it may take too much space in memory
-        buf_io = BytesIO(base64.b64decode(archive_encoded))
+        buf_io = io.BytesIO(base64.b64decode(archive_encoded))
-        f = tarfile.open(fileobj=buf_io, mode='r:bz2')
+        if zipfile.is_zipfile(buf_io):
-        self.files = {
+            # the archive was probably downloaded from the WebUI
-            info.name: f.extractfile(info)
+            buf_io.seek(0)  # don't forget this not to read an empty buffer
-            for info in f.getmembers()
+            z = zipfile.ZipFile(buf_io, 'r')
-        }
+            self.files = {
+                info.filename: z.open(info)
+                for info in z.filelist
+                # only extract the regular files and dirs, we don't
+                # want any symbolic link
+                if stat.S_ISREG(info.external_attr >> 16)
+                or stat.S_ISDIR(info.external_attr >> 16)
+            }
+        else:
+            # the archive was probably downloaded from the API
+            buf_io.seek(0)  # don't forget this not to read an empty buffer
+            f = tarfile.open(fileobj=buf_io, mode='r:bz2')
+            self.files = {
+                info.name: f.extractfile(info)
+                for info in f.getmembers()
+                # only extract the regular files and dirs, we don't
+                # want any symbolic link
+                if info.isreg() or info.isdir()
+            }
 
         # We want to keep the order of the keys of sub-dicts in the report,
         # eg. the signatures have marks with unknown keys such as
@@ -280,7 +312,7 @@ class CuckooParser():
             log.debug("Sample is a file, uploading it")
             self.read_malware()
             file_o, bin_type_o, bin_section_li = make_binary_objects(
-                pseudofile=BytesIO(self.malware_binary),
+                pseudofile=io.BytesIO(self.malware_binary),
                 filename=target["file"]["name"],
             )
 
@@ -548,7 +580,7 @@ class CuckooParser():
             filename = posixpath.basename(path)
 
         dropped_file = self.get_file(path)
-        dropped_binary = BytesIO(dropped_file.read())
+        dropped_binary = io.BytesIO(dropped_file.read())
         # create ad hoc objects
         file_o, bin_type_o, bin_section_li = make_binary_objects(
             pseudofile=dropped_binary, filename=filename,

tests/test_expansions.py

@@ -43,7 +43,7 @@ class TestExpansions(unittest.TestCase):
         query = {"module": "hibp", "email-src": "info@circl.lu"}
         response = self.misp_modules_post(query)
         to_check = self.get_values(response)
-        if to_check == "haveibeenpwned.com API not accessible (HTTP 403)":
+        if to_check == "haveibeenpwned.com API not accessible (HTTP 401)":
             self.skipTest(f"haveibeenpwned blocks travis IPs: {response}")
         self.assertEqual(to_check, 'OK (Not Found)', response)