diff --git a/.travis.yml b/.travis.yml index 18c02c6..db66efd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,9 @@ python: - "3.6-dev" - "3.7-dev" +before_install: + - docker build -t misp-modules --build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") docker/ + install: - sudo apt-get install libzbar0 libzbar-dev libpoppler-cpp-dev - pip install pipenv diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..8ac6d9f --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,130 @@ +FROM python:3.7-buster AS build + +ENV DEBIAN_FRONTEND noninteractive +ENV WORKDIR="/usr/local/src/misp_modules" +ENV VENV_DIR="/misp_modules" + +# Install Packages for build +RUN set -eu \ + ;mkdir -p ${WORKDIR} ${VENV_DIR} \ + ;apt-get update \ + ;apt-get install -y \ + git \ + libpq5 \ + libjpeg-dev \ + tesseract-ocr \ + libpoppler-cpp-dev \ + imagemagick \ + virtualenv \ + libopencv-dev \ + zbar-tools \ + libzbar0 \ + libzbar-dev \ + libfuzzy-dev \ + ;apt-get -y autoremove \ + ;apt-get -y clean \ + ;rm -rf /var/lib/apt/lists/* \ + ; + +# Create MISP Modules +RUN set -eu \ + ;git clone https://github.com/MISP/misp-modules.git ${WORKDIR} \ + ;virtualenv -p python3 ${VENV_DIR}/venv \ + ;cd ${WORKDIR} \ + ;${VENV_DIR}/venv/bin/pip3 install -I -r REQUIREMENTS --no-cache-dir \ + ;${VENV_DIR}/venv/bin/pip3 install . --no-cache-dir \ + ; + +######################################### +# Start Final Docker Image +# +FROM python:3.7-slim-buster AS final + +ENV DEBIAN_FRONTEND noninteractive +ENV VENV_DIR="/misp_modules" + +# Copy all builded files from build stage +COPY --from=build ${VENV_DIR} ${VENV_DIR} + +# Install Packages to run it +RUN set -eu \ + ;apt-get update \ + ;apt-get install -y \ + curl \ + libpq5 \ + # libjpeg-dev \ + tesseract-ocr \ + libpoppler-cpp-dev \ + imagemagick \ + # virtualenv \ + # libopencv-dev \ + zbar-tools \ + libzbar0 \ + # libzbar-dev \ + # libfuzzy-dev \ + ;apt-get -y autoremove \ + ;apt-get -y clean \ + ;rm -rf /var/lib/apt/lists/* \ + ;chown -R nobody ${VENV_DIR} \ + ; + +# Entrypoint + COPY files/entrypoint.sh /entrypoint.sh + ENTRYPOINT [ "/entrypoint.sh" ] + +# Add Healthcheck Config + COPY files/healthcheck.sh /healthcheck.sh + HEALTHCHECK --interval=1m --timeout=45s --retries=3 CMD ["/healthcheck.sh"] + +# Change Workdir + WORKDIR ${VENV_DIR} + +# Change from root to www-data + USER nobody + +# Expose Port + EXPOSE 6666 + +# Shortterm ARG Variables: + ARG VENDOR="MISP" + ARG COMPONENT="misp-modules" + ARG BUILD_DATE + ARG GIT_REPO="https://github.com/MISP/misp-modules" + ARG VCS_REF + ARG RELEASE_DATE + ARG NAME="MISP-dockerized-misp-modules" + ARG DESCRIPTION="This docker container contains MISP modules in an Debian Container." + ARG DOCUMENTATION="https://misp.github.io/misp-modules/" + ARG AUTHOR="MISP" + ARG LICENSE="BSD-3-Clause" + +# Longterm Environment Variables +ENV \ + BUILD_DATE=${BUILD_DATE} \ + NAME=${NAME} \ + PATH=$PATH:${VENV_DIR}/venv/bin + +# Labels +LABEL org.label-schema.build-date="${BUILD_DATE}" \ + org.label-schema.name="${NAME}" \ + org.label-schema.description="${DESCRIPTION}" \ + org.label-schema.vcs-ref="${VCS_REF}" \ + org.label-schema.vcs-url="${GIT_REPO}" \ + org.label-schema.url="${GIT_REPO}" \ + org.label-schema.vendor="${VENDOR}" \ + org.label-schema.version="${VERSION}" \ + org.label-schema.usage="${DOCUMENTATION}" \ + org.label-schema.schema-version="1.0.0-rc1" + +LABEL org.opencontainers.image.created="${BUILD_DATE}" \ + org.opencontainers.image.url="${GIT_REPO}" \ + org.opencontainers.image.source="${GIT_REPO}" \ + org.opencontainers.image.version="${VERSION}" \ + org.opencontainers.image.revision="${VCS_REF}" \ + org.opencontainers.image.vendor="${VENDOR}" \ + org.opencontainers.image.title="${NAME}" \ + org.opencontainers.image.description="${DESCRIPTION}" \ + org.opencontainers.image.documentation="${DOCUMENTATION}" \ + org.opencontainers.image.authors="${AUTHOR}" \ + org.opencontainers.image.licenses="${LICENSE}" + diff --git a/docker/files/entrypoint.sh b/docker/files/entrypoint.sh new file mode 100755 index 0000000..73d8f39 --- /dev/null +++ b/docker/files/entrypoint.sh @@ -0,0 +1,37 @@ +#!/bin/sh +set -eu + +# Variables +NC='\033[0m' # No Color +Light_Green='\033[1;32m' +STARTMSG="${Light_Green}[ENTRYPOINT_MISP_MODULES]${NC}" +VENV_DIR=${VENV_DIR:-"/misp-modules"} +MISP_MODULES_BINARY="${VENV_DIR}/venv/bin/misp-modules" +DEBUG="" + +# Functions +echo (){ + command echo "$STARTMSG $*" +} + +# Environment Variables +MISP_MODULES_DEBUG=${MISP_MODULES_DEBUG:-"false"} + +# +# MAIN +# + + +# Check if debugging mode should be enabled +[ "$MISP_MODULES_DEBUG" = "true" ] && DEBUG="-d" + +# check if a command parameter exists and start misp-modules +if [ $# = 0 ] +then + # If no cmd parameter is set + echo "Start MISP Modules" && $MISP_MODULES_BINARY $DEBUG -l 0.0.0.0 > /dev/stdout 2> /dev/stderr +else + # If cmd parameter is set + echo "Start MISP Modules" && $MISP_MODULES_BINARY $DEBUG -l 0.0.0.0 > /dev/stdout 2> /dev/stderr & + exec "$@" +fi diff --git a/docker/files/healthcheck.sh b/docker/files/healthcheck.sh new file mode 100755 index 0000000..d6a1f91 --- /dev/null +++ b/docker/files/healthcheck.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +# If no contain is there or curl get an error back: exit 1. Docker restart then the container. +curl -fk http://0.0.0.0:6666/modules || exit 1 \ No newline at end of file diff --git a/docs/install.md b/docs/install.md index 7fbd9c7..72cf9d6 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1,41 +1,114 @@ -## How to install and start MISP modules in a Python virtualenv? +## How to install and start MISP modules (in a Python virtualenv)? ~~~~bash -sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick -sudo -u www-data virtualenv -p python3 /var/www/MISP/venv +SUDO_WWW="sudo -u www-data" + +sudo apt-get install -y \ + git \ + libpq5 \ + libjpeg-dev \ + tesseract-ocr \ + libpoppler-cpp-dev \ + imagemagick virtualenv \ + libopencv-dev \ + zbar-tools \ + libzbar0 \ + libzbar-dev \ + libfuzzy-dev + +# BEGIN with virtualenv: +$SUDO_WWW virtualenv -p python3 /var/www/MISP/venv +# END with virtualenv + cd /usr/local/src/ sudo git clone https://github.com/MISP/misp-modules.git cd misp-modules -sudo -u www-data /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS -sudo -u www-data /var/www/MISP/venv/bin/pip install . -sudo apt install ruby-pygments.rb -y -sudo gem install asciidoctor-pdf --pre -sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local + +# BEGIN with virtualenv: +$SUDO_WWW /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS +$SUDO_WWW /var/www/MISP/venv/bin/pip install . +# END with virtualenv + +# BEGIN without virtualenv: +sudo pip install -I -r REQUIREMENTS +sudo pip install . +# END without virtualenv + +# Start misp-modules as a service +sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now misp-modules /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules ~~~~ -## How to install and start MISP modules? +## How to install and start MISP modules on RHEL-based distributions ? + +As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the SCL repository. ~~~~bash -sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick +SUDO_WWW="sudo -u apache" +sudo yum install \ + rh-ruby22 \ + openjpeg-devel \ + rubygem-rouge \ + rubygem-asciidoctor \ + zbar-devel \ + opencv-devel \ + gcc-c++ \ + pkgconfig \ + poppler-cpp-devel \ + python-devel \ + redhat-rpm-config cd /usr/local/src/ sudo git clone https://github.com/MISP/misp-modules.git cd misp-modules -sudo pip3 install -I -r REQUIREMENTS -sudo pip3 install -I . -sudo apt install ruby-pygments.rb -y -sudo gem install asciidoctor-pdf --pre -sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local -/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules +$SUDO_WWW /usr/bin/scl enable rh-python36 "virtualenv -p python3 /var/www/MISP/venv" +$SUDO_WWW /var/www/MISP/venv/bin/pip install -U -I -r REQUIREMENTS +$SUDO_WWW /var/www/MISP/venv/bin/pip install -U . +~~~~ + +Create the service file /etc/systemd/system/misp-modules.service : + +~~~~bash +echo "[Unit] +Description=MISP's modules +After=misp-workers.service + +[Service] +Type=simple +User=apache +Group=apache +ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 '/var/www/MISP/venv/bin/misp-modules –l 127.0.0.1 –s' +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target" | sudo tee /etc/systemd/system/misp-modules.service +~~~~ + +The After=misp-workers.service must be changed or removed if you have not created a misp-workers service. Then, enable the misp-modules service and start it: + +~~~~bash +systemctl daemon-reload +systemctl enable --now misp-modules ~~~~ ## How to use an MISP modules Docker container +### Docker build + +~~~~bash +docker build -t misp-modules \ + --build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") \ + docker/ +~~~~ + ### Docker run ~~~~bash # Start Redis docker run --rm -d --name=misp-redis redis:alpine +# Start MISP-modules docker run \ --rm -d --name=misp-modules \ -e REDIS_BACKEND=misp-redis \ @@ -43,7 +116,7 @@ docker run \ -e REDIS_PW="" \ -e REDIS_DATABASE="245" \ -e MISP_MODULES_DEBUG="false" \ - dcso/misp-dockerized-redis + dcso/misp-dockerized-misp-modules ~~~~ ### Docker-compose @@ -53,6 +126,12 @@ services: misp-modules: # https://hub.docker.com/r/dcso/misp-dockerized-misp-modules image: dcso/misp-dockerized-misp-modules:3 + + # Local image: + #image: misp-modules + #build: + # context: docker/ + environment: # Redis REDIS_BACKEND: misp-redis diff --git a/misp_modules/modules/import_mod/cuckooimport.py b/misp_modules/modules/import_mod/cuckooimport.py index c89e88d..3ed52bd 100755 --- a/misp_modules/modules/import_mod/cuckooimport.py +++ b/misp_modules/modules/import_mod/cuckooimport.py @@ -1,9 +1,11 @@ import json import base64 -import tarfile +import io import logging import posixpath -from io import BytesIO, BufferedReader +import stat +import tarfile +import zipfile from pymisp import MISPEvent, MISPObject, MISPAttribute from pymisp.tools import make_binary_objects from collections import OrderedDict @@ -12,10 +14,14 @@ log = logging.getLogger(__name__) misperrors = {'error': 'Error'} -moduleinfo = {'version': '1.0', - 'author': 'Pierre-Jean Grenier', - 'description': 'Cuckoo archive import', - 'module-type': ['import']} +moduleinfo = { + 'version': '1.1', + 'author': 'Pierre-Jean Grenier', + 'description': "Import a Cuckoo archive (zipfile or bzip2 tarball), " + "either downloaded manually or exported from the " + "API (/tasks/report/{task_id}/all).", + 'module-type': ['import'], +} moduleconfig = [] @@ -202,13 +208,21 @@ class CuckooParser(): self.files = None self.malware_binary = None self.report = None - self.config = {key: int(on) for key, on in config.items()} + self.config = { + # if an option is missing (we receive None as a value), + # fall back to the default specified in the options + key: int( + on if on is not None + else self.options[key]["userConfig"]["checked"] == 'true' + ) + for key, on in config.items() + } def get_file(self, relative_filepath): - """Return a BufferedReader for the corresponding relative_filepath - in the Cuckoo archive. If not found, return an empty BufferedReader + """Return an io.BufferedIOBase for the corresponding relative_filepath + in the Cuckoo archive. If not found, return an empty io.BufferedReader to avoid fatal errors.""" - blackhole = BufferedReader(open('/dev/null', 'rb')) + blackhole = io.BufferedReader(open('/dev/null', 'rb')) res = self.files.get(relative_filepath, blackhole) if res == blackhole: log.debug(f"Did not find file {relative_filepath}, " @@ -220,12 +234,30 @@ class CuckooParser(): # archive_encoded is base 64 encoded content # we extract the info about each file but do not retrieve # it automatically, as it may take too much space in memory - buf_io = BytesIO(base64.b64decode(archive_encoded)) - f = tarfile.open(fileobj=buf_io, mode='r:bz2') - self.files = { - info.name: f.extractfile(info) - for info in f.getmembers() - } + buf_io = io.BytesIO(base64.b64decode(archive_encoded)) + if zipfile.is_zipfile(buf_io): + # the archive was probably downloaded from the WebUI + buf_io.seek(0) # don't forget this not to read an empty buffer + z = zipfile.ZipFile(buf_io, 'r') + self.files = { + info.filename: z.open(info) + for info in z.filelist + # only extract the regular files and dirs, we don't + # want any symbolic link + if stat.S_ISREG(info.external_attr >> 16) + or stat.S_ISDIR(info.external_attr >> 16) + } + else: + # the archive was probably downloaded from the API + buf_io.seek(0) # don't forget this not to read an empty buffer + f = tarfile.open(fileobj=buf_io, mode='r:bz2') + self.files = { + info.name: f.extractfile(info) + for info in f.getmembers() + # only extract the regular files and dirs, we don't + # want any symbolic link + if info.isreg() or info.isdir() + } # We want to keep the order of the keys of sub-dicts in the report, # eg. the signatures have marks with unknown keys such as @@ -280,7 +312,7 @@ class CuckooParser(): log.debug("Sample is a file, uploading it") self.read_malware() file_o, bin_type_o, bin_section_li = make_binary_objects( - pseudofile=BytesIO(self.malware_binary), + pseudofile=io.BytesIO(self.malware_binary), filename=target["file"]["name"], ) @@ -548,7 +580,7 @@ class CuckooParser(): filename = posixpath.basename(path) dropped_file = self.get_file(path) - dropped_binary = BytesIO(dropped_file.read()) + dropped_binary = io.BytesIO(dropped_file.read()) # create ad hoc objects file_o, bin_type_o, bin_section_li = make_binary_objects( pseudofile=dropped_binary, filename=filename, diff --git a/tests/test_expansions.py b/tests/test_expansions.py index 493cb4d..364f63b 100644 --- a/tests/test_expansions.py +++ b/tests/test_expansions.py @@ -43,7 +43,7 @@ class TestExpansions(unittest.TestCase): query = {"module": "hibp", "email-src": "info@circl.lu"} response = self.misp_modules_post(query) to_check = self.get_values(response) - if to_check == "haveibeenpwned.com API not accessible (HTTP 403)": + if to_check == "haveibeenpwned.com API not accessible (HTTP 401)": self.skipTest(f"haveibeenpwned blocks travis IPs: {response}") self.assertEqual(to_check, 'OK (Not Found)', response)