From 696bafa749da1cbc0866f332ab801bd90fe06962 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Mon, 19 Aug 2019 11:37:43 +0200 Subject: [PATCH 01/13] fix: have I been pwned API changed again. --- tests/test_expansions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_expansions.py b/tests/test_expansions.py index 493cb4d..364f63b 100644 --- a/tests/test_expansions.py +++ b/tests/test_expansions.py @@ -43,7 +43,7 @@ class TestExpansions(unittest.TestCase): query = {"module": "hibp", "email-src": "info@circl.lu"} response = self.misp_modules_post(query) to_check = self.get_values(response) - if to_check == "haveibeenpwned.com API not accessible (HTTP 403)": + if to_check == "haveibeenpwned.com API not accessible (HTTP 401)": self.skipTest(f"haveibeenpwned blocks travis IPs: {response}") self.assertEqual(to_check, 'OK (Not Found)', response) From 413cc2469fa7dacc4e0f07dbe87063771fbfb01e Mon Sep 17 00:00:00 2001 From: Pierre-Jean Grenier Date: Wed, 21 Aug 2019 16:35:11 +0200 Subject: [PATCH 02/13] chg: [cuckooimport] Handle archives downloaded from both the WebUI and the API --- .../modules/import_mod/cuckooimport.py | 60 +++++++++++++------ 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/misp_modules/modules/import_mod/cuckooimport.py b/misp_modules/modules/import_mod/cuckooimport.py index c89e88d..ddb8957 100755 --- a/misp_modules/modules/import_mod/cuckooimport.py +++ b/misp_modules/modules/import_mod/cuckooimport.py @@ -1,9 +1,10 @@ import json import base64 -import tarfile +import io import logging import posixpath -from io import BytesIO, BufferedReader +import tarfile +import zipfile from pymisp import MISPEvent, MISPObject, MISPAttribute from pymisp.tools import make_binary_objects from collections import OrderedDict @@ -12,10 +13,14 @@ log = logging.getLogger(__name__) misperrors = {'error': 'Error'} -moduleinfo = {'version': '1.0', - 'author': 'Pierre-Jean Grenier', - 'description': 'Cuckoo archive import', - 'module-type': ['import']} +moduleinfo = { + 'version': '1.1', + 'author': 'Pierre-Jean Grenier', + 'description': "Import a Cuckoo archive (zipfile or bzip2 tarball), " + "either downloaded manually or exported from the " + "API (/tasks/report/{task_id}/all).", + 'module-type': ['import'], +} moduleconfig = [] @@ -202,13 +207,21 @@ class CuckooParser(): self.files = None self.malware_binary = None self.report = None - self.config = {key: int(on) for key, on in config.items()} + self.config = { + # if an option is missing (we receive None as a value), + # fall back to the default specified in the options + key: int( + on if on is not None + else self.options[key]["userConfig"]["checked"] == 'true' + ) + for key, on in config.items() + } def get_file(self, relative_filepath): - """Return a BufferedReader for the corresponding relative_filepath - in the Cuckoo archive. If not found, return an empty BufferedReader + """Return an io.BufferedIOBase for the corresponding relative_filepath + in the Cuckoo archive. If not found, return an empty io.BufferedReader to avoid fatal errors.""" - blackhole = BufferedReader(open('/dev/null', 'rb')) + blackhole = io.BufferedReader(open('/dev/null', 'rb')) res = self.files.get(relative_filepath, blackhole) if res == blackhole: log.debug(f"Did not find file {relative_filepath}, " @@ -220,12 +233,23 @@ class CuckooParser(): # archive_encoded is base 64 encoded content # we extract the info about each file but do not retrieve # it automatically, as it may take too much space in memory - buf_io = BytesIO(base64.b64decode(archive_encoded)) - f = tarfile.open(fileobj=buf_io, mode='r:bz2') - self.files = { - info.name: f.extractfile(info) - for info in f.getmembers() - } + buf_io = io.BytesIO(base64.b64decode(archive_encoded)) + if zipfile.is_zipfile(buf_io): + # the archive was probably downloaded from the WebUI + buf_io.seek(0) # don't forget this not to read an empty buffer + z = zipfile.ZipFile(buf_io, 'r') + self.files = { + info.filename: z.open(info) + for info in z.filelist + } + else: + # the archive was probably downloaded from the API + buf_io.seek(0) # don't forget this not to read an empty buffer + f = tarfile.open(fileobj=buf_io, mode='r:bz2') + self.files = { + info.name: f.extractfile(info) + for info in f.getmembers() + } # We want to keep the order of the keys of sub-dicts in the report, # eg. the signatures have marks with unknown keys such as @@ -280,7 +304,7 @@ class CuckooParser(): log.debug("Sample is a file, uploading it") self.read_malware() file_o, bin_type_o, bin_section_li = make_binary_objects( - pseudofile=BytesIO(self.malware_binary), + pseudofile=io.BytesIO(self.malware_binary), filename=target["file"]["name"], ) @@ -548,7 +572,7 @@ class CuckooParser(): filename = posixpath.basename(path) dropped_file = self.get_file(path) - dropped_binary = BytesIO(dropped_file.read()) + dropped_binary = io.BytesIO(dropped_file.read()) # create ad hoc objects file_o, bin_type_o, bin_section_li = make_binary_objects( pseudofile=dropped_binary, filename=filename, From b2ab727f9b2a2abf48128a606a10aba71b31836a Mon Sep 17 00:00:00 2001 From: Pierre-Jean Grenier Date: Thu, 22 Aug 2019 11:16:18 +0200 Subject: [PATCH 03/13] fix: prevent symlink attacks --- misp_modules/modules/import_mod/cuckooimport.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/misp_modules/modules/import_mod/cuckooimport.py b/misp_modules/modules/import_mod/cuckooimport.py index ddb8957..3ed52bd 100755 --- a/misp_modules/modules/import_mod/cuckooimport.py +++ b/misp_modules/modules/import_mod/cuckooimport.py @@ -3,6 +3,7 @@ import base64 import io import logging import posixpath +import stat import tarfile import zipfile from pymisp import MISPEvent, MISPObject, MISPAttribute @@ -241,6 +242,10 @@ class CuckooParser(): self.files = { info.filename: z.open(info) for info in z.filelist + # only extract the regular files and dirs, we don't + # want any symbolic link + if stat.S_ISREG(info.external_attr >> 16) + or stat.S_ISDIR(info.external_attr >> 16) } else: # the archive was probably downloaded from the API @@ -249,6 +254,9 @@ class CuckooParser(): self.files = { info.name: f.extractfile(info) for info in f.getmembers() + # only extract the regular files and dirs, we don't + # want any symbolic link + if info.isreg() or info.isdir() } # We want to keep the order of the keys of sub-dicts in the report, From ed1ebef7b328ca073ba4356cf727b7d84e7018f1 Mon Sep 17 00:00:00 2001 From: 8ear Date: Mon, 2 Sep 2019 10:03:32 +0200 Subject: [PATCH 04/13] Bugfixing for MISP-modules --- docs/install.md | 31 +++++++++---------------------- 1 file changed, 9 insertions(+), 22 deletions(-) diff --git a/docs/install.md b/docs/install.md index 7fbd9c7..bc3a13a 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1,29 +1,15 @@ -## How to install and start MISP modules in a Python virtualenv? +## How to install and start MISP modules (in a Python virtualenv)? ~~~~bash -sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick -sudo -u www-data virtualenv -p python3 /var/www/MISP/venv +sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick ruby-pygments.rb +# With virtualenv: sudo -u www-data virtualenv -p python3 /var/www/MISP/venv cd /usr/local/src/ sudo git clone https://github.com/MISP/misp-modules.git cd misp-modules -sudo -u www-data /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS -sudo -u www-data /var/www/MISP/venv/bin/pip install . -sudo apt install ruby-pygments.rb -y -sudo gem install asciidoctor-pdf --pre -sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local -/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules -~~~~ - -## How to install and start MISP modules? - -~~~~bash -sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick -cd /usr/local/src/ -sudo git clone https://github.com/MISP/misp-modules.git -cd misp-modules -sudo pip3 install -I -r REQUIREMENTS -sudo pip3 install -I . -sudo apt install ruby-pygments.rb -y +# With virtualenv: sudo -u www-data /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS +# With virtualenv: sudo -u www-data /var/www/MISP/venv/bin/pip install . +# Without virtualenv: pip install -I -r REQUIREMENTS +# Without virtualenv: pip install . sudo gem install asciidoctor-pdf --pre sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules @@ -36,6 +22,7 @@ sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127. ~~~~bash # Start Redis docker run --rm -d --name=misp-redis redis:alpine +# Start MISP-modules docker run \ --rm -d --name=misp-modules \ -e REDIS_BACKEND=misp-redis \ @@ -43,7 +30,7 @@ docker run \ -e REDIS_PW="" \ -e REDIS_DATABASE="245" \ -e MISP_MODULES_DEBUG="false" \ - dcso/misp-dockerized-redis + dcso/misp-dockerized-misp-modules ~~~~ ### Docker-compose From a5345c52c8c9563d05fa185a322c6f744ec7a13b Mon Sep 17 00:00:00 2001 From: 8ear Date: Mon, 2 Sep 2019 10:21:37 +0200 Subject: [PATCH 05/13] Update install doc --- docs/install.md | 63 ++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 57 insertions(+), 6 deletions(-) diff --git a/docs/install.md b/docs/install.md index bc3a13a..f1a7469 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1,20 +1,71 @@ ## How to install and start MISP modules (in a Python virtualenv)? ~~~~bash -sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick ruby-pygments.rb +sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr libpoppler-cpp-dev imagemagick virtualenv libopencv-dev zbar-tools libzbar0 libzbar-dev libfuzzy-dev # With virtualenv: sudo -u www-data virtualenv -p python3 /var/www/MISP/venv cd /usr/local/src/ sudo git clone https://github.com/MISP/misp-modules.git cd misp-modules -# With virtualenv: sudo -u www-data /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS -# With virtualenv: sudo -u www-data /var/www/MISP/venv/bin/pip install . -# Without virtualenv: pip install -I -r REQUIREMENTS -# Without virtualenv: pip install . -sudo gem install asciidoctor-pdf --pre + +# BEGIN with virtualenv: +sudo -u www-data /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS +sudo -u www-data /var/www/MISP/venv/bin/pip install . +# END with virtualenv + +# BEGIN without virtualenv: +pip install -I -r REQUIREMENTS +pip install . +# END without virtualenv + +# To start after reboot: sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local + +# Start the Module: /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules ~~~~ +## How to install and start MISP modules on RHEL-based distributions ? + +As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the SCL repository. + +~~~~bash +sudo yum install rh-ruby22 +sudo yum install openjpeg-devel +sudo yum install rubygem-rouge rubygem-asciidoctor zbar-devel opencv-devel gcc-c++ pkgconfig poppler-cpp-devel python-devel redhat-rpm-config +cd /var/www/MISP +git clone https://github.com/MISP/misp-modules.git +cd misp-modules +sudo -u apache /usr/bin/scl enable rh-python36 "virtualenv -p python3 /var/www/MISP/venv" +sudo -u apache /var/www/MISP/venv/bin/pip install -U -I -r REQUIREMENTS +sudo -u apache /var/www/MISP/venv/bin/pip install -U . +~~~~ + +Create the service file /etc/systemd/system/misp-modules.service : + +~~~~bash +echo "[Unit] +Description=MISP's modules +After=misp-workers.service + +[Service] +Type=simple +User=apache +Group=apache +ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 '/var/www/MISP/venv/bin/misp-modules –l 127.0.0.1 –s' +Restart=always +RestartSec=10 + +[Install] +WantedBy=multi-user.target" | sudo tee /etc/systemd/system/misp-modules.service +~~~~ + +The After=misp-workers.service must be changed or removed if you have not created a misp-workers service. Then, enable the misp-modules service and start it: + +~~~~bash +systemctl daemon-reload +systemctl enable --now misp-modules +~~~~ + ## How to use an MISP modules Docker container ### Docker run From 3eee1c88f32d541aa10e0d8fbf582b89dbf51c9b Mon Sep 17 00:00:00 2001 From: 8ear Date: Mon, 2 Sep 2019 11:44:54 +0200 Subject: [PATCH 06/13] Change Install documentation --- docs/install.md | 32 +++++++++++++++++++++++++++----- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/docs/install.md b/docs/install.md index f1a7469..2d6fde7 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1,7 +1,21 @@ ## How to install and start MISP modules (in a Python virtualenv)? +Required Packages to install: +{!apt_package.list!} + ~~~~bash -sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr libpoppler-cpp-dev imagemagick virtualenv libopencv-dev zbar-tools libzbar0 libzbar-dev libfuzzy-dev +sudo apt-get install -y \ + git \ + libpq5 \ + libjpeg-dev \ + tesseract-ocr \ + libpoppler-cpp-dev \ + imagemagick virtualenv \ + libopencv-dev \ + zbar-tools \ + libzbar0 \ + libzbar-dev \ + libfuzzy-dev # With virtualenv: sudo -u www-data virtualenv -p python3 /var/www/MISP/venv cd /usr/local/src/ sudo git clone https://github.com/MISP/misp-modules.git @@ -17,10 +31,10 @@ pip install -I -r REQUIREMENTS pip install . # END without virtualenv -# To start after reboot: -sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local - -# Start the Module: +# Start misp-modules as a service +sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/ +sudo systemctl daemon-reload +sudo systemctl enable --now misp-modules /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules ~~~~ @@ -68,6 +82,14 @@ systemctl enable --now misp-modules ## How to use an MISP modules Docker container +### Docker build + +~~~~bash +docker build -t misp-modules \ + --build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") \ + docker/ +~~~~ + ### Docker run ~~~~bash From 241824870e9a7057eaead167209c0e3d882e9ccf Mon Sep 17 00:00:00 2001 From: 8ear Date: Mon, 2 Sep 2019 11:45:26 +0200 Subject: [PATCH 07/13] Add Dockerfile, Entrypoint and Healthcheck script --- docker/Dockerfile | 129 ++++++++++++++++++++++++++++++++++++ docker/files/entrypoint.sh | 37 +++++++++++ docker/files/healthcheck.sh | 4 ++ 3 files changed, 170 insertions(+) create mode 100644 docker/Dockerfile create mode 100755 docker/files/entrypoint.sh create mode 100755 docker/files/healthcheck.sh diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..579f56f --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,129 @@ +FROM python:3.7-buster AS build + +ENV DEBIAN_FRONTEND noninteractive +ENV WORKDIR="/usr/local/src/misp_modules" +ENV VENV_DIR="/misp_modules" + +# Install Packages for build +RUN set -eu \ + ;mkdir -p ${WORKDIR} ${VENV_DIR} \ + ;apt-get update \ + ;apt-get install -y \ + git \ + libpq5 \ + libjpeg-dev \ + tesseract-ocr \ + libpoppler-cpp-dev \ + imagemagick virtualenv \ + libopencv-dev \ + zbar-tools \ + libzbar0 \ + libzbar-dev \ + libfuzzy-dev \ + ;apt-get -y autoremove \ + ;apt-get -y clean \ + ;rm -rf /var/lib/apt/lists/* \ + ; + +# Create MISP Modules +RUN set -eu \ + ;git clone https://github.com/MISP/misp-modules.git ${WORKDIR} \ + ;virtualenv -p python3 ${VENV_DIR}/venv \ + ;cd ${WORKDIR} \ + ;${VENV_DIR}/venv/bin/pip3 install -I -r REQUIREMENTS --no-cache-dir \ + ;${VENV_DIR}/venv/bin/pip3 install . --no-cache-dir \ + ;chown -R nobody ${VENV_DIR} \ + ;rm -rf ${WORKDIR} \ + ; + +######################################### + +FROM python:3.7-slim-buster AS final + +ENV DEBIAN_FRONTEND noninteractive +ENV VENV_DIR="/misp_modules" + +# Copy all builded files from build stage +COPY --from=build ${VENV_DIR} ${VENV_DIR} + +# Install Packages to run it +RUN set -eu \ + ;apt-get update \ + ;apt-get install -y \ + curl \ + libpq5 \ + libjpeg-dev \ + tesseract-ocr \ + libpoppler-cpp-dev \ + imagemagick virtualenv \ + libopencv-dev \ + zbar-tools \ + libzbar0 \ + libzbar-dev \ + libfuzzy-dev \ + ;apt-get -y autoremove \ + ;apt-get -y clean \ + ;rm -rf /var/lib/apt/lists/* \ + ;chown -R nobody ${VENV_DIR} \ + ; + +# Entrypoint + COPY files/entrypoint.sh /entrypoint.sh + ENTRYPOINT [ "/entrypoint.sh" ] + +# Add Healthcheck Config + COPY files/healthcheck.sh /healthcheck.sh + HEALTHCHECK --interval=1m --timeout=45s --retries=3 CMD ["/healthcheck.sh"] + +# Change Workdir + WORKDIR ${VENV_DIR} + +# Change from root to www-data + USER nobody + +# Expose Port + EXPOSE 6666 + +# Shortterm ARG Variables: + ARG VENDOR="MISP" + ARG COMPONENT="misp-modules" + ARG BUILD_DATE + ARG GIT_REPO="https://github.com/MISP/misp-modules" + ARG VCS_REF + ARG RELEASE_DATE + ARG NAME="MISP-dockerized-misp-modules" + ARG DESCRIPTION="This docker container contains MISP modules in an Debian Container." + ARG DOCUMENTATION="https://misp.github.io/misp-modules/" + ARG AUTHOR="MISP" + ARG LICENSE="BSD-3-Clause" + +# Longterm Environment Variables +ENV \ + BUILD_DATE=${BUILD_DATE} \ + NAME=${NAME} \ + PATH=$PATH:${VENV_DIR}/venv/bin + +# Labels +LABEL org.label-schema.build-date="${BUILD_DATE}" \ + org.label-schema.name="${NAME}" \ + org.label-schema.description="${DESCRIPTION}" \ + org.label-schema.vcs-ref="${VCS_REF}" \ + org.label-schema.vcs-url="${GIT_REPO}" \ + org.label-schema.url="${GIT_REPO}" \ + org.label-schema.vendor="${VENDOR}" \ + org.label-schema.version="${VERSION}" \ + org.label-schema.usage="${DOCUMENTATION}" \ + org.label-schema.schema-version="1.0.0-rc1" + +LABEL org.opencontainers.image.created="${BUILD_DATE}" \ + org.opencontainers.image.url="${GIT_REPO}" \ + org.opencontainers.image.source="${GIT_REPO}" \ + org.opencontainers.image.version="${VERSION}" \ + org.opencontainers.image.revision="${VCS_REF}" \ + org.opencontainers.image.vendor="${VENDOR}" \ + org.opencontainers.image.title="${NAME}" \ + org.opencontainers.image.description="${DESCRIPTION}" \ + org.opencontainers.image.documentation="${DOCUMENTATION}" \ + org.opencontainers.image.authors="${AUTHOR}" \ + org.opencontainers.image.licenses="${LICENSE}" + diff --git a/docker/files/entrypoint.sh b/docker/files/entrypoint.sh new file mode 100755 index 0000000..fda2af4 --- /dev/null +++ b/docker/files/entrypoint.sh @@ -0,0 +1,37 @@ +#!/bin/sh +set -eu + +# Variables +NC='\033[0m' # No Color +Light_Green='\033[1;32m' +STARTMSG="${Light_Green}[ENTRYPOINT_MISP_MODULES]${NC}" +VENV_DIR=${VENV_DIR:-"/misp-modules"} +MISP_MODULES_BINARY="${VENV_DIR}/venv/bin/misp-modules" +DEBUG="" + +# Functions +echo (){ + command echo -e "$STARTMSG $*" +} + +# Environment Variables +MISP_MODULES_DEBUG=${MISP_MODULES_DEBUG:-"false"} + +# +# MAIN +# + + +# Check if debugging mode should be enabled +[ "$MISP_MODULES_DEBUG" = "true" ] && DEBUG="-d" + +# check if a command parameter exists and start misp-modules +if [ $# = 0 ] +then + # If no cmd parameter is set + echo "Start MISP Modules" && $MISP_MODULES_BINARY $DEBUG -l 0.0.0.0 > /dev/stdout 2> /dev/stderr +else + # If cmd parameter is set + echo "Start MISP Modules" && $MISP_MODULES_BINARY $DEBUG -l 0.0.0.0 > /dev/stdout 2> /dev/stderr & + exec "$@" +fi diff --git a/docker/files/healthcheck.sh b/docker/files/healthcheck.sh new file mode 100755 index 0000000..d6a1f91 --- /dev/null +++ b/docker/files/healthcheck.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +# If no contain is there or curl get an error back: exit 1. Docker restart then the container. +curl -fk http://0.0.0.0:6666/modules || exit 1 \ No newline at end of file From 33f858fe977019c5a497554e9472b0118b479d10 Mon Sep 17 00:00:00 2001 From: 8ear Date: Mon, 2 Sep 2019 11:49:56 +0200 Subject: [PATCH 08/13] Fix Install.md --- docs/install.md | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/docs/install.md b/docs/install.md index 2d6fde7..0efda30 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1,8 +1,5 @@ ## How to install and start MISP modules (in a Python virtualenv)? -Required Packages to install: -{!apt_package.list!} - ~~~~bash sudo apt-get install -y \ git \ @@ -43,10 +40,20 @@ sudo systemctl enable --now misp-modules As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the SCL repository. ~~~~bash -sudo yum install rh-ruby22 -sudo yum install openjpeg-devel -sudo yum install rubygem-rouge rubygem-asciidoctor zbar-devel opencv-devel gcc-c++ pkgconfig poppler-cpp-devel python-devel redhat-rpm-config -cd /var/www/MISP +sudo yum install \ + rh-ruby22 \ + openjpeg-devel \ + rubygem-rouge \ + rubygem-asciidoctor \ + zbar-devel \ + opencv-devel \ + gcc-c++ \ + pkgconfig \ + poppler-cpp-devel \ + python-devel \ + redhat-rpm-config + +cd /usr/local/src/ git clone https://github.com/MISP/misp-modules.git cd misp-modules sudo -u apache /usr/bin/scl enable rh-python36 "virtualenv -p python3 /var/www/MISP/venv" From d7bf9e4df8524d5c1b35bf3061301dfb397c102c Mon Sep 17 00:00:00 2001 From: 8ear Date: Mon, 2 Sep 2019 11:56:04 +0200 Subject: [PATCH 09/13] Fixing Install.md --- docs/install.md | 54 ++++++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/docs/install.md b/docs/install.md index 0efda30..72cf9d6 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1,31 +1,37 @@ ## How to install and start MISP modules (in a Python virtualenv)? ~~~~bash +SUDO_WWW="sudo -u www-data" + sudo apt-get install -y \ - git \ - libpq5 \ - libjpeg-dev \ - tesseract-ocr \ - libpoppler-cpp-dev \ - imagemagick virtualenv \ - libopencv-dev \ - zbar-tools \ - libzbar0 \ - libzbar-dev \ - libfuzzy-dev -# With virtualenv: sudo -u www-data virtualenv -p python3 /var/www/MISP/venv + git \ + libpq5 \ + libjpeg-dev \ + tesseract-ocr \ + libpoppler-cpp-dev \ + imagemagick virtualenv \ + libopencv-dev \ + zbar-tools \ + libzbar0 \ + libzbar-dev \ + libfuzzy-dev + +# BEGIN with virtualenv: +$SUDO_WWW virtualenv -p python3 /var/www/MISP/venv +# END with virtualenv + cd /usr/local/src/ sudo git clone https://github.com/MISP/misp-modules.git cd misp-modules # BEGIN with virtualenv: -sudo -u www-data /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS -sudo -u www-data /var/www/MISP/venv/bin/pip install . +$SUDO_WWW /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS +$SUDO_WWW /var/www/MISP/venv/bin/pip install . # END with virtualenv # BEGIN without virtualenv: -pip install -I -r REQUIREMENTS -pip install . +sudo pip install -I -r REQUIREMENTS +sudo pip install . # END without virtualenv # Start misp-modules as a service @@ -40,6 +46,7 @@ sudo systemctl enable --now misp-modules As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the SCL repository. ~~~~bash +SUDO_WWW="sudo -u apache" sudo yum install \ rh-ruby22 \ openjpeg-devel \ @@ -52,13 +59,12 @@ sudo yum install \ poppler-cpp-devel \ python-devel \ redhat-rpm-config - cd /usr/local/src/ -git clone https://github.com/MISP/misp-modules.git +sudo git clone https://github.com/MISP/misp-modules.git cd misp-modules -sudo -u apache /usr/bin/scl enable rh-python36 "virtualenv -p python3 /var/www/MISP/venv" -sudo -u apache /var/www/MISP/venv/bin/pip install -U -I -r REQUIREMENTS -sudo -u apache /var/www/MISP/venv/bin/pip install -U . +$SUDO_WWW /usr/bin/scl enable rh-python36 "virtualenv -p python3 /var/www/MISP/venv" +$SUDO_WWW /var/www/MISP/venv/bin/pip install -U -I -r REQUIREMENTS +$SUDO_WWW /var/www/MISP/venv/bin/pip install -U . ~~~~ Create the service file /etc/systemd/system/misp-modules.service : @@ -120,6 +126,12 @@ services: misp-modules: # https://hub.docker.com/r/dcso/misp-dockerized-misp-modules image: dcso/misp-dockerized-misp-modules:3 + + # Local image: + #image: misp-modules + #build: + # context: docker/ + environment: # Redis REDIS_BACKEND: misp-redis From e82789cba82048df124a08e3c0d5b48750a67e5e Mon Sep 17 00:00:00 2001 From: 8ear Date: Mon, 2 Sep 2019 12:12:31 +0200 Subject: [PATCH 10/13] Improve the Dockerfile --- docker/Dockerfile | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 579f56f..e7a4eec 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -14,7 +14,8 @@ RUN set -eu \ libjpeg-dev \ tesseract-ocr \ libpoppler-cpp-dev \ - imagemagick virtualenv \ + imagemagick \ + virtualenv \ libopencv-dev \ zbar-tools \ libzbar0 \ @@ -32,12 +33,11 @@ RUN set -eu \ ;cd ${WORKDIR} \ ;${VENV_DIR}/venv/bin/pip3 install -I -r REQUIREMENTS --no-cache-dir \ ;${VENV_DIR}/venv/bin/pip3 install . --no-cache-dir \ - ;chown -R nobody ${VENV_DIR} \ - ;rm -rf ${WORKDIR} \ ; ######################################### - +# Start Final Docker Image +# FROM python:3.7-slim-buster AS final ENV DEBIAN_FRONTEND noninteractive @@ -52,15 +52,16 @@ RUN set -eu \ ;apt-get install -y \ curl \ libpq5 \ - libjpeg-dev \ + # libjpeg-dev \ tesseract-ocr \ libpoppler-cpp-dev \ - imagemagick virtualenv \ - libopencv-dev \ + imagemagick \ + virtualenv \ + # libopencv-dev \ zbar-tools \ libzbar0 \ - libzbar-dev \ - libfuzzy-dev \ + # libzbar-dev \ + # libfuzzy-dev \ ;apt-get -y autoremove \ ;apt-get -y clean \ ;rm -rf /var/lib/apt/lists/* \ From cdbe99824ec50a22af1e91424441b38f95e8615f Mon Sep 17 00:00:00 2001 From: 8ear Date: Mon, 2 Sep 2019 12:19:57 +0200 Subject: [PATCH 11/13] Fix entrypoint bug --- docker/files/entrypoint.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/files/entrypoint.sh b/docker/files/entrypoint.sh index fda2af4..73d8f39 100755 --- a/docker/files/entrypoint.sh +++ b/docker/files/entrypoint.sh @@ -11,7 +11,7 @@ DEBUG="" # Functions echo (){ - command echo -e "$STARTMSG $*" + command echo "$STARTMSG $*" } # Environment Variables From a9a4ec385180118a83aaeb7a0dacbc21a15d3cba Mon Sep 17 00:00:00 2001 From: 8ear Date: Mon, 2 Sep 2019 12:20:18 +0200 Subject: [PATCH 12/13] Disable not required package virtualenv for final stage --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index e7a4eec..8ac6d9f 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -56,7 +56,7 @@ RUN set -eu \ tesseract-ocr \ libpoppler-cpp-dev \ imagemagick \ - virtualenv \ + # virtualenv \ # libopencv-dev \ zbar-tools \ libzbar0 \ From 4f0237508e57d9273cfb39dae9ad39116821b3ab Mon Sep 17 00:00:00 2001 From: 8ear Date: Mon, 2 Sep 2019 14:10:49 +0200 Subject: [PATCH 13/13] Add .travis.yml command for docker build --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 18c02c6..db66efd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,6 +10,9 @@ python: - "3.6-dev" - "3.7-dev" +before_install: + - docker build -t misp-modules --build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") docker/ + install: - sudo apt-get install libzbar0 libzbar-dev libpoppler-cpp-dev - pip install pipenv