Browse Source

Merge branch 'master' of github.com:MISP/misp-modules

pull/334/head v2.4.116
chrisr3d 2 years ago
parent
commit
5ebd0bd4fc
  1. 3
      .travis.yml
  2. 130
      docker/Dockerfile
  3. 37
      docker/files/entrypoint.sh
  4. 4
      docker/files/healthcheck.sh
  5. 113
      docs/install.md
  6. 68
      misp_modules/modules/import_mod/cuckooimport.py
  7. 2
      tests/test_expansions.py

3
.travis.yml

@ -10,6 +10,9 @@ python:
- "3.6-dev"
- "3.7-dev"
before_install:
- docker build -t misp-modules --build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") docker/
install:
- sudo apt-get install libzbar0 libzbar-dev libpoppler-cpp-dev
- pip install pipenv

130
docker/Dockerfile

@ -0,0 +1,130 @@
FROM python:3.7-buster AS build
ENV DEBIAN_FRONTEND noninteractive
ENV WORKDIR="/usr/local/src/misp_modules"
ENV VENV_DIR="/misp_modules"
# Install Packages for build
RUN set -eu \
;mkdir -p ${WORKDIR} ${VENV_DIR} \
;apt-get update \
;apt-get install -y \
git \
libpq5 \
libjpeg-dev \
tesseract-ocr \
libpoppler-cpp-dev \
imagemagick \
virtualenv \
libopencv-dev \
zbar-tools \
libzbar0 \
libzbar-dev \
libfuzzy-dev \
;apt-get -y autoremove \
;apt-get -y clean \
;rm -rf /var/lib/apt/lists/* \
;
# Create MISP Modules
RUN set -eu \
;git clone https://github.com/MISP/misp-modules.git ${WORKDIR} \
;virtualenv -p python3 ${VENV_DIR}/venv \
;cd ${WORKDIR} \
;${VENV_DIR}/venv/bin/pip3 install -I -r REQUIREMENTS --no-cache-dir \
;${VENV_DIR}/venv/bin/pip3 install . --no-cache-dir \
;
#########################################
# Start Final Docker Image
#
FROM python:3.7-slim-buster AS final
ENV DEBIAN_FRONTEND noninteractive
ENV VENV_DIR="/misp_modules"
# Copy all builded files from build stage
COPY --from=build ${VENV_DIR} ${VENV_DIR}
# Install Packages to run it
RUN set -eu \
;apt-get update \
;apt-get install -y \
curl \
libpq5 \
# libjpeg-dev \
tesseract-ocr \
libpoppler-cpp-dev \
imagemagick \
# virtualenv \
# libopencv-dev \
zbar-tools \
libzbar0 \
# libzbar-dev \
# libfuzzy-dev \
;apt-get -y autoremove \
;apt-get -y clean \
;rm -rf /var/lib/apt/lists/* \
;chown -R nobody ${VENV_DIR} \
;
# Entrypoint
COPY files/entrypoint.sh /entrypoint.sh
ENTRYPOINT [ "/entrypoint.sh" ]
# Add Healthcheck Config
COPY files/healthcheck.sh /healthcheck.sh
HEALTHCHECK --interval=1m --timeout=45s --retries=3 CMD ["/healthcheck.sh"]
# Change Workdir
WORKDIR ${VENV_DIR}
# Change from root to www-data
USER nobody
# Expose Port
EXPOSE 6666
# Shortterm ARG Variables:
ARG VENDOR="MISP"
ARG COMPONENT="misp-modules"
ARG BUILD_DATE
ARG GIT_REPO="https://github.com/MISP/misp-modules"
ARG VCS_REF
ARG RELEASE_DATE
ARG NAME="MISP-dockerized-misp-modules"
ARG DESCRIPTION="This docker container contains MISP modules in an Debian Container."
ARG DOCUMENTATION="https://misp.github.io/misp-modules/"
ARG AUTHOR="MISP"
ARG LICENSE="BSD-3-Clause"
# Longterm Environment Variables
ENV \
BUILD_DATE=${BUILD_DATE} \
NAME=${NAME} \
PATH=$PATH:${VENV_DIR}/venv/bin
# Labels
LABEL org.label-schema.build-date="${BUILD_DATE}" \
org.label-schema.name="${NAME}" \
org.label-schema.description="${DESCRIPTION}" \
org.label-schema.vcs-ref="${VCS_REF}" \
org.label-schema.vcs-url="${GIT_REPO}" \
org.label-schema.url="${GIT_REPO}" \
org.label-schema.vendor="${VENDOR}" \
org.label-schema.version="${VERSION}" \
org.label-schema.usage="${DOCUMENTATION}" \
org.label-schema.schema-version="1.0.0-rc1"
LABEL org.opencontainers.image.created="${BUILD_DATE}" \
org.opencontainers.image.url="${GIT_REPO}" \
org.opencontainers.image.source="${GIT_REPO}" \
org.opencontainers.image.version="${VERSION}" \
org.opencontainers.image.revision="${VCS_REF}" \
org.opencontainers.image.vendor="${VENDOR}" \
org.opencontainers.image.title="${NAME}" \
org.opencontainers.image.description="${DESCRIPTION}" \
org.opencontainers.image.documentation="${DOCUMENTATION}" \
org.opencontainers.image.authors="${AUTHOR}" \
org.opencontainers.image.licenses="${LICENSE}"

37
docker/files/entrypoint.sh

@ -0,0 +1,37 @@
#!/bin/sh
set -eu
# Variables
NC='\033[0m' # No Color
Light_Green='\033[1;32m'
STARTMSG="${Light_Green}[ENTRYPOINT_MISP_MODULES]${NC}"
VENV_DIR=${VENV_DIR:-"/misp-modules"}
MISP_MODULES_BINARY="${VENV_DIR}/venv/bin/misp-modules"
DEBUG=""
# Functions
echo (){
command echo "$STARTMSG $*"
}
# Environment Variables
MISP_MODULES_DEBUG=${MISP_MODULES_DEBUG:-"false"}
#
# MAIN
#
# Check if debugging mode should be enabled
[ "$MISP_MODULES_DEBUG" = "true" ] && DEBUG="-d"
# check if a command parameter exists and start misp-modules
if [ $# = 0 ]
then
# If no cmd parameter is set
echo "Start MISP Modules" && $MISP_MODULES_BINARY $DEBUG -l 0.0.0.0 > /dev/stdout 2> /dev/stderr
else
# If cmd parameter is set
echo "Start MISP Modules" && $MISP_MODULES_BINARY $DEBUG -l 0.0.0.0 > /dev/stdout 2> /dev/stderr &
exec "$@"
fi

4
docker/files/healthcheck.sh

@ -0,0 +1,4 @@
#!/bin/sh
# If no contain is there or curl get an error back: exit 1. Docker restart then the container.
curl -fk http://0.0.0.0:6666/modules || exit 1

113
docs/install.md

@ -1,41 +1,114 @@
## How to install and start MISP modules in a Python virtualenv?
## How to install and start MISP modules (in a Python virtualenv)?
~~~~bash
sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick
sudo -u www-data virtualenv -p python3 /var/www/MISP/venv
SUDO_WWW="sudo -u www-data"
sudo apt-get install -y \
git \
libpq5 \
libjpeg-dev \
tesseract-ocr \
libpoppler-cpp-dev \
imagemagick virtualenv \
libopencv-dev \
zbar-tools \
libzbar0 \
libzbar-dev \
libfuzzy-dev
# BEGIN with virtualenv:
$SUDO_WWW virtualenv -p python3 /var/www/MISP/venv
# END with virtualenv
cd /usr/local/src/
sudo git clone https://github.com/MISP/misp-modules.git
cd misp-modules
sudo -u www-data /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS
sudo -u www-data /var/www/MISP/venv/bin/pip install .
sudo apt install ruby-pygments.rb -y
sudo gem install asciidoctor-pdf --pre
sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local
# BEGIN with virtualenv:
$SUDO_WWW /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS
$SUDO_WWW /var/www/MISP/venv/bin/pip install .
# END with virtualenv
# BEGIN without virtualenv:
sudo pip install -I -r REQUIREMENTS
sudo pip install .
# END without virtualenv
# Start misp-modules as a service
sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/
sudo systemctl daemon-reload
sudo systemctl enable --now misp-modules
/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules
~~~~
## How to install and start MISP modules?
## How to install and start MISP modules on RHEL-based distributions ?
As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the SCL repository.
~~~~bash
sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick
SUDO_WWW="sudo -u apache"
sudo yum install \
rh-ruby22 \
openjpeg-devel \
rubygem-rouge \
rubygem-asciidoctor \
zbar-devel \
opencv-devel \
gcc-c++ \
pkgconfig \
poppler-cpp-devel \
python-devel \
redhat-rpm-config
cd /usr/local/src/
sudo git clone https://github.com/MISP/misp-modules.git
cd misp-modules
sudo pip3 install -I -r REQUIREMENTS
sudo pip3 install -I .
sudo apt install ruby-pygments.rb -y
sudo gem install asciidoctor-pdf --pre
sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local
/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules
$SUDO_WWW /usr/bin/scl enable rh-python36 "virtualenv -p python3 /var/www/MISP/venv"
$SUDO_WWW /var/www/MISP/venv/bin/pip install -U -I -r REQUIREMENTS
$SUDO_WWW /var/www/MISP/venv/bin/pip install -U .
~~~~
Create the service file /etc/systemd/system/misp-modules.service :
~~~~bash
echo "[Unit]
Description=MISP's modules
After=misp-workers.service
[Service]
Type=simple
User=apache
Group=apache
ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 '/var/www/MISP/venv/bin/misp-modules –l 127.0.0.1 –s'
Restart=always
RestartSec=10
[Install]
WantedBy=multi-user.target" | sudo tee /etc/systemd/system/misp-modules.service
~~~~
The After=misp-workers.service must be changed or removed if you have not created a misp-workers service. Then, enable the misp-modules service and start it:
~~~~bash
systemctl daemon-reload
systemctl enable --now misp-modules
~~~~
## How to use an MISP modules Docker container
### Docker build
~~~~bash
docker build -t misp-modules \
--build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") \
docker/
~~~~
### Docker run
~~~~bash
# Start Redis
docker run --rm -d --name=misp-redis redis:alpine
# Start MISP-modules
docker run \
--rm -d --name=misp-modules \
-e REDIS_BACKEND=misp-redis \
@ -43,7 +116,7 @@ docker run \
-e REDIS_PW="" \
-e REDIS_DATABASE="245" \
-e MISP_MODULES_DEBUG="false" \
dcso/misp-dockerized-redis
dcso/misp-dockerized-misp-modules
~~~~
### Docker-compose
@ -53,6 +126,12 @@ services:
misp-modules:
# https://hub.docker.com/r/dcso/misp-dockerized-misp-modules
image: dcso/misp-dockerized-misp-modules:3
# Local image:
#image: misp-modules
#build:
# context: docker/
environment:
# Redis
REDIS_BACKEND: misp-redis

68
misp_modules/modules/import_mod/cuckooimport.py

@ -1,9 +1,11 @@
import json
import base64
import tarfile
import io
import logging
import posixpath
from io import BytesIO, BufferedReader
import stat
import tarfile
import zipfile
from pymisp import MISPEvent, MISPObject, MISPAttribute
from pymisp.tools import make_binary_objects
from collections import OrderedDict
@ -12,10 +14,14 @@ log = logging.getLogger(__name__)
misperrors = {'error': 'Error'}
moduleinfo = {'version': '1.0',
'author': 'Pierre-Jean Grenier',
'description': 'Cuckoo archive import',
'module-type': ['import']}
moduleinfo = {
'version': '1.1',
'author': 'Pierre-Jean Grenier',
'description': "Import a Cuckoo archive (zipfile or bzip2 tarball), "
"either downloaded manually or exported from the "
"API (/tasks/report/{task_id}/all).",
'module-type': ['import'],
}
moduleconfig = []
@ -202,13 +208,21 @@ class CuckooParser():
self.files = None
self.malware_binary = None
self.report = None
self.config = {key: int(on) for key, on in config.items()}
self.config = {
# if an option is missing (we receive None as a value),
# fall back to the default specified in the options
key: int(
on if on is not None
else self.options[key]["userConfig"]["checked"] == 'true'
)
for key, on in config.items()
}
def get_file(self, relative_filepath):
"""Return a BufferedReader for the corresponding relative_filepath
in the Cuckoo archive. If not found, return an empty BufferedReader
"""Return an io.BufferedIOBase for the corresponding relative_filepath
in the Cuckoo archive. If not found, return an empty io.BufferedReader
to avoid fatal errors."""
blackhole = BufferedReader(open('/dev/null', 'rb'))
blackhole = io.BufferedReader(open('/dev/null', 'rb'))
res = self.files.get(relative_filepath, blackhole)
if res == blackhole:
log.debug(f"Did not find file {relative_filepath}, "
@ -220,12 +234,30 @@ class CuckooParser():
# archive_encoded is base 64 encoded content
# we extract the info about each file but do not retrieve
# it automatically, as it may take too much space in memory
buf_io = BytesIO(base64.b64decode(archive_encoded))
f = tarfile.open(fileobj=buf_io, mode='r:bz2')
self.files = {
info.name: f.extractfile(info)
for info in f.getmembers()
}
buf_io = io.BytesIO(base64.b64decode(archive_encoded))
if zipfile.is_zipfile(buf_io):
# the archive was probably downloaded from the WebUI
buf_io.seek(0) # don't forget this not to read an empty buffer
z = zipfile.ZipFile(buf_io, 'r')
self.files = {
info.filename: z.open(info)
for info in z.filelist
# only extract the regular files and dirs, we don't
# want any symbolic link
if stat.S_ISREG(info.external_attr >> 16)
or stat.S_ISDIR(info.external_attr >> 16)
}
else:
# the archive was probably downloaded from the API
buf_io.seek(0) # don't forget this not to read an empty buffer
f = tarfile.open(fileobj=buf_io, mode='r:bz2')
self.files = {
info.name: f.extractfile(info)
for info in f.getmembers()
# only extract the regular files and dirs, we don't
# want any symbolic link
if info.isreg() or info.isdir()
}
# We want to keep the order of the keys of sub-dicts in the report,
# eg. the signatures have marks with unknown keys such as
@ -280,7 +312,7 @@ class CuckooParser():
log.debug("Sample is a file, uploading it")
self.read_malware()
file_o, bin_type_o, bin_section_li = make_binary_objects(
pseudofile=BytesIO(self.malware_binary),
pseudofile=io.BytesIO(self.malware_binary),
filename=target["file"]["name"],
)
@ -548,7 +580,7 @@ class CuckooParser():
filename = posixpath.basename(path)
dropped_file = self.get_file(path)
dropped_binary = BytesIO(dropped_file.read())
dropped_binary = io.BytesIO(dropped_file.read())
# create ad hoc objects
file_o, bin_type_o, bin_section_li = make_binary_objects(
pseudofile=dropped_binary, filename=filename,

2
tests/test_expansions.py

@ -43,7 +43,7 @@ class TestExpansions(unittest.TestCase):
query = {"module": "hibp", "email-src": "info@circl.lu"}
response = self.misp_modules_post(query)
to_check = self.get_values(response)
if to_check == "haveibeenpwned.com API not accessible (HTTP 403)":
if to_check == "haveibeenpwned.com API not accessible (HTTP 401)":
self.skipTest(f"haveibeenpwned blocks travis IPs: {response}")
self.assertEqual(to_check, 'OK (Not Found)', response)

Loading…
Cancel
Save