mirror of https://github.com/MISP/misp-modules
chrisr3d
commit
120a510dda
|
|
@ -10,6 +10,9 @@ python:
|
|||
- "3.6-dev"
|
||||
- "3.7-dev"
|
||||
|
||||
before_install:
|
||||
- docker build -t misp-modules --build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") docker/
|
||||
|
||||
install:
|
||||
- sudo apt-get install libzbar0 libzbar-dev libpoppler-cpp-dev
|
||||
- pip install pipenv
|
||||
|
|
|
|||
|
|
@ -0,0 +1,130 @@
|
|||
FROM python:3.7-buster AS build
|
||||
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
ENV WORKDIR="/usr/local/src/misp_modules"
|
||||
ENV VENV_DIR="/misp_modules"
|
||||
|
||||
# Install Packages for build
|
||||
RUN set -eu \
|
||||
;mkdir -p ${WORKDIR} ${VENV_DIR} \
|
||||
;apt-get update \
|
||||
;apt-get install -y \
|
||||
git \
|
||||
libpq5 \
|
||||
libjpeg-dev \
|
||||
tesseract-ocr \
|
||||
libpoppler-cpp-dev \
|
||||
imagemagick \
|
||||
virtualenv \
|
||||
libopencv-dev \
|
||||
zbar-tools \
|
||||
libzbar0 \
|
||||
libzbar-dev \
|
||||
libfuzzy-dev \
|
||||
;apt-get -y autoremove \
|
||||
;apt-get -y clean \
|
||||
;rm -rf /var/lib/apt/lists/* \
|
||||
;
|
||||
|
||||
# Create MISP Modules
|
||||
RUN set -eu \
|
||||
;git clone https://github.com/MISP/misp-modules.git ${WORKDIR} \
|
||||
;virtualenv -p python3 ${VENV_DIR}/venv \
|
||||
;cd ${WORKDIR} \
|
||||
;${VENV_DIR}/venv/bin/pip3 install -I -r REQUIREMENTS --no-cache-dir \
|
||||
;${VENV_DIR}/venv/bin/pip3 install . --no-cache-dir \
|
||||
;
|
||||
|
||||
#########################################
|
||||
# Start Final Docker Image
|
||||
#
|
||||
FROM python:3.7-slim-buster AS final
|
||||
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
ENV VENV_DIR="/misp_modules"
|
||||
|
||||
# Copy all builded files from build stage
|
||||
COPY --from=build ${VENV_DIR} ${VENV_DIR}
|
||||
|
||||
# Install Packages to run it
|
||||
RUN set -eu \
|
||||
;apt-get update \
|
||||
;apt-get install -y \
|
||||
curl \
|
||||
libpq5 \
|
||||
# libjpeg-dev \
|
||||
tesseract-ocr \
|
||||
libpoppler-cpp-dev \
|
||||
imagemagick \
|
||||
# virtualenv \
|
||||
# libopencv-dev \
|
||||
zbar-tools \
|
||||
libzbar0 \
|
||||
# libzbar-dev \
|
||||
# libfuzzy-dev \
|
||||
;apt-get -y autoremove \
|
||||
;apt-get -y clean \
|
||||
;rm -rf /var/lib/apt/lists/* \
|
||||
;chown -R nobody ${VENV_DIR} \
|
||||
;
|
||||
|
||||
# Entrypoint
|
||||
COPY files/entrypoint.sh /entrypoint.sh
|
||||
ENTRYPOINT [ "/entrypoint.sh" ]
|
||||
|
||||
# Add Healthcheck Config
|
||||
COPY files/healthcheck.sh /healthcheck.sh
|
||||
HEALTHCHECK --interval=1m --timeout=45s --retries=3 CMD ["/healthcheck.sh"]
|
||||
|
||||
# Change Workdir
|
||||
WORKDIR ${VENV_DIR}
|
||||
|
||||
# Change from root to www-data
|
||||
USER nobody
|
||||
|
||||
# Expose Port
|
||||
EXPOSE 6666
|
||||
|
||||
# Shortterm ARG Variables:
|
||||
ARG VENDOR="MISP"
|
||||
ARG COMPONENT="misp-modules"
|
||||
ARG BUILD_DATE
|
||||
ARG GIT_REPO="https://github.com/MISP/misp-modules"
|
||||
ARG VCS_REF
|
||||
ARG RELEASE_DATE
|
||||
ARG NAME="MISP-dockerized-misp-modules"
|
||||
ARG DESCRIPTION="This docker container contains MISP modules in an Debian Container."
|
||||
ARG DOCUMENTATION="https://misp.github.io/misp-modules/"
|
||||
ARG AUTHOR="MISP"
|
||||
ARG LICENSE="BSD-3-Clause"
|
||||
|
||||
# Longterm Environment Variables
|
||||
ENV \
|
||||
BUILD_DATE=${BUILD_DATE} \
|
||||
NAME=${NAME} \
|
||||
PATH=$PATH:${VENV_DIR}/venv/bin
|
||||
|
||||
# Labels
|
||||
LABEL org.label-schema.build-date="${BUILD_DATE}" \
|
||||
org.label-schema.name="${NAME}" \
|
||||
org.label-schema.description="${DESCRIPTION}" \
|
||||
org.label-schema.vcs-ref="${VCS_REF}" \
|
||||
org.label-schema.vcs-url="${GIT_REPO}" \
|
||||
org.label-schema.url="${GIT_REPO}" \
|
||||
org.label-schema.vendor="${VENDOR}" \
|
||||
org.label-schema.version="${VERSION}" \
|
||||
org.label-schema.usage="${DOCUMENTATION}" \
|
||||
org.label-schema.schema-version="1.0.0-rc1"
|
||||
|
||||
LABEL org.opencontainers.image.created="${BUILD_DATE}" \
|
||||
org.opencontainers.image.url="${GIT_REPO}" \
|
||||
org.opencontainers.image.source="${GIT_REPO}" \
|
||||
org.opencontainers.image.version="${VERSION}" \
|
||||
org.opencontainers.image.revision="${VCS_REF}" \
|
||||
org.opencontainers.image.vendor="${VENDOR}" \
|
||||
org.opencontainers.image.title="${NAME}" \
|
||||
org.opencontainers.image.description="${DESCRIPTION}" \
|
||||
org.opencontainers.image.documentation="${DOCUMENTATION}" \
|
||||
org.opencontainers.image.authors="${AUTHOR}" \
|
||||
org.opencontainers.image.licenses="${LICENSE}"
|
||||
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
# Variables
|
||||
NC='\033[0m' # No Color
|
||||
Light_Green='\033[1;32m'
|
||||
STARTMSG="${Light_Green}[ENTRYPOINT_MISP_MODULES]${NC}"
|
||||
VENV_DIR=${VENV_DIR:-"/misp-modules"}
|
||||
MISP_MODULES_BINARY="${VENV_DIR}/venv/bin/misp-modules"
|
||||
DEBUG=""
|
||||
|
||||
# Functions
|
||||
echo (){
|
||||
command echo "$STARTMSG $*"
|
||||
}
|
||||
|
||||
# Environment Variables
|
||||
MISP_MODULES_DEBUG=${MISP_MODULES_DEBUG:-"false"}
|
||||
|
||||
#
|
||||
# MAIN
|
||||
#
|
||||
|
||||
|
||||
# Check if debugging mode should be enabled
|
||||
[ "$MISP_MODULES_DEBUG" = "true" ] && DEBUG="-d"
|
||||
|
||||
# check if a command parameter exists and start misp-modules
|
||||
if [ $# = 0 ]
|
||||
then
|
||||
# If no cmd parameter is set
|
||||
echo "Start MISP Modules" && $MISP_MODULES_BINARY $DEBUG -l 0.0.0.0 > /dev/stdout 2> /dev/stderr
|
||||
else
|
||||
# If cmd parameter is set
|
||||
echo "Start MISP Modules" && $MISP_MODULES_BINARY $DEBUG -l 0.0.0.0 > /dev/stdout 2> /dev/stderr &
|
||||
exec "$@"
|
||||
fi
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
#!/bin/sh
|
||||
|
||||
# If no contain is there or curl get an error back: exit 1. Docker restart then the container.
|
||||
curl -fk http://0.0.0.0:6666/modules || exit 1
|
||||
113
docs/install.md
113
docs/install.md
|
|
@ -1,41 +1,114 @@
|
|||
## How to install and start MISP modules in a Python virtualenv?
|
||||
## How to install and start MISP modules (in a Python virtualenv)?
|
||||
|
||||
~~~~bash
|
||||
sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick
|
||||
sudo -u www-data virtualenv -p python3 /var/www/MISP/venv
|
||||
SUDO_WWW="sudo -u www-data"
|
||||
|
||||
sudo apt-get install -y \
|
||||
git \
|
||||
libpq5 \
|
||||
libjpeg-dev \
|
||||
tesseract-ocr \
|
||||
libpoppler-cpp-dev \
|
||||
imagemagick virtualenv \
|
||||
libopencv-dev \
|
||||
zbar-tools \
|
||||
libzbar0 \
|
||||
libzbar-dev \
|
||||
libfuzzy-dev
|
||||
|
||||
# BEGIN with virtualenv:
|
||||
$SUDO_WWW virtualenv -p python3 /var/www/MISP/venv
|
||||
# END with virtualenv
|
||||
|
||||
cd /usr/local/src/
|
||||
sudo git clone https://github.com/MISP/misp-modules.git
|
||||
cd misp-modules
|
||||
sudo -u www-data /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS
|
||||
sudo -u www-data /var/www/MISP/venv/bin/pip install .
|
||||
sudo apt install ruby-pygments.rb -y
|
||||
sudo gem install asciidoctor-pdf --pre
|
||||
sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local
|
||||
|
||||
# BEGIN with virtualenv:
|
||||
$SUDO_WWW /var/www/MISP/venv/bin/pip install -I -r REQUIREMENTS
|
||||
$SUDO_WWW /var/www/MISP/venv/bin/pip install .
|
||||
# END with virtualenv
|
||||
|
||||
# BEGIN without virtualenv:
|
||||
sudo pip install -I -r REQUIREMENTS
|
||||
sudo pip install .
|
||||
# END without virtualenv
|
||||
|
||||
# Start misp-modules as a service
|
||||
sudo cp etc/systemd/system/misp-modules.service /etc/systemd/system/
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl enable --now misp-modules
|
||||
/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules
|
||||
~~~~
|
||||
|
||||
## How to install and start MISP modules?
|
||||
## How to install and start MISP modules on RHEL-based distributions ?
|
||||
|
||||
As of this writing, the official RHEL repositories only contain Ruby 2.0.0 and Ruby 2.1 or higher is required. As such, this guide installs Ruby 2.2 from the SCL repository.
|
||||
|
||||
~~~~bash
|
||||
sudo apt-get install python3-dev python3-pip libpq5 libjpeg-dev tesseract-ocr imagemagick
|
||||
SUDO_WWW="sudo -u apache"
|
||||
sudo yum install \
|
||||
rh-ruby22 \
|
||||
openjpeg-devel \
|
||||
rubygem-rouge \
|
||||
rubygem-asciidoctor \
|
||||
zbar-devel \
|
||||
opencv-devel \
|
||||
gcc-c++ \
|
||||
pkgconfig \
|
||||
poppler-cpp-devel \
|
||||
python-devel \
|
||||
redhat-rpm-config
|
||||
cd /usr/local/src/
|
||||
sudo git clone https://github.com/MISP/misp-modules.git
|
||||
cd misp-modules
|
||||
sudo pip3 install -I -r REQUIREMENTS
|
||||
sudo pip3 install -I .
|
||||
sudo apt install ruby-pygments.rb -y
|
||||
sudo gem install asciidoctor-pdf --pre
|
||||
sudo sed -i -e '$i \sudo -u www-data /var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s > /tmp/misp-modules_rc.local.log &\n' /etc/rc.local
|
||||
/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s & #to start the modules
|
||||
$SUDO_WWW /usr/bin/scl enable rh-python36 "virtualenv -p python3 /var/www/MISP/venv"
|
||||
$SUDO_WWW /var/www/MISP/venv/bin/pip install -U -I -r REQUIREMENTS
|
||||
$SUDO_WWW /var/www/MISP/venv/bin/pip install -U .
|
||||
~~~~
|
||||
|
||||
Create the service file /etc/systemd/system/misp-modules.service :
|
||||
|
||||
~~~~bash
|
||||
echo "[Unit]
|
||||
Description=MISP's modules
|
||||
After=misp-workers.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
User=apache
|
||||
Group=apache
|
||||
ExecStart=/usr/bin/scl enable rh-python36 rh-ruby22 '/var/www/MISP/venv/bin/misp-modules –l 127.0.0.1 –s'
|
||||
Restart=always
|
||||
RestartSec=10
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target" | sudo tee /etc/systemd/system/misp-modules.service
|
||||
~~~~
|
||||
|
||||
The After=misp-workers.service must be changed or removed if you have not created a misp-workers service. Then, enable the misp-modules service and start it:
|
||||
|
||||
~~~~bash
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now misp-modules
|
||||
~~~~
|
||||
|
||||
## How to use an MISP modules Docker container
|
||||
|
||||
### Docker build
|
||||
|
||||
~~~~bash
|
||||
docker build -t misp-modules \
|
||||
--build-arg BUILD_DATE=$(date -u +"%Y-%m-%d") \
|
||||
docker/
|
||||
~~~~
|
||||
|
||||
### Docker run
|
||||
|
||||
~~~~bash
|
||||
# Start Redis
|
||||
docker run --rm -d --name=misp-redis redis:alpine
|
||||
# Start MISP-modules
|
||||
docker run \
|
||||
--rm -d --name=misp-modules \
|
||||
-e REDIS_BACKEND=misp-redis \
|
||||
|
|
@ -43,7 +116,7 @@ docker run \
|
|||
-e REDIS_PW="" \
|
||||
-e REDIS_DATABASE="245" \
|
||||
-e MISP_MODULES_DEBUG="false" \
|
||||
dcso/misp-dockerized-redis
|
||||
dcso/misp-dockerized-misp-modules
|
||||
~~~~
|
||||
|
||||
### Docker-compose
|
||||
|
|
@ -53,6 +126,12 @@ services:
|
|||
misp-modules:
|
||||
# https://hub.docker.com/r/dcso/misp-dockerized-misp-modules
|
||||
image: dcso/misp-dockerized-misp-modules:3
|
||||
|
||||
# Local image:
|
||||
#image: misp-modules
|
||||
#build:
|
||||
# context: docker/
|
||||
|
||||
environment:
|
||||
# Redis
|
||||
REDIS_BACKEND: misp-redis
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
import json
|
||||
import base64
|
||||
import tarfile
|
||||
import io
|
||||
import logging
|
||||
import posixpath
|
||||
from io import BytesIO, BufferedReader
|
||||
import stat
|
||||
import tarfile
|
||||
import zipfile
|
||||
from pymisp import MISPEvent, MISPObject, MISPAttribute
|
||||
from pymisp.tools import make_binary_objects
|
||||
from collections import OrderedDict
|
||||
|
|
@ -12,10 +14,14 @@ log = logging.getLogger(__name__)
|
|||
|
||||
misperrors = {'error': 'Error'}
|
||||
|
||||
moduleinfo = {'version': '1.0',
|
||||
moduleinfo = {
|
||||
'version': '1.1',
|
||||
'author': 'Pierre-Jean Grenier',
|
||||
'description': 'Cuckoo archive import',
|
||||
'module-type': ['import']}
|
||||
'description': "Import a Cuckoo archive (zipfile or bzip2 tarball), "
|
||||
"either downloaded manually or exported from the "
|
||||
"API (/tasks/report/{task_id}/all).",
|
||||
'module-type': ['import'],
|
||||
}
|
||||
|
||||
moduleconfig = []
|
||||
|
||||
|
|
@ -202,13 +208,21 @@ class CuckooParser():
|
|||
self.files = None
|
||||
self.malware_binary = None
|
||||
self.report = None
|
||||
self.config = {key: int(on) for key, on in config.items()}
|
||||
self.config = {
|
||||
# if an option is missing (we receive None as a value),
|
||||
# fall back to the default specified in the options
|
||||
key: int(
|
||||
on if on is not None
|
||||
else self.options[key]["userConfig"]["checked"] == 'true'
|
||||
)
|
||||
for key, on in config.items()
|
||||
}
|
||||
|
||||
def get_file(self, relative_filepath):
|
||||
"""Return a BufferedReader for the corresponding relative_filepath
|
||||
in the Cuckoo archive. If not found, return an empty BufferedReader
|
||||
"""Return an io.BufferedIOBase for the corresponding relative_filepath
|
||||
in the Cuckoo archive. If not found, return an empty io.BufferedReader
|
||||
to avoid fatal errors."""
|
||||
blackhole = BufferedReader(open('/dev/null', 'rb'))
|
||||
blackhole = io.BufferedReader(open('/dev/null', 'rb'))
|
||||
res = self.files.get(relative_filepath, blackhole)
|
||||
if res == blackhole:
|
||||
log.debug(f"Did not find file {relative_filepath}, "
|
||||
|
|
@ -220,11 +234,29 @@ class CuckooParser():
|
|||
# archive_encoded is base 64 encoded content
|
||||
# we extract the info about each file but do not retrieve
|
||||
# it automatically, as it may take too much space in memory
|
||||
buf_io = BytesIO(base64.b64decode(archive_encoded))
|
||||
buf_io = io.BytesIO(base64.b64decode(archive_encoded))
|
||||
if zipfile.is_zipfile(buf_io):
|
||||
# the archive was probably downloaded from the WebUI
|
||||
buf_io.seek(0) # don't forget this not to read an empty buffer
|
||||
z = zipfile.ZipFile(buf_io, 'r')
|
||||
self.files = {
|
||||
info.filename: z.open(info)
|
||||
for info in z.filelist
|
||||
# only extract the regular files and dirs, we don't
|
||||
# want any symbolic link
|
||||
if stat.S_ISREG(info.external_attr >> 16)
|
||||
or stat.S_ISDIR(info.external_attr >> 16)
|
||||
}
|
||||
else:
|
||||
# the archive was probably downloaded from the API
|
||||
buf_io.seek(0) # don't forget this not to read an empty buffer
|
||||
f = tarfile.open(fileobj=buf_io, mode='r:bz2')
|
||||
self.files = {
|
||||
info.name: f.extractfile(info)
|
||||
for info in f.getmembers()
|
||||
# only extract the regular files and dirs, we don't
|
||||
# want any symbolic link
|
||||
if info.isreg() or info.isdir()
|
||||
}
|
||||
|
||||
# We want to keep the order of the keys of sub-dicts in the report,
|
||||
|
|
@ -280,7 +312,7 @@ class CuckooParser():
|
|||
log.debug("Sample is a file, uploading it")
|
||||
self.read_malware()
|
||||
file_o, bin_type_o, bin_section_li = make_binary_objects(
|
||||
pseudofile=BytesIO(self.malware_binary),
|
||||
pseudofile=io.BytesIO(self.malware_binary),
|
||||
filename=target["file"]["name"],
|
||||
)
|
||||
|
||||
|
|
@ -548,7 +580,7 @@ class CuckooParser():
|
|||
filename = posixpath.basename(path)
|
||||
|
||||
dropped_file = self.get_file(path)
|
||||
dropped_binary = BytesIO(dropped_file.read())
|
||||
dropped_binary = io.BytesIO(dropped_file.read())
|
||||
# create ad hoc objects
|
||||
file_o, bin_type_o, bin_section_li = make_binary_objects(
|
||||
pseudofile=dropped_binary, filename=filename,
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ class TestExpansions(unittest.TestCase):
|
|||
query = {"module": "hibp", "email-src": "info@circl.lu"}
|
||||
response = self.misp_modules_post(query)
|
||||
to_check = self.get_values(response)
|
||||
if to_check == "haveibeenpwned.com API not accessible (HTTP 403)":
|
||||
if to_check == "haveibeenpwned.com API not accessible (HTTP 401)":
|
||||
self.skipTest(f"haveibeenpwned blocks travis IPs: {response}")
|
||||
self.assertEqual(to_check, 'OK (Not Found)', response)
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue