Create smaller image, faster build times, rework dependencies (#27)

Changes:
* fetch pymisp version from submodule, remove erroneous module from additional dependencies
* fix heredoc indentation, move files dist and permission to same layer to avoid duplicating
* fix cybox addition, codecov removal
* pinned pip versions for our own imports
* size optimization by applying the intended file permissions from later step in initial copy
* bind-mount wheels to reduce image size
* fix var init, rework py module add script to require version and only overwrite when we have a version defined
* handle missing MISP/MODULES_TAG in env
* remove git package and (almost all of) .git directory
* split MISP and PyMISP steps to allow faster iteration in module step
pull/1/head
Øivind Hoel 2023-09-15 12:50:30 +02:00 committed by GitHub
parent 1f3b79c6aa
commit 5a3d5a9e0a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 157 additions and 103 deletions

View File

@ -2,6 +2,38 @@ variable "PLATFORMS" {
default = ["linux/amd64", "linux/arm64"] default = ["linux/amd64", "linux/arm64"]
} }
variable "PYPI_REDIS_VERSION" {
default = ""
}
variable "PYPI_LIEF_VERSION" {
default = ""
}
variable "PYPI_PYDEEP2_VERSION" {
default = ""
}
variable "PYPI_PYTHON_MAGIC_VERSION" {
default = ""
}
variable "PYPI_MISP_LIB_STIX2_VERSION" {
default = ""
}
variable "PYPI_MAEC_VERSION" {
default = ""
}
variable "PYPI_MIXBOX_VERSION" {
default = ""
}
variable "PYPI_CYBOX_VERSION" {
default = ""
}
variable "DOCKER_USERNAME" { variable "DOCKER_USERNAME" {
default = null default = null
} }
@ -44,10 +76,10 @@ group "default" {
target "misp-modules" { target "misp-modules" {
context = "modules/." context = "modules/."
dockerfile = "Dockerfile" dockerfile = "Dockerfile"
tags = ["${DOCKER_USERNAME}/misp-docker:modules-latest", "${DOCKER_USERNAME}/misp-docker:modules-${DOCKER_IMG_TAG}", "${DOCKER_USERNAME}/misp-docker:modules-${MODULES_TAG}"] tags = flatten(["${DOCKER_USERNAME}/misp-docker:modules-latest", "${DOCKER_USERNAME}/misp-docker:modules-${DOCKER_IMG_TAG}", MODULES_TAG != "" ? ["${DOCKER_USERNAME}/misp-docker:modules-${MODULES_TAG}"] : []])
args = { args = {
"MODULES_TAG": "${MODULES_TAG}", "MODULES_TAG": "${MODULES_TAG}",
"MODULES_COMMIT": "${MODULES_COMMIT}" "MODULES_COMMIT": "${MODULES_COMMIT}",
} }
platforms = "${PLATFORMS}" platforms = "${PLATFORMS}"
} }
@ -55,11 +87,19 @@ target "misp-modules" {
target "misp" { target "misp" {
context = "server/." context = "server/."
dockerfile = "Dockerfile" dockerfile = "Dockerfile"
tags = ["${DOCKER_USERNAME}/misp-docker:core-latest", "${DOCKER_USERNAME}/misp-docker:core-${DOCKER_IMG_TAG}", "${DOCKER_USERNAME}/misp-docker:core-${MISP_TAG}"] tags = flatten(["${DOCKER_USERNAME}/misp-docker:core-latest", "${DOCKER_USERNAME}/misp-docker:core-${DOCKER_IMG_TAG}", MISP_TAG != "" ? ["${DOCKER_USERNAME}/misp-docker:core-${MISP_TAG}"] : []])
args = { args = {
"MISP_TAG": "${MISP_TAG}", "MISP_TAG": "${MISP_TAG}",
"MISP_COMMIT": "${MISP_COMMIT}", "MISP_COMMIT": "${MISP_COMMIT}",
"PHP_VER": "${PHP_VER}", "PHP_VER": "${PHP_VER}",
"PYPI_REDIS_VERSION": "${PYPI_REDIS_VERSION}",
"PYPI_LIEF_VERSION": "${PYPI_LIEF_VERSION}",
"PYPI_PYDEEP2_VERSION": "${PYPI_PYDEEP2_VERSION}",
"PYPI_PYTHON_MAGIC_VERSION": "${PYPI_PYTHON_MAGIC_VERSION}",
"PYPI_MISP_LIB_STIX2_VERSION": "${PYPI_MISP_LIB_STIX2_VERSION}",
"PYPI_MAEC_VERSION": "${PYPI_MAEC_VERSION}",
"PYPI_MIXBOX_VERSION": "${PYPI_MIXBOX_VERSION}",
"PYPI_CYBOX_VERSION": "${PYPI_CYBOX_VERSION}",
} }
platforms = "${PLATFORMS}" platforms = "${PLATFORMS}"
} }

View File

@ -1,6 +1,7 @@
ARG DOCKER_HUB_PROXY="" ARG DOCKER_HUB_PROXY=""
FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build
ENV DEBIAN_FRONTEND noninteractive ENV DEBIAN_FRONTEND noninteractive
ENV COMPOSER_ALLOW_SUPERUSER 1 ENV COMPOSER_ALLOW_SUPERUSER 1
ARG MISP_TAG ARG MISP_TAG
@ -19,6 +20,7 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build
php-gd \ php-gd \
php-fpm \ php-fpm \
php-zip \ php-zip \
unzip \
&& apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
WORKDIR /tmp WORKDIR /tmp
@ -26,19 +28,20 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build
COPY --from=composer:latest /usr/bin/composer /usr/bin/composer COPY --from=composer:latest /usr/bin/composer /usr/bin/composer
RUN composer config --no-interaction allow-plugins.composer/installers true RUN composer config --no-interaction allow-plugins.composer/installers true
RUN composer install RUN composer install
RUN composer require --with-all-dependencies \ RUN composer require --with-all-dependencies --no-interaction \
supervisorphp/supervisor:^4.0 \ supervisorphp/supervisor:^4.0 \
guzzlehttp/guzzle \ guzzlehttp/guzzle \
lstrojny/fxmlrpc \ lstrojny/fxmlrpc \
php-http/message \ php-http/message \
php-http/message-factory php-http/message-factory \
RUN composer require --with-all-dependencies \ # docker image specific dependencies
elasticsearch/elasticsearch:^8.7.0 \ elasticsearch/elasticsearch:^8.7.0 \
jakub-onderka/openid-connect-php:^1.0.0 \ jakub-onderka/openid-connect-php:^1.0.0 \
aws/aws-sdk-php aws/aws-sdk-php
FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as php-build FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as php-build
ENV DEBIAN_FRONTEND noninteractive ENV DEBIAN_FRONTEND noninteractive
ENV TZ Etc/UTC
RUN apt-get update; apt-get install -y --no-install-recommends \ RUN apt-get update; apt-get install -y --no-install-recommends \
gcc \ gcc \
g++ \ g++ \
@ -47,82 +50,91 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as php-build
ca-certificates \ ca-certificates \
php \ php \
php-dev \ php-dev \
php-xml \
php-pear \ php-pear \
librdkafka-dev \ librdkafka-dev \
libsimdjson-dev \ libsimdjson-dev \
git \ git \
&& apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib; pecl channel-update pecl.php.net && pecl install ssdeep && pecl install rdkafka && pecl install simdjson
RUN pecl channel-update pecl.php.net
RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib; pecl install ssdeep && pecl install rdkafka && pecl install simdjson
RUN git clone --recursive --depth=1 https://github.com/kjdev/php-ext-brotli.git && \ RUN git clone --recursive --depth=1 https://github.com/kjdev/php-ext-brotli.git && \
cd php-ext-brotli && phpize && ./configure && make && make install cd php-ext-brotli && phpize && ./configure && make && make install
FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build
ENV DEBIAN_FRONTEND noninteractive ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update; apt-get install -y --no-install-recommends \
gcc \
git \
python3 \
python3-dev \
python3-pip \
python3-setuptools \
python3-venv \
python3-wheel \
libfuzzy-dev \
libffi-dev \
ca-certificates \
curl \
&& apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
RUN mkdir /wheels RUN apt-get update; apt-get install -y --no-install-recommends python3-pip git && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
ARG MISP_TAG
ARG MISP_COMMIT
# Download MISP using git in the /var/www/ directory. Remove unnecessary items.
RUN <<-EOF
if [ ! -z "${MISP_COMMIT}" ]; then
git clone https://github.com/MISP/MISP.git /var/www/MISP && cd /var/www/MISP && git checkout "${MISP_COMMIT}"
else
git clone --branch "${MISP_TAG}" --depth 1 https://github.com/MISP/MISP.git /var/www/MISP
fi
WORKDIR /tmp cd /var/www/MISP || exit; git submodule update --init --recursive .
RUN git clone --depth 1 https://github.com/CybOXProject/mixbox.git; \ # Until MISP project specifies its required PyMISP version, we grab this from the submodule
cd mixbox || exit; python3 setup.py bdist_wheel -d /wheels; \ pymispversion=$(head -n1 PyMISP/pymisp/__init__.py | awk '{print $3}' | tr -d \')
sed -i 's/-e //g' requirements.txt; pip3 wheel -r requirements.txt --no-cache-dir -w /wheels/
# install python-maec if [ -n "$pymispversion" ]; then
RUN git clone --depth 1 https://github.com/MAECProject/python-maec.git; \ sed -i "s/pymisp$/pymisp==$pymispversion/" requirements.txt
cd python-maec || exit; python3 setup.py bdist_wheel -d /wheels cat requirements.txt
fi
# End of pymisp version fixing hack
EOF
# install python-cybox RUN <<-EOF
RUN git clone --depth 1 https://github.com/CybOXProject/python-cybox.git; \ mkdir /wheels
cd python-cybox || exit; python3 setup.py bdist_wheel -d /wheels; \
sed -i 's/-e //g' requirements.txt; pip3 wheel -r requirements.txt --no-cache-dir -w /wheels/
# install python stix # Remove modules only used during MISP build
RUN git clone --depth 1 https://github.com/STIXProject/python-stix.git; \ set -- 'coveralls' 'codecov' 'requests-mock' 'nose' 'pip'
cd python-stix || exit; python3 setup.py bdist_wheel -d /wheels; \ for mod in "$@"; do
sed -i 's/-e //g' requirements.txt; pip3 wheel -r requirements.txt --no-cache-dir -w /wheels/ sed "/${mod}/d" -i /var/www/MISP/requirements.txt
done;
# install STIX2.0 library to support STIX 2.0 export: # Add additional dependencies (container specific)
# Original Requirements has a bunch of non-required pacakges, force it to only grab wheels for deps from setup.py # The "set" line contains the list of modules we want to ensure are present.
RUN git clone --depth 1 https://github.com/MISP/cti-python-stix2.git; \ # PYPI_MODULE_NAME_VERSION env vars can be set to specify the version desired,
cd cti-python-stix2 || exit; python3 setup.py bdist_wheel -d /wheels; \ # e.g. PYPI_SURICATA_VERSION="==2.0" to specify exactly version 2.0 for the suricata package
echo "-e ." > requirements.txt; pip3 wheel -r requirements.txt --no-cache-dir -w /wheels/ #
# 1. Check for presence of each module in requirements.txt
# 2. If missing, add it (with optional version from env (defaults to empty string))
# 3. If present, replace with our specified version if it exists, otherwise leave
# the upstream version alone.
set -- "redis" "lief" "pydeep2" "python-magic" "misp-lib-stix2" "maec" "mixbox" "cybox"
for mod in "$@"; do
mod_version_var=$(echo "PYPI_${mod}_VERSION" | tr '[:lower:]' '[:upper:]' | tr '-' '_')
mod_version=$(eval "echo \"\$$mod_version_var\"")
grep -q ${mod} /var/www/MISP/requirements.txt
exists=$?
if [ "${exists}" -eq "1" ]; then
echo "Adding missing module ${mod} with version '${mod_version}'"
echo ${mod}${mod_version} >> /var/www/MISP/requirements.txt
else
if [ "$(echo ${mod_version} | wc -m)" -gt 1 ]; then
echo "Overwriting existing module ${mod}, version '${mod_version}'"
sed -i "/${mod}/s/.*/${mod}${mod_version}/" /var/www/MISP/requirements.txt
else
echo "Skipping overwriting ${mod} due to missing version variable"
fi
fi
done;
# Install the new build tool pip3 wheel --no-cache-dir -w /wheels/ -r /var/www/MISP/requirements.txt
RUN pip3 install build
# install PyMISP
RUN git clone --depth 1 https://github.com/MISP/PyMISP.git; \
cd PyMISP || exit; python3 -m build --wheel --outdir /wheels
# install pydeep2 (drop-in replacement for pydeep)
RUN git clone --depth 1 https://github.com/JakubOnderka/pydeep.git; \
cd pydeep || exit; python3 setup.py bdist_wheel -d /wheels
# Grab other modules we need
RUN pip3 wheel --no-cache-dir -w /wheels/ plyara pyzmq redis python-magic lief>=0.13.1
# Remove extra packages due to incompatible requirements.txt files
WORKDIR /wheels
RUN find . -name "Sphinx*" | tee /dev/stderr | grep -v "Sphinx-1.5.5" | xargs rm -f
# Remove files we do not care for
rm -r /var/www/MISP/PyMISP
find /var/www/MISP/INSTALL/* ! -name 'MYSQL.sql' -type f -exec rm {} +
find /var/www/MISP/INSTALL/* ! -name 'MYSQL.sql' -type l -exec rm {} +
# Remove most files in .git - we do not use git functionality in docker
find /var/www/MISP/.git/* ! -name HEAD -exec rm -rf {} +
EOF
FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim"
ENV DEBIAN_FRONTEND noninteractive ENV DEBIAN_FRONTEND noninteractive
ARG MISP_TAG ARG MISP_TAG
ARG MISP_COMMIT ARG MISP_COMMIT
@ -132,10 +144,10 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim"
sudo \ sudo \
nginx \ nginx \
supervisor \ supervisor \
git \
cron \ cron \
openssl \ openssl \
gpg-agent gpg \ gpg \
gpg-agent \
ssdeep \ ssdeep \
libfuzzy2 \ libfuzzy2 \
mariadb-client \ mariadb-client \
@ -166,61 +178,52 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim"
curl jq \ curl jq \
&& apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
# Download MISP using git in the /var/www/ directory. # Install python modules
RUN if [ ! -z ${MISP_COMMIT} ]; then \
git clone https://github.com/MISP/MISP.git /var/www/MISP && cd /var/www/MISP && git checkout ${MISP_COMMIT}; \
else git clone --branch ${MISP_TAG} --depth 1 https://github.com/MISP/MISP.git /var/www/MISP; fi
RUN cd /var/www/MISP; git submodule update --init --recursive .; cd /var/www/MISP/app; \
# Remove some old and broken links that pollute the log files
rm -rf /var/www/MISP/INSTALL/old
# Python Modules
COPY --from=python-build /wheels /wheels COPY --from=python-build /wheels /wheels
RUN pip3 install --no-cache-dir /wheels/*.whl && rm -rf /wheels RUN pip3 install --no-cache-dir /wheels/*.whl && rm -rf /wheels
# PHP: install prebuilt libraries, then install the app's PHP deps # PHP: install prebuilt libraries, then install the app's PHP deps
COPY --from=php-build /usr/lib/php/${PHP_VER}/ssdeep.so /usr/lib/php/${PHP_VER}/ssdeep.so COPY --from=php-build ["/usr/lib/php/${PHP_VER}/ssdeep.so", "/usr/lib/php/${PHP_VER}/rdkafka.so", "/usr/lib/php/${PHP_VER}/brotli.so", "/usr/lib/php/${PHP_VER}/simdjson.so", "/usr/lib/php/${PHP_VER}/"]
COPY --from=php-build /usr/lib/php/${PHP_VER}/rdkafka.so /usr/lib/php/${PHP_VER}/rdkafka.so
COPY --from=php-build /usr/lib/php/${PHP_VER}/brotli.so /usr/lib/php/${PHP_VER}/brotli.so
COPY --from=php-build /usr/lib/php/${PHP_VER}/simdjson.so /usr/lib/php/${PHP_VER}/simdjson.so
# Do an early chown to limit image size # Do an early chown to limit image size
COPY --from=composer-build --chown=www-data:www-data /tmp/Vendor /var/www/MISP/app/Vendor COPY --from=python-build --chown=www-data:www-data --chmod=0550 /var/www/MISP /var/www/MISP
COPY --from=composer-build --chown=www-data:www-data /tmp/Plugin /var/www/MISP/app/Plugin COPY --from=composer-build --chown=www-data:www-data --chmod=0550 /tmp/Vendor /var/www/MISP/app/Vendor
COPY --from=composer-build --chown=www-data:www-data --chmod=0550 /tmp/Plugin /var/www/MISP/app/Plugin
RUN for dir in /etc/php/*; do echo "extension=ssdeep.so" > "$dir/mods-available/ssdeep.ini"; done; phpenmod ssdeep # Gather these in one layer, only act on actual directories under /etc/php/
RUN for dir in /etc/php/*; do echo "extension=rdkafka.so" > "$dir/mods-available/rdkafka.ini"; done; phpenmod rdkafka RUN <<-EOF
RUN for dir in /etc/php/*; do echo "extension=brotli.so" > "$dir/mods-available/brotli.ini"; done; phpenmod brotli set -- "ssdeep" "rdkafka" "brotli" "simdjson"
RUN for dir in /etc/php/*; do echo "extension=simdjson.so" > "$dir/mods-available/simdjson.ini"; done; phpenmod simdjson for mod in "$@"; do
RUN phpenmod redis for dir in /etc/php/*/; do
echo "extension=${mod}.so" > "${dir}mods-available/${mod}.ini"
done;
phpenmod "${mod}"
done;
phpenmod redis
EOF
# nginx # nginx
RUN rm /etc/nginx/sites-enabled/*; mkdir /run/php /etc/nginx/certs RUN rm /etc/nginx/sites-enabled/*; mkdir /run/php /etc/nginx/certs
COPY files/etc/nginx/misp /etc/nginx/sites-available/misp
COPY files/etc/nginx/misp80 /etc/nginx/sites-available/misp80
# Make a copy of the file and configuration stores, so we can sync from it # Make a copy of the file and configuration stores, so we can sync from it
RUN cp -R /var/www/MISP/app/files /var/www/MISP/app/files.dist; \
cp -R /var/www/MISP/app/Config /var/www/MISP/app/Config.dist;
# The spirit of the upstrem dockerization is to make: # The spirit of the upstream dockerization is to make:
# 1) User and group aligned in terms of permissions # 1) User and group aligned in terms of permissions
# 2) Files executable and read only, because of some rogue scripts like 'cake' # 2) Files executable and read only, because of some rogue scripts like 'cake'
# 3) Directories writable, because sometimes MISP add new new files # 3) Directories writable, because sometimes MISP add new files
RUN find /var/www/MISP \( ! -user www-data -or ! -group www-data \) -exec chown www-data:www-data '{}' +; \
find /var/www/MISP -not -perm 550 -type f -exec chmod 0550 '{}' +; \
find /var/www/MISP -not -perm 770 -type d -exec chmod 0770 '{}' +;
# Entrypoints RUN <<-EOF
COPY files/etc/supervisor/supervisor.conf /etc/supervisor/conf.d/10-supervisor.conf cp -R /var/www/MISP/app/files /var/www/MISP/app/files.dist
COPY files/etc/supervisor/workers.conf /etc/supervisor/conf.d/50-workers.conf cp -R /var/www/MISP/app/Config /var/www/MISP/app/Config.dist
COPY files/var/www/html/index.php /var/www/html/index.php find /var/www/MISP \( ! -user www-data -or ! -group www-data \) -exec chown www-data:www-data '{}' +;
COPY files/configure_misp.sh / find /var/www/MISP -not -perm 550 -type f -exec chmod 0550 '{}' +;
COPY files/rest_client.sh / find /var/www/MISP -not -perm 770 -type d -exec chmod 0770 '{}' +;
COPY files/entrypoint_fpm.sh / # Diagnostics wants this file to be present and writable even if we do not use git in docker land
COPY files/entrypoint_nginx.sh / touch /var/www/MISP/.git/ORIG_HEAD && chmod 0600 /var/www/MISP/.git/ORIG_HEAD && chown www-data:www-data /var/www/MISP/.git/ORIG_HEAD
COPY files/entrypoint_cron.sh / EOF
COPY files/entrypoint.sh /
# Copy all our image specific files to appropriate locations
COPY files/ /
ENTRYPOINT [ "/entrypoint.sh" ] ENTRYPOINT [ "/entrypoint.sh" ]
# Change Workdirectory # Change Workdirectory

View File

@ -157,6 +157,8 @@ apply_critical_fixes() {
\"auth\": \"\" \"auth\": \"\"
} }
}" > /dev/null }" > /dev/null
# Avoids displaying errors not relevant to a docker container
sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting -q "MISP.self_update" false
} }
apply_optional_fixes() { apply_optional_fixes() {

View File

@ -3,6 +3,15 @@ MODULES_TAG=v2.4.176
PHP_VER=20190902 PHP_VER=20190902
LIBFAUP_COMMIT=3a26d0a LIBFAUP_COMMIT=3a26d0a
PYPI_REDIS_VERSION="==5.0.*"
PYPI_LIEF_VERSION=">=0.13.1"
PYPI_PYDEEP2_VERSION="==0.5.*"
PYPI_PYTHON_MAGIC_VERSION="==0.4.*"
PYPI_MISP_LIB_STIX2_VERSION="==3.0.*"
PYPI_MAEC_VERSION="==4.1.*"
PYPI_MIXBOX_VERSION="==1.0.*"
PYPI_CYBOX_VERSION="==2.1.*"
# MISP_COMMIT takes precedence over MISP_TAG # MISP_COMMIT takes precedence over MISP_TAG
# MISP_COMMIT=c56d537 # MISP_COMMIT=c56d537
# MODULES_COMMIT takes precedence over MODULES_TAG # MODULES_COMMIT takes precedence over MODULES_TAG