From 5a3d5a9e0a6f1a92aa9247f9f90878b42c507a6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=98ivind=20Hoel?= Date: Fri, 15 Sep 2023 12:50:30 +0200 Subject: [PATCH] Create smaller image, faster build times, rework dependencies (#27) Changes: * fetch pymisp version from submodule, remove erroneous module from additional dependencies * fix heredoc indentation, move files dist and permission to same layer to avoid duplicating * fix cybox addition, codecov removal * pinned pip versions for our own imports * size optimization by applying the intended file permissions from later step in initial copy * bind-mount wheels to reduce image size * fix var init, rework py module add script to require version and only overwrite when we have a version defined * handle missing MISP/MODULES_TAG in env * remove git package and (almost all of) .git directory * split MISP and PyMISP steps to allow faster iteration in module step --- docker-bake.hcl | 46 +++- server/Dockerfile | 203 +++++++++--------- server/files/configure_misp.sh | 2 + .../etc/nginx/{ => sites-available}/misp | 0 .../etc/nginx/{ => sites-available}/misp80 | 0 .../10-supervisor.conf} | 0 .../{workers.conf => conf.d/50-workers.conf} | 0 template.env | 9 + 8 files changed, 157 insertions(+), 103 deletions(-) rename server/files/etc/nginx/{ => sites-available}/misp (100%) rename server/files/etc/nginx/{ => sites-available}/misp80 (100%) rename server/files/etc/supervisor/{supervisor.conf => conf.d/10-supervisor.conf} (100%) rename server/files/etc/supervisor/{workers.conf => conf.d/50-workers.conf} (100%) diff --git a/docker-bake.hcl b/docker-bake.hcl index ece59df..019868c 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -2,6 +2,38 @@ variable "PLATFORMS" { default = ["linux/amd64", "linux/arm64"] } +variable "PYPI_REDIS_VERSION" { + default = "" +} + +variable "PYPI_LIEF_VERSION" { + default = "" +} + +variable "PYPI_PYDEEP2_VERSION" { + default = "" +} + +variable "PYPI_PYTHON_MAGIC_VERSION" { + default = "" +} + +variable "PYPI_MISP_LIB_STIX2_VERSION" { + default = "" +} + +variable "PYPI_MAEC_VERSION" { + default = "" +} + +variable "PYPI_MIXBOX_VERSION" { + default = "" +} + +variable "PYPI_CYBOX_VERSION" { + default = "" +} + variable "DOCKER_USERNAME" { default = null } @@ -44,10 +76,10 @@ group "default" { target "misp-modules" { context = "modules/." dockerfile = "Dockerfile" - tags = ["${DOCKER_USERNAME}/misp-docker:modules-latest", "${DOCKER_USERNAME}/misp-docker:modules-${DOCKER_IMG_TAG}", "${DOCKER_USERNAME}/misp-docker:modules-${MODULES_TAG}"] + tags = flatten(["${DOCKER_USERNAME}/misp-docker:modules-latest", "${DOCKER_USERNAME}/misp-docker:modules-${DOCKER_IMG_TAG}", MODULES_TAG != "" ? ["${DOCKER_USERNAME}/misp-docker:modules-${MODULES_TAG}"] : []]) args = { "MODULES_TAG": "${MODULES_TAG}", - "MODULES_COMMIT": "${MODULES_COMMIT}" + "MODULES_COMMIT": "${MODULES_COMMIT}", } platforms = "${PLATFORMS}" } @@ -55,11 +87,19 @@ target "misp-modules" { target "misp" { context = "server/." dockerfile = "Dockerfile" - tags = ["${DOCKER_USERNAME}/misp-docker:core-latest", "${DOCKER_USERNAME}/misp-docker:core-${DOCKER_IMG_TAG}", "${DOCKER_USERNAME}/misp-docker:core-${MISP_TAG}"] + tags = flatten(["${DOCKER_USERNAME}/misp-docker:core-latest", "${DOCKER_USERNAME}/misp-docker:core-${DOCKER_IMG_TAG}", MISP_TAG != "" ? ["${DOCKER_USERNAME}/misp-docker:core-${MISP_TAG}"] : []]) args = { "MISP_TAG": "${MISP_TAG}", "MISP_COMMIT": "${MISP_COMMIT}", "PHP_VER": "${PHP_VER}", + "PYPI_REDIS_VERSION": "${PYPI_REDIS_VERSION}", + "PYPI_LIEF_VERSION": "${PYPI_LIEF_VERSION}", + "PYPI_PYDEEP2_VERSION": "${PYPI_PYDEEP2_VERSION}", + "PYPI_PYTHON_MAGIC_VERSION": "${PYPI_PYTHON_MAGIC_VERSION}", + "PYPI_MISP_LIB_STIX2_VERSION": "${PYPI_MISP_LIB_STIX2_VERSION}", + "PYPI_MAEC_VERSION": "${PYPI_MAEC_VERSION}", + "PYPI_MIXBOX_VERSION": "${PYPI_MIXBOX_VERSION}", + "PYPI_CYBOX_VERSION": "${PYPI_CYBOX_VERSION}", } platforms = "${PLATFORMS}" } diff --git a/server/Dockerfile b/server/Dockerfile index 3c14488..8098d59 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -1,6 +1,7 @@ ARG DOCKER_HUB_PROXY="" FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build + ENV DEBIAN_FRONTEND noninteractive ENV COMPOSER_ALLOW_SUPERUSER 1 ARG MISP_TAG @@ -19,6 +20,7 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build php-gd \ php-fpm \ php-zip \ + unzip \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* WORKDIR /tmp @@ -26,19 +28,20 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build COPY --from=composer:latest /usr/bin/composer /usr/bin/composer RUN composer config --no-interaction allow-plugins.composer/installers true RUN composer install - RUN composer require --with-all-dependencies \ + RUN composer require --with-all-dependencies --no-interaction \ supervisorphp/supervisor:^4.0 \ guzzlehttp/guzzle \ lstrojny/fxmlrpc \ php-http/message \ - php-http/message-factory - RUN composer require --with-all-dependencies \ + php-http/message-factory \ + # docker image specific dependencies elasticsearch/elasticsearch:^8.7.0 \ jakub-onderka/openid-connect-php:^1.0.0 \ aws/aws-sdk-php FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as php-build ENV DEBIAN_FRONTEND noninteractive + ENV TZ Etc/UTC RUN apt-get update; apt-get install -y --no-install-recommends \ gcc \ g++ \ @@ -47,82 +50,91 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as php-build ca-certificates \ php \ php-dev \ + php-xml \ php-pear \ librdkafka-dev \ libsimdjson-dev \ git \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* - - RUN pecl channel-update pecl.php.net - RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib; pecl install ssdeep && pecl install rdkafka && pecl install simdjson + RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib; pecl channel-update pecl.php.net && pecl install ssdeep && pecl install rdkafka && pecl install simdjson RUN git clone --recursive --depth=1 https://github.com/kjdev/php-ext-brotli.git && \ cd php-ext-brotli && phpize && ./configure && make && make install FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build ENV DEBIAN_FRONTEND noninteractive - RUN apt-get update; apt-get install -y --no-install-recommends \ - gcc \ - git \ - python3 \ - python3-dev \ - python3-pip \ - python3-setuptools \ - python3-venv \ - python3-wheel \ - libfuzzy-dev \ - libffi-dev \ - ca-certificates \ - curl \ - && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* - RUN mkdir /wheels + RUN apt-get update; apt-get install -y --no-install-recommends python3-pip git && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* + ARG MISP_TAG + ARG MISP_COMMIT + # Download MISP using git in the /var/www/ directory. Remove unnecessary items. + RUN <<-EOF + if [ ! -z "${MISP_COMMIT}" ]; then + git clone https://github.com/MISP/MISP.git /var/www/MISP && cd /var/www/MISP && git checkout "${MISP_COMMIT}" + else + git clone --branch "${MISP_TAG}" --depth 1 https://github.com/MISP/MISP.git /var/www/MISP + fi - WORKDIR /tmp + cd /var/www/MISP || exit; git submodule update --init --recursive . - RUN git clone --depth 1 https://github.com/CybOXProject/mixbox.git; \ - cd mixbox || exit; python3 setup.py bdist_wheel -d /wheels; \ - sed -i 's/-e //g' requirements.txt; pip3 wheel -r requirements.txt --no-cache-dir -w /wheels/ + # Until MISP project specifies its required PyMISP version, we grab this from the submodule + pymispversion=$(head -n1 PyMISP/pymisp/__init__.py | awk '{print $3}' | tr -d \') - # install python-maec - RUN git clone --depth 1 https://github.com/MAECProject/python-maec.git; \ - cd python-maec || exit; python3 setup.py bdist_wheel -d /wheels + if [ -n "$pymispversion" ]; then + sed -i "s/pymisp$/pymisp==$pymispversion/" requirements.txt + cat requirements.txt + fi + # End of pymisp version fixing hack +EOF - # install python-cybox - RUN git clone --depth 1 https://github.com/CybOXProject/python-cybox.git; \ - cd python-cybox || exit; python3 setup.py bdist_wheel -d /wheels; \ - sed -i 's/-e //g' requirements.txt; pip3 wheel -r requirements.txt --no-cache-dir -w /wheels/ + RUN <<-EOF + mkdir /wheels - # install python stix - RUN git clone --depth 1 https://github.com/STIXProject/python-stix.git; \ - cd python-stix || exit; python3 setup.py bdist_wheel -d /wheels; \ - sed -i 's/-e //g' requirements.txt; pip3 wheel -r requirements.txt --no-cache-dir -w /wheels/ + # Remove modules only used during MISP build + set -- 'coveralls' 'codecov' 'requests-mock' 'nose' 'pip' + for mod in "$@"; do + sed "/${mod}/d" -i /var/www/MISP/requirements.txt + done; - # install STIX2.0 library to support STIX 2.0 export: - # Original Requirements has a bunch of non-required pacakges, force it to only grab wheels for deps from setup.py - RUN git clone --depth 1 https://github.com/MISP/cti-python-stix2.git; \ - cd cti-python-stix2 || exit; python3 setup.py bdist_wheel -d /wheels; \ - echo "-e ." > requirements.txt; pip3 wheel -r requirements.txt --no-cache-dir -w /wheels/ - - # Install the new build tool - RUN pip3 install build - - # install PyMISP - RUN git clone --depth 1 https://github.com/MISP/PyMISP.git; \ - cd PyMISP || exit; python3 -m build --wheel --outdir /wheels - - # install pydeep2 (drop-in replacement for pydeep) - RUN git clone --depth 1 https://github.com/JakubOnderka/pydeep.git; \ - cd pydeep || exit; python3 setup.py bdist_wheel -d /wheels - - # Grab other modules we need - RUN pip3 wheel --no-cache-dir -w /wheels/ plyara pyzmq redis python-magic lief>=0.13.1 - - # Remove extra packages due to incompatible requirements.txt files - WORKDIR /wheels - RUN find . -name "Sphinx*" | tee /dev/stderr | grep -v "Sphinx-1.5.5" | xargs rm -f + # Add additional dependencies (container specific) + # The "set" line contains the list of modules we want to ensure are present. + # PYPI_MODULE_NAME_VERSION env vars can be set to specify the version desired, + # e.g. PYPI_SURICATA_VERSION="==2.0" to specify exactly version 2.0 for the suricata package + # + # 1. Check for presence of each module in requirements.txt + # 2. If missing, add it (with optional version from env (defaults to empty string)) + # 3. If present, replace with our specified version if it exists, otherwise leave + # the upstream version alone. + set -- "redis" "lief" "pydeep2" "python-magic" "misp-lib-stix2" "maec" "mixbox" "cybox" + for mod in "$@"; do + mod_version_var=$(echo "PYPI_${mod}_VERSION" | tr '[:lower:]' '[:upper:]' | tr '-' '_') + mod_version=$(eval "echo \"\$$mod_version_var\"") + grep -q ${mod} /var/www/MISP/requirements.txt + exists=$? + if [ "${exists}" -eq "1" ]; then + echo "Adding missing module ${mod} with version '${mod_version}'" + echo ${mod}${mod_version} >> /var/www/MISP/requirements.txt + else + if [ "$(echo ${mod_version} | wc -m)" -gt 1 ]; then + echo "Overwriting existing module ${mod}, version '${mod_version}'" + sed -i "/${mod}/s/.*/${mod}${mod_version}/" /var/www/MISP/requirements.txt + else + echo "Skipping overwriting ${mod} due to missing version variable" + fi + fi + done; + pip3 wheel --no-cache-dir -w /wheels/ -r /var/www/MISP/requirements.txt + + # Remove files we do not care for + rm -r /var/www/MISP/PyMISP + find /var/www/MISP/INSTALL/* ! -name 'MYSQL.sql' -type f -exec rm {} + + find /var/www/MISP/INSTALL/* ! -name 'MYSQL.sql' -type l -exec rm {} + + # Remove most files in .git - we do not use git functionality in docker + find /var/www/MISP/.git/* ! -name HEAD -exec rm -rf {} + +EOF FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" + ENV DEBIAN_FRONTEND noninteractive ARG MISP_TAG ARG MISP_COMMIT @@ -132,10 +144,10 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" sudo \ nginx \ supervisor \ - git \ cron \ openssl \ - gpg-agent gpg \ + gpg \ + gpg-agent \ ssdeep \ libfuzzy2 \ mariadb-client \ @@ -166,61 +178,52 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" curl jq \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* - # Download MISP using git in the /var/www/ directory. - RUN if [ ! -z ${MISP_COMMIT} ]; then \ - git clone https://github.com/MISP/MISP.git /var/www/MISP && cd /var/www/MISP && git checkout ${MISP_COMMIT}; \ - else git clone --branch ${MISP_TAG} --depth 1 https://github.com/MISP/MISP.git /var/www/MISP; fi - RUN cd /var/www/MISP; git submodule update --init --recursive .; cd /var/www/MISP/app; \ - # Remove some old and broken links that pollute the log files - rm -rf /var/www/MISP/INSTALL/old - - # Python Modules + # Install python modules COPY --from=python-build /wheels /wheels RUN pip3 install --no-cache-dir /wheels/*.whl && rm -rf /wheels # PHP: install prebuilt libraries, then install the app's PHP deps - COPY --from=php-build /usr/lib/php/${PHP_VER}/ssdeep.so /usr/lib/php/${PHP_VER}/ssdeep.so - COPY --from=php-build /usr/lib/php/${PHP_VER}/rdkafka.so /usr/lib/php/${PHP_VER}/rdkafka.so - COPY --from=php-build /usr/lib/php/${PHP_VER}/brotli.so /usr/lib/php/${PHP_VER}/brotli.so - COPY --from=php-build /usr/lib/php/${PHP_VER}/simdjson.so /usr/lib/php/${PHP_VER}/simdjson.so + COPY --from=php-build ["/usr/lib/php/${PHP_VER}/ssdeep.so", "/usr/lib/php/${PHP_VER}/rdkafka.so", "/usr/lib/php/${PHP_VER}/brotli.so", "/usr/lib/php/${PHP_VER}/simdjson.so", "/usr/lib/php/${PHP_VER}/"] # Do an early chown to limit image size - COPY --from=composer-build --chown=www-data:www-data /tmp/Vendor /var/www/MISP/app/Vendor - COPY --from=composer-build --chown=www-data:www-data /tmp/Plugin /var/www/MISP/app/Plugin + COPY --from=python-build --chown=www-data:www-data --chmod=0550 /var/www/MISP /var/www/MISP + COPY --from=composer-build --chown=www-data:www-data --chmod=0550 /tmp/Vendor /var/www/MISP/app/Vendor + COPY --from=composer-build --chown=www-data:www-data --chmod=0550 /tmp/Plugin /var/www/MISP/app/Plugin - RUN for dir in /etc/php/*; do echo "extension=ssdeep.so" > "$dir/mods-available/ssdeep.ini"; done; phpenmod ssdeep - RUN for dir in /etc/php/*; do echo "extension=rdkafka.so" > "$dir/mods-available/rdkafka.ini"; done; phpenmod rdkafka - RUN for dir in /etc/php/*; do echo "extension=brotli.so" > "$dir/mods-available/brotli.ini"; done; phpenmod brotli - RUN for dir in /etc/php/*; do echo "extension=simdjson.so" > "$dir/mods-available/simdjson.ini"; done; phpenmod simdjson - RUN phpenmod redis + # Gather these in one layer, only act on actual directories under /etc/php/ + RUN <<-EOF + set -- "ssdeep" "rdkafka" "brotli" "simdjson" + for mod in "$@"; do + for dir in /etc/php/*/; do + echo "extension=${mod}.so" > "${dir}mods-available/${mod}.ini" + done; + phpenmod "${mod}" + done; + phpenmod redis +EOF # nginx RUN rm /etc/nginx/sites-enabled/*; mkdir /run/php /etc/nginx/certs - COPY files/etc/nginx/misp /etc/nginx/sites-available/misp - COPY files/etc/nginx/misp80 /etc/nginx/sites-available/misp80 # Make a copy of the file and configuration stores, so we can sync from it - RUN cp -R /var/www/MISP/app/files /var/www/MISP/app/files.dist; \ - cp -R /var/www/MISP/app/Config /var/www/MISP/app/Config.dist; - # The spirit of the upstrem dockerization is to make: + # The spirit of the upstream dockerization is to make: # 1) User and group aligned in terms of permissions # 2) Files executable and read only, because of some rogue scripts like 'cake' - # 3) Directories writable, because sometimes MISP add new new files - RUN find /var/www/MISP \( ! -user www-data -or ! -group www-data \) -exec chown www-data:www-data '{}' +; \ - find /var/www/MISP -not -perm 550 -type f -exec chmod 0550 '{}' +; \ - find /var/www/MISP -not -perm 770 -type d -exec chmod 0770 '{}' +; + # 3) Directories writable, because sometimes MISP add new files - # Entrypoints - COPY files/etc/supervisor/supervisor.conf /etc/supervisor/conf.d/10-supervisor.conf - COPY files/etc/supervisor/workers.conf /etc/supervisor/conf.d/50-workers.conf - COPY files/var/www/html/index.php /var/www/html/index.php - COPY files/configure_misp.sh / - COPY files/rest_client.sh / - COPY files/entrypoint_fpm.sh / - COPY files/entrypoint_nginx.sh / - COPY files/entrypoint_cron.sh / - COPY files/entrypoint.sh / + RUN <<-EOF + cp -R /var/www/MISP/app/files /var/www/MISP/app/files.dist + cp -R /var/www/MISP/app/Config /var/www/MISP/app/Config.dist + find /var/www/MISP \( ! -user www-data -or ! -group www-data \) -exec chown www-data:www-data '{}' +; + find /var/www/MISP -not -perm 550 -type f -exec chmod 0550 '{}' +; + find /var/www/MISP -not -perm 770 -type d -exec chmod 0770 '{}' +; + # Diagnostics wants this file to be present and writable even if we do not use git in docker land + touch /var/www/MISP/.git/ORIG_HEAD && chmod 0600 /var/www/MISP/.git/ORIG_HEAD && chown www-data:www-data /var/www/MISP/.git/ORIG_HEAD +EOF + + # Copy all our image specific files to appropriate locations + COPY files/ / ENTRYPOINT [ "/entrypoint.sh" ] # Change Workdirectory diff --git a/server/files/configure_misp.sh b/server/files/configure_misp.sh index bdb4515..e00191a 100755 --- a/server/files/configure_misp.sh +++ b/server/files/configure_misp.sh @@ -157,6 +157,8 @@ apply_critical_fixes() { \"auth\": \"\" } }" > /dev/null + # Avoids displaying errors not relevant to a docker container + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting -q "MISP.self_update" false } apply_optional_fixes() { diff --git a/server/files/etc/nginx/misp b/server/files/etc/nginx/sites-available/misp similarity index 100% rename from server/files/etc/nginx/misp rename to server/files/etc/nginx/sites-available/misp diff --git a/server/files/etc/nginx/misp80 b/server/files/etc/nginx/sites-available/misp80 similarity index 100% rename from server/files/etc/nginx/misp80 rename to server/files/etc/nginx/sites-available/misp80 diff --git a/server/files/etc/supervisor/supervisor.conf b/server/files/etc/supervisor/conf.d/10-supervisor.conf similarity index 100% rename from server/files/etc/supervisor/supervisor.conf rename to server/files/etc/supervisor/conf.d/10-supervisor.conf diff --git a/server/files/etc/supervisor/workers.conf b/server/files/etc/supervisor/conf.d/50-workers.conf similarity index 100% rename from server/files/etc/supervisor/workers.conf rename to server/files/etc/supervisor/conf.d/50-workers.conf diff --git a/template.env b/template.env index f3b6e71..16ed9db 100644 --- a/template.env +++ b/template.env @@ -3,6 +3,15 @@ MODULES_TAG=v2.4.176 PHP_VER=20190902 LIBFAUP_COMMIT=3a26d0a +PYPI_REDIS_VERSION="==5.0.*" +PYPI_LIEF_VERSION=">=0.13.1" +PYPI_PYDEEP2_VERSION="==0.5.*" +PYPI_PYTHON_MAGIC_VERSION="==0.4.*" +PYPI_MISP_LIB_STIX2_VERSION="==3.0.*" +PYPI_MAEC_VERSION="==4.1.*" +PYPI_MIXBOX_VERSION="==1.0.*" +PYPI_CYBOX_VERSION="==2.1.*" + # MISP_COMMIT takes precedence over MISP_TAG # MISP_COMMIT=c56d537 # MODULES_COMMIT takes precedence over MODULES_TAG