From dcea8d579188accce454565a4de565c6e2bd7e0b Mon Sep 17 00:00:00 2001 From: Stefano Ortolani Date: Fri, 12 Jul 2024 17:41:37 +0100 Subject: [PATCH] Replace Debian with Ubuntu 24.04 LTS --- README.md | 8 +++ core/Dockerfile | 111 +++++++++++++++++------------ core/files/entrypoint_fpm.sh | 2 +- core/files/etc/nginx/includes/misp | 2 +- modules/Dockerfile | 18 +++-- template.env | 2 +- 6 files changed, 89 insertions(+), 54 deletions(-) diff --git a/README.md b/README.md index ea319da..47a5b45 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,14 @@ Notable features: The underlying spirit of this project is to allow "repeatable deployments", and all pull requests in this direction will be merged post-haste. +## Warning + +As part of our recent efforts to reduce the number of CVEs affecting the Docker images, we recently changed the base image from Debian Bookworm to Ubuntu 24.04. + +While the transition did not affect MISP and MISP modules, the GitHub Action triggered a bug affecting `libcurl` and Ubuntu 24.04 when running on `linux/arm64` and establishing TLS connections to `api.github.com` when the server decides toreturn a 302. The issue is being discussed here https://github.com/curl/curl/issues/14154 and being further investigated here https://bugs.launchpad.net/ubuntu/+source/curl/+bug/2073448. + +To allow the build to complete, we temporarily disabled TLS verification (see `core/Dockerfile` when using `composer` to install PHP dependencies; the temporary workaround affects only the build when the target platform is `linux/arm64`, leaving the `linux/amd64` build unaffected. + ## Getting Started - Copy the `template.env` to `.env` diff --git a/core/Dockerfile b/core/Dockerfile index d090a9e..012a39e 100644 --- a/core/Dockerfile +++ b/core/Dockerfile @@ -1,7 +1,7 @@ ARG DOCKER_HUB_PROXY="" -FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" AS php-base +FROM "${DOCKER_HUB_PROXY}ubuntu:24.04" AS php-base ENV DEBIAN_FRONTEND noninteractive # Uncomment when building in corporate environments @@ -19,29 +19,48 @@ FROM php-base AS composer-build ENV COMPOSER_ALLOW_SUPERUSER 1 ARG CORE_TAG ARG CORE_COMMIT + ARG TARGETPLATFORM RUN apt-get install -y --no-install-recommends \ - php8.2 \ - php8.2-apcu \ - php8.2-curl \ - php8.2-xml \ - php8.2-intl \ - php8.2-bcmath \ - php8.2-mbstring \ - php8.2-mysql \ - php8.2-redis \ - php8.2-gd \ - php8.2-fpm \ - php8.2-zip \ + php8.3 \ + php8.3-apcu \ + php8.3-curl \ + php8.3-xml \ + php8.3-intl \ + php8.3-bcmath \ + php8.3-mbstring \ + php8.3-mysql \ + php8.3-redis \ + php8.3-gd \ + php8.3-fpm \ + php8.3-zip \ unzip \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* WORKDIR /tmp RUN curl -o /tmp/composer.json https://raw.githubusercontent.com/MISP/MISP/${CORE_COMMIT:-${CORE_TAG}}/app/composer.json - COPY --from=composer:latest /usr/bin/composer /usr/bin/composer - RUN composer config --no-interaction allow-plugins.composer/installers true - RUN composer install - RUN composer require --with-all-dependencies --no-interaction \ + COPY --from=composer:2.7.7 /usr/bin/composer /usr/bin/composer + + # See: + # - https://github.com/curl/curl/issues/14154 + # - https://bugs.launchpad.net/ubuntu/+source/curl/+bug/2073448 + RUN <<-EOF + if [ "$TARGETPLATFORM" = "linux/arm64" ]; then + cp /usr/bin/composer /composer.phar + mkdir /out/ + php -r '$phar = new Phar("/composer.phar"); $phar->extractTo("/out/");' + sed -i "/'verify_peer_name' =>.*/a 'verify_peer_status' => CURLOPT_SSL_VERIFYSTATUS," /out/src/Composer/Util/Http/CurlDownloader.php + sed -i "/\$options = StreamContextFactory.*/a \$options['ssl']['verify_peer'] = false;" /out/src/Composer/Util/Http/CurlDownloader.php + sed -i "/\$options = StreamContextFactory.*/a \$options['ssl']['verify_peer_name'] = false;" /out/src/Composer/Util/Http/CurlDownloader.php + sed -i "/\$options = StreamContextFactory.*/a \$options['ssl']['verify_peer_status'] = false;" /out/src/Composer/Util/Http/CurlDownloader.php + rm /usr/bin/composer + ln -s /out/bin/composer /usr/bin/composer + fi +EOF + + RUN php /usr/bin/composer config --no-interaction allow-plugins.composer/installers true + RUN php /usr/bin/composer install + RUN php /usr/bin/composer require --with-all-dependencies --no-interaction \ elasticsearch/elasticsearch:^8.7.0 \ jakub-onderka/openid-connect-php:^1.0.0 \ aws/aws-sdk-php @@ -56,9 +75,9 @@ FROM php-base AS php-build g++ \ git \ make \ - php8.2 \ - php8.2-dev \ - php8.2-xml \ + php8.3 \ + php8.3-dev \ + php8.3-xml \ php-pear \ libbrotli-dev \ libfuzzy-dev \ @@ -67,9 +86,9 @@ FROM php-base AS php-build libzstd-dev \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* - RUN update-alternatives --set php /usr/bin/php8.2 - RUN update-alternatives --set php-config /usr/bin/php-config8.2 - RUN update-alternatives --set phpize /usr/bin/phpize8.2 + RUN update-alternatives --set php /usr/bin/php8.3 + RUN update-alternatives --set php-config /usr/bin/php-config8.3 + RUN update-alternatives --set phpize /usr/bin/phpize8.3 RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib RUN pecl channel-update pecl.php.net && \ @@ -99,6 +118,8 @@ FROM php-base AS python-build RUN apt-get install -y --no-install-recommends \ git \ + python3-pip \ + python3-wheel \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* # Download MISP using git in the /var/www/ directory. Remove unnecessary items. @@ -178,25 +199,28 @@ FROM php-base gpg-agent \ mariadb-client \ rsync \ + python3-setuptools \ + python3-pip \ + python3-wheel \ # PHP Requirements - php8.2 \ - php8.2-apcu \ - php8.2-curl \ - php8.2-xml \ - php8.2-intl \ - php8.2-bcmath \ - php8.2-mbstring \ - php8.2-mysql \ - php8.2-redis \ - php8.2-gd \ - php8.2-fpm \ - php8.2-zip \ - php8.2-ldap \ + php8.3 \ + php8.3-apcu \ + php8.3-curl \ + php8.3-xml \ + php8.3-intl \ + php8.3-bcmath \ + php8.3-mbstring \ + php8.3-mysql \ + php8.3-redis \ + php8.3-gd \ + php8.3-fpm \ + php8.3-zip \ + php8.3-ldap \ libmagic1 \ libldap-common \ librdkafka1 \ libbrotli1 \ - libsimdjson14 \ + libsimdjson19 \ libzstd1 \ ssdeep \ libfuzzy2 \ @@ -206,11 +230,11 @@ FROM php-base curl jq \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* - RUN update-alternatives --set php /usr/bin/php8.2 + RUN update-alternatives --set php /usr/bin/php8.3 # Install python modules COPY --from=python-build /wheels /wheels - RUN pip install --no-cache-dir /wheels/*.whl && rm -rf /wheels + RUN pip install --break-system-packages --no-cache-dir /wheels/*.whl && rm -rf /wheels # PHP: install prebuilt libraries, then install the app's PHP deps COPY --from=php-build ["/usr/lib/php/${PHP_VER}/ssdeep.so", "/usr/lib/php/${PHP_VER}/rdkafka.so", "/usr/lib/php/${PHP_VER}/brotli.so", "/usr/lib/php/${PHP_VER}/simdjson.so", "/usr/lib/php/${PHP_VER}/zstd.so", "/usr/lib/php/${PHP_VER}/"] @@ -222,12 +246,11 @@ FROM php-base COPY --from=composer-build --chown=www-data:www-data --chmod=0550 /tmp/Plugin /var/www/MISP/app/Plugin # 'setuptools' is needed at runtime by 'mixbox' - RUN pip install setuptools - RUN cp /usr/local/lib/python3.12/site-packages/setuptools/_distutils/version.py \ - /usr/local/lib/python3.12/site-packages/mixbox/distutils_version.py + RUN cp /usr/lib/python3/dist-packages/setuptools/_distutils/version.py \ + /usr/local/lib/python3.12/dist-packages/mixbox/distutils_version.py RUN sed -i 's/from distutils\.version/from mixbox.distutils_version/' \ - /usr/local/lib/python3.12/site-packages/mixbox/parser.py - RUN pip uninstall -y setuptools pip + /usr/local/lib/python3.12/dist-packages/mixbox/parser.py + RUN apt-get remove --purge python3-pip python3-wheel python3-setuptools -y # Gather these in one layer, only act on actual directories under /etc/php/ RUN <<-EOF diff --git a/core/files/entrypoint_fpm.sh b/core/files/entrypoint_fpm.sh index 1b78d2d..231f4f9 100755 --- a/core/files/entrypoint_fpm.sh +++ b/core/files/entrypoint_fpm.sh @@ -34,7 +34,7 @@ change_php_vars() { echo "Configure PHP | Change PHP values ..." && change_php_vars echo "Configure PHP | Starting PHP FPM" -/usr/sbin/php-fpm8.2 -R -F & master_pid=$! +/usr/sbin/php-fpm8.3 -R -F & master_pid=$! # Wait for it wait "$master_pid" diff --git a/core/files/etc/nginx/includes/misp b/core/files/etc/nginx/includes/misp index 7f748e1..b7f8ba0 100644 --- a/core/files/etc/nginx/includes/misp +++ b/core/files/etc/nginx/includes/misp @@ -23,7 +23,7 @@ location / { location ~ ^/[^/]+\.php(/|$) { include snippets/fastcgi-php.conf; - fastcgi_pass unix:/var/run/php/php8.2-fpm.sock; + fastcgi_pass unix:/var/run/php/php8.3-fpm.sock; fastcgi_read_timeout 300s; fastcgi_send_timeout 300s; fastcgi_connect_timeout 300s; diff --git a/modules/Dockerfile b/modules/Dockerfile index dca7bb1..ef2dc03 100644 --- a/modules/Dockerfile +++ b/modules/Dockerfile @@ -1,6 +1,6 @@ ARG DOCKER_HUB_PROXY="" -FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" AS python-build +FROM "${DOCKER_HUB_PROXY}ubuntu:24.04" AS python-build ENV DEBIAN_FRONTEND noninteractive ARG MODULES_TAG ARG MODULES_COMMIT @@ -14,6 +14,9 @@ FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" AS python-build ca-certificates \ cmake \ git \ + python3-dev \ + python3-pip \ + python3-wheel \ build-essential \ libpoppler-cpp-dev \ libfuzzy-dev \ @@ -35,8 +38,8 @@ FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" AS python-build EOF WORKDIR /srv/misp-modules - RUN pip install poetry - RUN sed -i "s/^python = .*/python = \"$(python -c 'import platform; print(platform.python_version())')\"/" pyproject.toml + RUN pip install --break-system-packages poetry + RUN sed -i "s/^python = .*/python = \"$(python3 -c 'import platform; print(platform.python_version())')\"/" pyproject.toml RUN poetry lock RUN poetry self add poetry-plugin-export RUN poetry export --with unstable --without-hashes -f requirements.txt -o requirements.txt @@ -65,7 +68,7 @@ EOF RUN rm -rf /srv/faup -FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" +FROM "${DOCKER_HUB_PROXY}ubuntu:24.04" ENV DEBIAN_FRONTEND noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ @@ -78,18 +81,19 @@ FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" libxml2 \ libxslt1.1 \ libzbar0 \ + python3-pip \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* COPY --from=python-build /wheels /wheels COPY --from=python-build /usr/local/lib/libfaupl* /usr/local/lib/ RUN ldconfig - RUN pip install --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl && rm -rf /wheels - RUN pip uninstall -y pip + RUN pip install --break-system-packages --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl && rm -rf /wheels + RUN apt-get remove --purge python3-pip python3-setuptools -y # Since we compile faup ourselves and lua is not required anymore, we can load our own library # and skip the pre-compiled blob to improve compatibility with other architectures like ARM RUN sed -i s/LoadLibrary\(LOAD_LIB\)/LoadLibrary\(\"\\/usr\\/local\\/lib\\/libfaupl.so\"\)/ \ - /usr/local/lib/python3.12/site-packages/pyfaup/__init__.py + /usr/local/lib/python3.12/dist-packages/pyfaup/__init__.py # Disable (all) warnings raised when using 'future' RUN sed -i '/import sys/a import warnings\nwarnings.warn = lambda *args, **kwargs: None' \ diff --git a/template.env b/template.env index 13b8c1e..d7f523f 100644 --- a/template.env +++ b/template.env @@ -4,7 +4,7 @@ CORE_TAG=v2.5.1 MODULES_TAG=v2.4.198 -PHP_VER=20220829 +PHP_VER=20230831 LIBFAUP_COMMIT=3a26d0a # PYPY_* vars take precedence over MISP's