Replace Debian with Ubuntu 24.04 LTS

2024-07-12 17:41:37 +01:00 · 2024-07-12 17:41:37 +01:00 · dcea8d5791
parent bae3346f7a
commit dcea8d5791
6 changed files with 89 additions and 54 deletions
--- a/README.md
+++ b/README.md
@ -27,6 +27,14 @@ Notable features:

 The underlying spirit of this project is to allow "repeatable deployments", and all pull requests in this direction will be merged post-haste.

+## Warning 
+
+As part of our recent efforts to reduce the number of CVEs affecting the Docker images, we recently changed the base image from Debian Bookworm to Ubuntu 24.04.
+
+While the transition did not affect MISP and MISP modules, the GitHub Action triggered a bug affecting `libcurl`  and Ubuntu 24.04 when running on `linux/arm64` and establishing TLS connections to `api.github.com` when the server decides toreturn a 302. The issue is being discussed here https://github.com/curl/curl/issues/14154 and being further investigated here https://bugs.launchpad.net/ubuntu/+source/curl/+bug/2073448.
+
+To allow the build to complete, we temporarily disabled TLS verification (see `core/Dockerfile` when using `composer` to install PHP dependencies; the temporary workaround affects only the build when the target platform is `linux/arm64`, leaving the `linux/amd64` build unaffected.
+
 ## Getting Started

 -   Copy the `template.env` to `.env` 
--- a/core/Dockerfile
+++ b/core/Dockerfile
@ -1,7 +1,7 @@
 ARG DOCKER_HUB_PROXY=""


-FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" AS php-base
+FROM "${DOCKER_HUB_PROXY}ubuntu:24.04" AS php-base
    ENV DEBIAN_FRONTEND noninteractive

    # Uncomment when building in corporate environments
@ -19,29 +19,48 @@ FROM php-base AS composer-build
    ENV COMPOSER_ALLOW_SUPERUSER 1
    ARG CORE_TAG
    ARG CORE_COMMIT
+    ARG TARGETPLATFORM

    RUN apt-get install -y --no-install-recommends \
-        php8.2 \
-        php8.2-apcu \
-        php8.2-curl \
-        php8.2-xml \
-        php8.2-intl \
-        php8.2-bcmath \
-        php8.2-mbstring \
-        php8.2-mysql \
-        php8.2-redis \
-        php8.2-gd \
-        php8.2-fpm \
-        php8.2-zip \
+        php8.3 \
+        php8.3-apcu \
+        php8.3-curl \
+        php8.3-xml \
+        php8.3-intl \
+        php8.3-bcmath \
+        php8.3-mbstring \
+        php8.3-mysql \
+        php8.3-redis \
+        php8.3-gd \
+        php8.3-fpm \
+        php8.3-zip \
        unzip \
        && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*

    WORKDIR /tmp
    RUN curl -o /tmp/composer.json https://raw.githubusercontent.com/MISP/MISP/${CORE_COMMIT:-${CORE_TAG}}/app/composer.json
-    COPY --from=composer:latest /usr/bin/composer /usr/bin/composer
-    RUN composer config --no-interaction allow-plugins.composer/installers true
-    RUN composer install
-    RUN composer require --with-all-dependencies --no-interaction \
+    COPY --from=composer:2.7.7 /usr/bin/composer /usr/bin/composer
+
+    # See:
+    #   - https://github.com/curl/curl/issues/14154
+    #   - https://bugs.launchpad.net/ubuntu/+source/curl/+bug/2073448
+    RUN <<-EOF
+        if [ "$TARGETPLATFORM" = "linux/arm64" ]; then
+            cp /usr/bin/composer /composer.phar
+            mkdir /out/
+            php -r '$phar = new Phar("/composer.phar"); $phar->extractTo("/out/");'
+            sed -i "/'verify_peer_name' =>.*/a 'verify_peer_status' => CURLOPT_SSL_VERIFYSTATUS," /out/src/Composer/Util/Http/CurlDownloader.php
+            sed -i "/\$options = StreamContextFactory.*/a \$options['ssl']['verify_peer'] = false;" /out/src/Composer/Util/Http/CurlDownloader.php
+            sed -i "/\$options = StreamContextFactory.*/a \$options['ssl']['verify_peer_name'] = false;" /out/src/Composer/Util/Http/CurlDownloader.php
+            sed -i "/\$options = StreamContextFactory.*/a \$options['ssl']['verify_peer_status'] = false;" /out/src/Composer/Util/Http/CurlDownloader.php
+            rm /usr/bin/composer
+            ln -s /out/bin/composer /usr/bin/composer
+        fi
+EOF
+
+    RUN php /usr/bin/composer config --no-interaction allow-plugins.composer/installers true
+    RUN php /usr/bin/composer install
+    RUN php /usr/bin/composer require --with-all-dependencies --no-interaction \
        elasticsearch/elasticsearch:^8.7.0 \
        jakub-onderka/openid-connect-php:^1.0.0 \
        aws/aws-sdk-php
@ -56,9 +75,9 @@ FROM php-base AS php-build
        g++ \
        git \
        make \
-        php8.2 \
-        php8.2-dev \
-        php8.2-xml \
+        php8.3 \
+        php8.3-dev \
+        php8.3-xml \
        php-pear \
        libbrotli-dev \
        libfuzzy-dev \
@ -67,9 +86,9 @@ FROM php-base AS php-build
        libzstd-dev \
        && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*

-    RUN update-alternatives --set php /usr/bin/php8.2
-    RUN update-alternatives --set php-config /usr/bin/php-config8.2
-    RUN update-alternatives --set phpize /usr/bin/phpize8.2
+    RUN update-alternatives --set php /usr/bin/php8.3
+    RUN update-alternatives --set php-config /usr/bin/php-config8.3
+    RUN update-alternatives --set phpize /usr/bin/phpize8.3

    RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib
    RUN pecl channel-update pecl.php.net && \
@ -99,6 +118,8 @@ FROM php-base AS python-build

    RUN apt-get install -y --no-install-recommends \
        git \
+        python3-pip \
+        python3-wheel \
        && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*

    # Download MISP using git in the /var/www/ directory. Remove unnecessary items.
@ -178,25 +199,28 @@ FROM php-base
        gpg-agent \
        mariadb-client \
        rsync \
+        python3-setuptools \
+        python3-pip \
+        python3-wheel \
        # PHP Requirements
-        php8.2 \
-        php8.2-apcu \
-        php8.2-curl \
-        php8.2-xml \
-        php8.2-intl \
-        php8.2-bcmath \
-        php8.2-mbstring \
-        php8.2-mysql \
-        php8.2-redis \
-        php8.2-gd \
-        php8.2-fpm \
-        php8.2-zip \
-        php8.2-ldap \
+        php8.3 \
+        php8.3-apcu \
+        php8.3-curl \
+        php8.3-xml \
+        php8.3-intl \
+        php8.3-bcmath \
+        php8.3-mbstring \
+        php8.3-mysql \
+        php8.3-redis \
+        php8.3-gd \
+        php8.3-fpm \
+        php8.3-zip \
+        php8.3-ldap \
        libmagic1 \
        libldap-common \
        librdkafka1 \
        libbrotli1 \
-        libsimdjson14 \
+        libsimdjson19 \
        libzstd1 \
        ssdeep \
        libfuzzy2 \
@ -206,11 +230,11 @@ FROM php-base
        curl jq \
        && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*

-    RUN update-alternatives --set php /usr/bin/php8.2
+    RUN update-alternatives --set php /usr/bin/php8.3

    # Install python modules
    COPY --from=python-build /wheels /wheels
-    RUN pip install --no-cache-dir /wheels/*.whl && rm -rf /wheels
+    RUN pip install --break-system-packages --no-cache-dir /wheels/*.whl && rm -rf /wheels

    # PHP: install prebuilt libraries, then install the app's PHP deps
    COPY --from=php-build ["/usr/lib/php/${PHP_VER}/ssdeep.so", "/usr/lib/php/${PHP_VER}/rdkafka.so", "/usr/lib/php/${PHP_VER}/brotli.so", "/usr/lib/php/${PHP_VER}/simdjson.so", "/usr/lib/php/${PHP_VER}/zstd.so", "/usr/lib/php/${PHP_VER}/"]
@ -222,12 +246,11 @@ FROM php-base
    COPY --from=composer-build --chown=www-data:www-data --chmod=0550 /tmp/Plugin /var/www/MISP/app/Plugin

    # 'setuptools' is needed at runtime by 'mixbox'
-    RUN pip install setuptools
-    RUN cp /usr/local/lib/python3.12/site-packages/setuptools/_distutils/version.py \
-        /usr/local/lib/python3.12/site-packages/mixbox/distutils_version.py
+    RUN cp /usr/lib/python3/dist-packages/setuptools/_distutils/version.py \
+        /usr/local/lib/python3.12/dist-packages/mixbox/distutils_version.py
    RUN sed -i 's/from distutils\.version/from mixbox.distutils_version/' \
-        /usr/local/lib/python3.12/site-packages/mixbox/parser.py
-    RUN pip uninstall -y setuptools pip
+        /usr/local/lib/python3.12/dist-packages/mixbox/parser.py
+    RUN apt-get remove --purge python3-pip python3-wheel python3-setuptools -y

    # Gather these in one layer, only act on actual directories under /etc/php/
    RUN <<-EOF
--- a/core/files/entrypoint_fpm.sh
+++ b/core/files/entrypoint_fpm.sh
@ -34,7 +34,7 @@ change_php_vars() {
 echo "Configure PHP | Change PHP values ..." && change_php_vars

 echo "Configure PHP | Starting PHP FPM"
-/usr/sbin/php-fpm8.2 -R -F & master_pid=$!
+/usr/sbin/php-fpm8.3 -R -F & master_pid=$!

 # Wait for it
 wait "$master_pid"
--- a/core/files/etc/nginx/includes/misp
+++ b/core/files/etc/nginx/includes/misp
@ -23,7 +23,7 @@ location / {

 location ~ ^/[^/]+\.php(/|$) {
    include snippets/fastcgi-php.conf;
-    fastcgi_pass unix:/var/run/php/php8.2-fpm.sock;
+    fastcgi_pass unix:/var/run/php/php8.3-fpm.sock;
    fastcgi_read_timeout 300s;
    fastcgi_send_timeout 300s;
    fastcgi_connect_timeout 300s;
--- a/modules/Dockerfile
+++ b/modules/Dockerfile
@ -1,6 +1,6 @@
 ARG DOCKER_HUB_PROXY=""

-FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" AS python-build
+FROM "${DOCKER_HUB_PROXY}ubuntu:24.04" AS python-build
    ENV DEBIAN_FRONTEND noninteractive
    ARG MODULES_TAG
    ARG MODULES_COMMIT
@ -14,6 +14,9 @@ FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" AS python-build
        ca-certificates \
        cmake \
        git \
+        python3-dev \
+        python3-pip \
+        python3-wheel \
        build-essential \
        libpoppler-cpp-dev \
        libfuzzy-dev \
@ -35,8 +38,8 @@ FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" AS python-build
 EOF

    WORKDIR /srv/misp-modules
-    RUN pip install poetry
-    RUN sed -i "s/^python = .*/python = \"$(python -c 'import platform; print(platform.python_version())')\"/" pyproject.toml
+    RUN pip install --break-system-packages poetry
+    RUN sed -i "s/^python = .*/python = \"$(python3 -c 'import platform; print(platform.python_version())')\"/" pyproject.toml
    RUN poetry lock
    RUN poetry self add poetry-plugin-export
    RUN poetry export --with unstable --without-hashes -f requirements.txt -o requirements.txt
@ -65,7 +68,7 @@ EOF
    RUN rm -rf /srv/faup


-FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm"
+FROM "${DOCKER_HUB_PROXY}ubuntu:24.04"
    ENV DEBIAN_FRONTEND noninteractive

    RUN apt-get update && apt-get install -y --no-install-recommends \
@ -78,18 +81,19 @@ FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm"
        libxml2 \
        libxslt1.1  \
        libzbar0 \
+        python3-pip \
        && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*

    COPY --from=python-build /wheels /wheels
    COPY --from=python-build /usr/local/lib/libfaupl* /usr/local/lib/
    RUN ldconfig
-    RUN pip install --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl && rm -rf /wheels
-    RUN pip uninstall -y pip
+    RUN pip install --break-system-packages --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl && rm -rf /wheels
+    RUN apt-get remove --purge python3-pip python3-setuptools -y

    # Since we compile faup ourselves and lua is not required anymore, we can load our own library 
    #   and skip the pre-compiled blob to improve compatibility with other architectures like ARM
    RUN sed -i s/LoadLibrary\(LOAD_LIB\)/LoadLibrary\(\"\\/usr\\/local\\/lib\\/libfaupl.so\"\)/ \
-        /usr/local/lib/python3.12/site-packages/pyfaup/__init__.py
+        /usr/local/lib/python3.12/dist-packages/pyfaup/__init__.py

    # Disable (all) warnings raised when using 'future'
    RUN sed -i '/import sys/a import warnings\nwarnings.warn = lambda *args, **kwargs: None' \
--- a/template.env
+++ b/template.env
@ -4,7 +4,7 @@

 CORE_TAG=v2.5.1
 MODULES_TAG=v2.4.198
-PHP_VER=20220829
+PHP_VER=20230831
 LIBFAUP_COMMIT=3a26d0a

 # PYPY_* vars take precedence over MISP's