Update to Debian bookworm and python 3.12 (#92)

* Update to debian bookworm and python 3.12

* Ship lock file

core/Dockerfile

@@ -1,25 +1,42 @@
 ARG DOCKER_HUB_PROXY=""
 
-FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build
+
+FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" as php-base
+    ENV DEBIAN_FRONTEND noninteractive
+
+    # Uncomment when building in corporate environments
+    # COPY ./rootca.crt /usr/local/share/ca-certificates/rootca.pem
+    # COPY ./rootca.crt /usr/lib/ssl/cert.pem
+
+    RUN apt-get update; apt-get install -y --no-install-recommends \
+        lsb-release \
+        ca-certificates \
+        curl
+    RUN curl -sSLo /tmp/debsuryorg-archive-keyring.deb https://packages.sury.org/debsuryorg-archive-keyring.deb
+    RUN dpkg -i /tmp/debsuryorg-archive-keyring.deb
+    RUN echo "deb [signed-by=/usr/share/keyrings/deb.sury.org-php.gpg] https://packages.sury.org/php/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/php.list
+    RUN apt-get update
+
+
+FROM php-base as composer-build
     ENV DEBIAN_FRONTEND noninteractive
     ENV COMPOSER_ALLOW_SUPERUSER 1
     ARG CORE_TAG
     ARG CORE_COMMIT
 
-    RUN apt-get update; apt-get install -y --no-install-recommends \
-        ca-certificates \
-        php \
-        php-apcu \
-        php-curl \
-        php-xml \
-        php-intl \
-        php-bcmath \
-        php-mbstring \
-        php-mysql \
-        php-redis \
-        php-gd \
-        php-fpm \
-        php-zip \
+    RUN apt-get install -y --no-install-recommends \
+        php7.4 \
+        php7.4-apcu \
+        php7.4-curl \
+        php7.4-xml \
+        php7.4-intl \
+        php7.4-bcmath \
+        php7.4-mbstring \
+        php7.4-mysql \
+        php7.4-redis \
+        php7.4-gd \
+        php7.4-fpm \
+        php7.4-zip \
         unzip \
         && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
@@ -29,41 +46,53 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build
     RUN composer config --no-interaction allow-plugins.composer/installers true
     RUN composer install
     RUN composer require --with-all-dependencies --no-interaction \
-            supervisorphp/supervisor:^4.0 \
-            guzzlehttp/guzzle \
-            lstrojny/fxmlrpc \
-            php-http/message \
-            php-http/message-factory \
-            # docker image specific dependencies
-            elasticsearch/elasticsearch:^8.7.0 \
-            jakub-onderka/openid-connect-php:^1.0.0 \
-            aws/aws-sdk-php
+        supervisorphp/supervisor:^4.0 \
+        guzzlehttp/guzzle \
+        lstrojny/fxmlrpc \
+        php-http/message \
+        php-http/message-factory \
+        # docker image specific dependencies
+        elasticsearch/elasticsearch:^8.7.0 \
+        jakub-onderka/openid-connect-php:^1.0.0 \
+        aws/aws-sdk-php
 
-FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as php-build
+
+FROM php-base as php-build
     ENV DEBIAN_FRONTEND noninteractive
     ENV TZ Etc/UTC
 
-    RUN apt-get update; apt-get install -y --no-install-recommends \
+    RUN apt-get install -y --no-install-recommends \
         gcc \
         g++ \
         make \
+        php7.4 \
+        php7.4-dev \
+        php7.4-xml \
         libfuzzy-dev \
-        ca-certificates \
-        php \
-        php-dev \
-        php-xml \
-        php-pear \
         librdkafka-dev \
         libsimdjson-dev \
         libzstd-dev \
         git \
+        php-pear \
         && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
-    RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib; pecl channel-update pecl.php.net && pecl install ssdeep && pecl install rdkafka && pecl install simdjson && pecl install zstd
-    RUN git clone --recursive --depth=1 https://github.com/kjdev/php-ext-brotli.git && \
-        cd php-ext-brotli && phpize && ./configure && make && make install
+    RUN apt-cache search pecl
 
-FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build
+    RUN update-alternatives --set php /usr/bin/php7.4
+    RUN update-alternatives --set php-config /usr/bin/php-config7.4
+    RUN update-alternatives --set phpize /usr/bin/phpize7.4
+
+    RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib
+    RUN pecl channel-update pecl.php.net && \
+        pecl install ssdeep && \
+        pecl install rdkafka && \
+        pecl install simdjson && \
+        pecl install zstd
+     RUN git clone --recursive --depth=1 https://github.com/kjdev/php-ext-brotli.git && \
+         cd php-ext-brotli && phpize && ./configure && make && make install
+
+
+FROM php-base as python-build
     ENV DEBIAN_FRONTEND noninteractive   
     ARG CORE_TAG
     ARG CORE_COMMIT
@@ -77,8 +106,7 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build
     ARG PYPI_CYBOX_VERSION
     ARG PYPI_PYMISP_VERSION
 
-    RUN apt-get update; apt-get install -y --no-install-recommends \
-        python3-pip \
+    RUN apt-get install -y --no-install-recommends \
         git \
         && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
     
@@ -89,7 +117,6 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build
         else
             git clone --branch "${CORE_TAG}" --depth 1 https://github.com/MISP/MISP.git /var/www/MISP
         fi
-
         cd /var/www/MISP || exit; git submodule update --init --recursive .
 EOF
 
@@ -124,7 +151,7 @@ EOF
             fi
         done;
 
-        pip3 wheel --no-cache-dir -w /wheels/ -r /var/www/MISP/requirements.txt
+        pip wheel --no-cache-dir -w /wheels/ -r /var/www/MISP/requirements.txt
         
         # Remove files we do not care for
         rm -r /var/www/MISP/PyMISP
@@ -134,13 +161,14 @@ EOF
         find /var/www/MISP/.git/* ! -name HEAD -exec rm -rf {} +
 EOF
 
-FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim"
+
+FROM php-base
     ENV DEBIAN_FRONTEND noninteractive
     ARG CORE_TAG
     ARG CORE_COMMIT
     ARG PHP_VER
 
-    RUN apt-get update; apt-get install -y --no-install-recommends \
+    RUN apt-get install -y --no-install-recommends \
         gettext \
         procps \
         sudo \
@@ -150,42 +178,41 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim"
         openssl \
         gpg \
         gpg-agent \
-        ssdeep \
-        libfuzzy2 \
         mariadb-client \
         rsync \
-        # Python Requirements
-        python3 \
-        python3-setuptools \
-        python3-pip \
         # PHP Requirements
-        php \
-        php-apcu \
-        php-curl \
-        php-xml \
-        php-intl \
-        php-bcmath \
-        php-mbstring \
-        php-mysql \
-        php-redis \
-        php-gd \
-        php-fpm \
-        php-zip \
-        php-ldap \
+        php7.4 \
+        php7.4-apcu \
+        php7.4-curl \
+        php7.4-xml \
+        php7.4-intl \
+        php7.4-bcmath \
+        php7.4-mbstring \
+        php7.4-mysql \
+        php7.4-redis \
+        php7.4-gd \
+        php7.4-fpm \
+        php7.4-zip \
+        php7.4-ldap \
+        libmagic1 \
         libldap-common \
         librdkafka1 \
         libbrotli1 \
-        libsimdjson5 \
+        libsimdjson14 \
         libzstd1 \
+        ssdeep \
+        libfuzzy2 \
         # Unsure we need these
         zip unzip \
         # Require for advanced an unattended configuration
         curl jq \
         && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
+    RUN update-alternatives --set php /usr/bin/php7.4
+
     # Install python modules
     COPY --from=python-build /wheels /wheels
-    RUN pip3 install --no-cache-dir /wheels/*.whl && rm -rf /wheels
+    RUN pip install --no-cache-dir /wheels/*.whl && rm -rf /wheels
 
     # PHP: install prebuilt libraries, then install the app's PHP deps
     COPY --from=php-build ["/usr/lib/php/${PHP_VER}/ssdeep.so", "/usr/lib/php/${PHP_VER}/rdkafka.so", "/usr/lib/php/${PHP_VER}/brotli.so", "/usr/lib/php/${PHP_VER}/simdjson.so", "/usr/lib/php/${PHP_VER}/zstd.so", "/usr/lib/php/${PHP_VER}/"]
@@ -208,7 +235,7 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim"
 EOF
 
     # nginx
-    RUN rm /etc/nginx/sites-enabled/*; mkdir /run/php /etc/nginx/certs
+    RUN rm /etc/nginx/sites-enabled/*; mkdir -p /run/php /etc/nginx/certs
 
     # Make a copy of the file and configuration stores, so we can sync from it
 

modules/Dockerfile

@@ -1,6 +1,6 @@
 ARG DOCKER_HUB_PROXY=""
 
-FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye" as python-build
+FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" as python-build
     ENV DEBIAN_FRONTEND noninteractive
     ARG MODULES_TAG
     ARG MODULES_COMMIT
@@ -9,19 +9,17 @@ FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye" as python-build
     RUN apt-get update && apt-get install -y --no-install-recommends \
         cmake \
         git \
-        python3-dev \
-        python3-pip \
-        python3-wheel \
         build-essential \
-        pkg-config \
         libpoppler-cpp-dev \
         libfuzzy-dev \
+        libffi-dev \
+        libxml2-dev \
+        libxslt-dev  \
         libssl-dev \
         ninja-build \
         && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
     RUN mkdir /wheels
-    WORKDIR /srv
 
     RUN <<-EOF
         if [ ! -z ${MODULES_COMMIT} ]; then
@@ -29,52 +27,60 @@ FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye" as python-build
         else
             git clone --branch ${MODULES_TAG} --depth 1 https://github.com/MISP/misp-modules.git /srv/misp-modules
         fi
-
-        cd /srv/misp-modules
-        sed -i "s/numpy.*/numpy<2/" REQUIREMENTS
-        echo "pyeti" >> REQUIREMENTS
-        echo "greynoise" >> REQUIREMENTS
-        echo "Google-Search-API" >> REQUIREMENTS
-        pip3 wheel -r REQUIREMENTS --no-cache-dir -w /wheels/
-        rm -rf /srv/misp-modules
 EOF
 
+    WORKDIR /srv/misp-modules
+    RUN pip install pipenv
+    COPY files/Pipfile Pipfile
+    COPY files/Pipfile.lock Pipfile.lock
+    RUN pipenv requirements > requirements.txt
+    RUN pip wheel -r requirements.txt --no-cache-dir -w /wheels/
+
+    WORKDIR /srv/
+    RUN rm -rf /srv/misp-modules
+
     RUN <<-EOF
         git clone --depth 1 https://github.com/stricaud/faup.git /srv/faup
         cd /srv/faup
         if [ ! -z ${LIBFAUP_COMMIT} ]; then
             git checkout ${LIBFAUP_COMMIT}
         fi
-
-        cd /srv/faup/build
-        cmake -G "Ninja" ../
-        ninja
-        ninja install
-        cd /srv/faup/src/lib/bindings/python
-        pip3 wheel --no-cache-dir --no-dependencies -w /wheels/ .
-        rm -rf /srv/faup
 EOF
 
-FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye"
+    WORKDIR /srv/faup/build
+    RUN cmake -G "Ninja" ../
+    RUN ninja
+    RUN ninja install
+    WORKDIR /srv/faup/src/lib/bindings/python
+    RUN pip wheel --no-cache-dir --no-dependencies -w /wheels/ .
+
+    WORKDIR /srv/
+    RUN rm -rf /srv/faup
+
+
+FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm"
     ENV DEBIAN_FRONTEND noninteractive
 
     RUN apt-get update && apt-get install -y --no-install-recommends \
         libglib2.0-0 \
-        libzbar0 \
-        libxrender1 \
-        libxext6 \
         libpoppler-cpp0v5 \
         libgl1 \
+        libfuzzy2 \
+        libffi8 \
+        libxext6 \
+        libxml2 \
+        libxslt1.1  \
+        libzbar0 \
         && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
     COPY --from=python-build /wheels /wheels
     COPY --from=python-build /usr/local/lib/libfaupl* /usr/local/lib/
-    RUN pip3 install --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl; ldconfig && rm -rf /wheels
+    RUN pip install --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl; ldconfig && rm -rf /wheels
 
     # Since we compile faup ourselves and lua is not required anymore, we can load our own library 
     #   and skip the pre-compiled blob to improve compatibility with other architectures like ARM
     RUN sed -i s/LoadLibrary\(LOAD_LIB\)/LoadLibrary\(\"\\/usr\\/local\\/lib\\/libfaupl.so\"\)/ \
-        /usr/local/lib/python3.11/site-packages/pyfaup/__init__.py
+        /usr/local/lib/python3.12/site-packages/pyfaup/__init__.py
 
     # Disable (all) warnings raised when using 'future'
     RUN sed -i '/import sys/a import warnings\nwarnings.warn = lambda *args, **kwargs: None' \

modules/files/Pipfile

@@ -0,0 +1,86 @@
+[[source]]
+name = "pypi"
+url = "https://pypi.org/simple"
+verify_ssl = true
+
+[dev-packages]
+nose = "*"
+codecov = "*"
+pytest = "*"
+flake8 = "*"
+
+[packages]
+numpy = "<2.0.0"
+matplotlib = "*"
+sigmf = "*"
+pysafebrowsing = "*"
+dnspython = "*"
+requests = { extras = ["security"], version = "*" }
+urlarchiver = "*"
+passivetotal = "*"
+pypdns = "*"
+pypssl = "*"
+pyeupi = "*"
+pymisp = { extras = ["fileobjects,openioc,pdfexport,email,url"], version = "*" }
+pyonyphe = { git = "https://github.com/sebdraven/pyonyphe" }
+pydnstrails = { git = "https://github.com/sebdraven/pydnstrails" }
+pytesseract = "*"
+pygeoip = "*"
+beautifulsoup4 = "*"
+oauth2 = "*"
+yara-python = "==3.8.1"
+sigmatools = "*"
+stix2 = "*"
+stix2-patterns = "*"
+taxii2-client = "*"
+maclookup = "*"
+vulners = "*"
+blockchain = "*"
+reportlab = "*"
+pyintel471 = { git = "https://github.com/MISP/PyIntel471.git" }
+shodan = "*"
+Pillow = ">=8.2.0"
+Wand = "*"
+SPARQLWrapper = "*"
+domaintools_api = "*"
+misp-modules = { path = "." }
+pybgpranking = { git = "https://github.com/D4-project/BGP-Ranking.git/", subdirectory = "client", ref = "68de39f6c5196f796055c1ac34504054d688aa59" }
+pyipasnhistory = { git = "https://github.com/D4-project/IPASN-History.git/", subdirectory = "client", ref = "a2853c39265cecdd0c0d16850bd34621c0551b87" }
+backscatter = "*"
+pyzbar = "*"
+opencv-python = "*"
+np = "*"
+ODTReader = { git = "https://github.com/cartertemm/ODTReader.git/" }
+python-pptx = "*"
+python-docx = "*"
+ezodf = "*"
+pandas = "==1.3.5"
+pandas_ods_reader = "==0.1.2"
+pdftotext = "*"
+lxml = "*"
+xlrd = "*"
+jbxapi = "*"
+geoip2 = "*"
+apiosintDS = "*"
+assemblyline_client = "*"
+vt-graph-api = "*"
+trustar = { git = "https://github.com/SteveClement/trustar-python.git" }
+markdownify = "==0.5.3"
+socialscan = "==1.4"
+pycountry = "==22.3.5"
+dnsdb2 = "*"
+clamd = "*"
+aiohttp = ">=3.7.4"
+tau-clients = "*"
+vt-py = ">=0.7.1"
+crowdstrike-falconpy = "0.9.0"
+censys = "2.0.9"
+mwdblib = "3.4.1"
+ndjson = "0.3.1"
+Jinja2 = "3.1.2"
+mattermostdriver = "7.3.2"
+openpyxl = "*"
+slack-sdk = "3.27.1"
+
+[requires]
+python_version = "3.12"