Update to Debian bookworm and python 3.12 (#92)

* Update to debian bookworm and python 3.12

* Ship lock file
pull/99/head
Stefano Ortolani 2024-07-18 15:36:06 +01:00 committed by GitHub
parent 1c89ebcd67
commit 162f99f541
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 3596 additions and 94 deletions

View File

@ -1,25 +1,42 @@
ARG DOCKER_HUB_PROXY="" ARG DOCKER_HUB_PROXY=""
FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build
FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" as php-base
ENV DEBIAN_FRONTEND noninteractive
# Uncomment when building in corporate environments
# COPY ./rootca.crt /usr/local/share/ca-certificates/rootca.pem
# COPY ./rootca.crt /usr/lib/ssl/cert.pem
RUN apt-get update; apt-get install -y --no-install-recommends \
lsb-release \
ca-certificates \
curl
RUN curl -sSLo /tmp/debsuryorg-archive-keyring.deb https://packages.sury.org/debsuryorg-archive-keyring.deb
RUN dpkg -i /tmp/debsuryorg-archive-keyring.deb
RUN echo "deb [signed-by=/usr/share/keyrings/deb.sury.org-php.gpg] https://packages.sury.org/php/ $(lsb_release -sc) main" > /etc/apt/sources.list.d/php.list
RUN apt-get update
FROM php-base as composer-build
ENV DEBIAN_FRONTEND noninteractive ENV DEBIAN_FRONTEND noninteractive
ENV COMPOSER_ALLOW_SUPERUSER 1 ENV COMPOSER_ALLOW_SUPERUSER 1
ARG CORE_TAG ARG CORE_TAG
ARG CORE_COMMIT ARG CORE_COMMIT
RUN apt-get update; apt-get install -y --no-install-recommends \ RUN apt-get install -y --no-install-recommends \
ca-certificates \ php7.4 \
php \ php7.4-apcu \
php-apcu \ php7.4-curl \
php-curl \ php7.4-xml \
php-xml \ php7.4-intl \
php-intl \ php7.4-bcmath \
php-bcmath \ php7.4-mbstring \
php-mbstring \ php7.4-mysql \
php-mysql \ php7.4-redis \
php-redis \ php7.4-gd \
php-gd \ php7.4-fpm \
php-fpm \ php7.4-zip \
php-zip \
unzip \ unzip \
&& apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
@ -29,41 +46,53 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build
RUN composer config --no-interaction allow-plugins.composer/installers true RUN composer config --no-interaction allow-plugins.composer/installers true
RUN composer install RUN composer install
RUN composer require --with-all-dependencies --no-interaction \ RUN composer require --with-all-dependencies --no-interaction \
supervisorphp/supervisor:^4.0 \ supervisorphp/supervisor:^4.0 \
guzzlehttp/guzzle \ guzzlehttp/guzzle \
lstrojny/fxmlrpc \ lstrojny/fxmlrpc \
php-http/message \ php-http/message \
php-http/message-factory \ php-http/message-factory \
# docker image specific dependencies # docker image specific dependencies
elasticsearch/elasticsearch:^8.7.0 \ elasticsearch/elasticsearch:^8.7.0 \
jakub-onderka/openid-connect-php:^1.0.0 \ jakub-onderka/openid-connect-php:^1.0.0 \
aws/aws-sdk-php aws/aws-sdk-php
FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as php-build
FROM php-base as php-build
ENV DEBIAN_FRONTEND noninteractive ENV DEBIAN_FRONTEND noninteractive
ENV TZ Etc/UTC ENV TZ Etc/UTC
RUN apt-get update; apt-get install -y --no-install-recommends \ RUN apt-get install -y --no-install-recommends \
gcc \ gcc \
g++ \ g++ \
make \ make \
php7.4 \
php7.4-dev \
php7.4-xml \
libfuzzy-dev \ libfuzzy-dev \
ca-certificates \
php \
php-dev \
php-xml \
php-pear \
librdkafka-dev \ librdkafka-dev \
libsimdjson-dev \ libsimdjson-dev \
libzstd-dev \ libzstd-dev \
git \ git \
php-pear \
&& apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib; pecl channel-update pecl.php.net && pecl install ssdeep && pecl install rdkafka && pecl install simdjson && pecl install zstd RUN apt-cache search pecl
RUN git clone --recursive --depth=1 https://github.com/kjdev/php-ext-brotli.git && \
cd php-ext-brotli && phpize && ./configure && make && make install
FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build RUN update-alternatives --set php /usr/bin/php7.4
RUN update-alternatives --set php-config /usr/bin/php-config7.4
RUN update-alternatives --set phpize /usr/bin/phpize7.4
RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib
RUN pecl channel-update pecl.php.net && \
pecl install ssdeep && \
pecl install rdkafka && \
pecl install simdjson && \
pecl install zstd
RUN git clone --recursive --depth=1 https://github.com/kjdev/php-ext-brotli.git && \
cd php-ext-brotli && phpize && ./configure && make && make install
FROM php-base as python-build
ENV DEBIAN_FRONTEND noninteractive ENV DEBIAN_FRONTEND noninteractive
ARG CORE_TAG ARG CORE_TAG
ARG CORE_COMMIT ARG CORE_COMMIT
@ -77,8 +106,7 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build
ARG PYPI_CYBOX_VERSION ARG PYPI_CYBOX_VERSION
ARG PYPI_PYMISP_VERSION ARG PYPI_PYMISP_VERSION
RUN apt-get update; apt-get install -y --no-install-recommends \ RUN apt-get install -y --no-install-recommends \
python3-pip \
git \ git \
&& apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
@ -89,7 +117,6 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build
else else
git clone --branch "${CORE_TAG}" --depth 1 https://github.com/MISP/MISP.git /var/www/MISP git clone --branch "${CORE_TAG}" --depth 1 https://github.com/MISP/MISP.git /var/www/MISP
fi fi
cd /var/www/MISP || exit; git submodule update --init --recursive . cd /var/www/MISP || exit; git submodule update --init --recursive .
EOF EOF
@ -124,7 +151,7 @@ EOF
fi fi
done; done;
pip3 wheel --no-cache-dir -w /wheels/ -r /var/www/MISP/requirements.txt pip wheel --no-cache-dir -w /wheels/ -r /var/www/MISP/requirements.txt
# Remove files we do not care for # Remove files we do not care for
rm -r /var/www/MISP/PyMISP rm -r /var/www/MISP/PyMISP
@ -134,13 +161,14 @@ EOF
find /var/www/MISP/.git/* ! -name HEAD -exec rm -rf {} + find /var/www/MISP/.git/* ! -name HEAD -exec rm -rf {} +
EOF EOF
FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim"
FROM php-base
ENV DEBIAN_FRONTEND noninteractive ENV DEBIAN_FRONTEND noninteractive
ARG CORE_TAG ARG CORE_TAG
ARG CORE_COMMIT ARG CORE_COMMIT
ARG PHP_VER ARG PHP_VER
RUN apt-get update; apt-get install -y --no-install-recommends \ RUN apt-get install -y --no-install-recommends \
gettext \ gettext \
procps \ procps \
sudo \ sudo \
@ -150,42 +178,41 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim"
openssl \ openssl \
gpg \ gpg \
gpg-agent \ gpg-agent \
ssdeep \
libfuzzy2 \
mariadb-client \ mariadb-client \
rsync \ rsync \
# Python Requirements
python3 \
python3-setuptools \
python3-pip \
# PHP Requirements # PHP Requirements
php \ php7.4 \
php-apcu \ php7.4-apcu \
php-curl \ php7.4-curl \
php-xml \ php7.4-xml \
php-intl \ php7.4-intl \
php-bcmath \ php7.4-bcmath \
php-mbstring \ php7.4-mbstring \
php-mysql \ php7.4-mysql \
php-redis \ php7.4-redis \
php-gd \ php7.4-gd \
php-fpm \ php7.4-fpm \
php-zip \ php7.4-zip \
php-ldap \ php7.4-ldap \
libmagic1 \
libldap-common \ libldap-common \
librdkafka1 \ librdkafka1 \
libbrotli1 \ libbrotli1 \
libsimdjson5 \ libsimdjson14 \
libzstd1 \ libzstd1 \
ssdeep \
libfuzzy2 \
# Unsure we need these # Unsure we need these
zip unzip \ zip unzip \
# Require for advanced an unattended configuration # Require for advanced an unattended configuration
curl jq \ curl jq \
&& apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
RUN update-alternatives --set php /usr/bin/php7.4
# Install python modules # Install python modules
COPY --from=python-build /wheels /wheels COPY --from=python-build /wheels /wheels
RUN pip3 install --no-cache-dir /wheels/*.whl && rm -rf /wheels RUN pip install --no-cache-dir /wheels/*.whl && rm -rf /wheels
# PHP: install prebuilt libraries, then install the app's PHP deps # PHP: install prebuilt libraries, then install the app's PHP deps
COPY --from=php-build ["/usr/lib/php/${PHP_VER}/ssdeep.so", "/usr/lib/php/${PHP_VER}/rdkafka.so", "/usr/lib/php/${PHP_VER}/brotli.so", "/usr/lib/php/${PHP_VER}/simdjson.so", "/usr/lib/php/${PHP_VER}/zstd.so", "/usr/lib/php/${PHP_VER}/"] COPY --from=php-build ["/usr/lib/php/${PHP_VER}/ssdeep.so", "/usr/lib/php/${PHP_VER}/rdkafka.so", "/usr/lib/php/${PHP_VER}/brotli.so", "/usr/lib/php/${PHP_VER}/simdjson.so", "/usr/lib/php/${PHP_VER}/zstd.so", "/usr/lib/php/${PHP_VER}/"]
@ -208,7 +235,7 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim"
EOF EOF
# nginx # nginx
RUN rm /etc/nginx/sites-enabled/*; mkdir /run/php /etc/nginx/certs RUN rm /etc/nginx/sites-enabled/*; mkdir -p /run/php /etc/nginx/certs
# Make a copy of the file and configuration stores, so we can sync from it # Make a copy of the file and configuration stores, so we can sync from it

View File

@ -1,6 +1,6 @@
ARG DOCKER_HUB_PROXY="" ARG DOCKER_HUB_PROXY=""
FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye" as python-build FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm" as python-build
ENV DEBIAN_FRONTEND noninteractive ENV DEBIAN_FRONTEND noninteractive
ARG MODULES_TAG ARG MODULES_TAG
ARG MODULES_COMMIT ARG MODULES_COMMIT
@ -9,19 +9,17 @@ FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye" as python-build
RUN apt-get update && apt-get install -y --no-install-recommends \ RUN apt-get update && apt-get install -y --no-install-recommends \
cmake \ cmake \
git \ git \
python3-dev \
python3-pip \
python3-wheel \
build-essential \ build-essential \
pkg-config \
libpoppler-cpp-dev \ libpoppler-cpp-dev \
libfuzzy-dev \ libfuzzy-dev \
libffi-dev \
libxml2-dev \
libxslt-dev \
libssl-dev \ libssl-dev \
ninja-build \ ninja-build \
&& apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
RUN mkdir /wheels RUN mkdir /wheels
WORKDIR /srv
RUN <<-EOF RUN <<-EOF
if [ ! -z ${MODULES_COMMIT} ]; then if [ ! -z ${MODULES_COMMIT} ]; then
@ -29,52 +27,60 @@ FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye" as python-build
else else
git clone --branch ${MODULES_TAG} --depth 1 https://github.com/MISP/misp-modules.git /srv/misp-modules git clone --branch ${MODULES_TAG} --depth 1 https://github.com/MISP/misp-modules.git /srv/misp-modules
fi fi
cd /srv/misp-modules
sed -i "s/numpy.*/numpy<2/" REQUIREMENTS
echo "pyeti" >> REQUIREMENTS
echo "greynoise" >> REQUIREMENTS
echo "Google-Search-API" >> REQUIREMENTS
pip3 wheel -r REQUIREMENTS --no-cache-dir -w /wheels/
rm -rf /srv/misp-modules
EOF EOF
WORKDIR /srv/misp-modules
RUN pip install pipenv
COPY files/Pipfile Pipfile
COPY files/Pipfile.lock Pipfile.lock
RUN pipenv requirements > requirements.txt
RUN pip wheel -r requirements.txt --no-cache-dir -w /wheels/
WORKDIR /srv/
RUN rm -rf /srv/misp-modules
RUN <<-EOF RUN <<-EOF
git clone --depth 1 https://github.com/stricaud/faup.git /srv/faup git clone --depth 1 https://github.com/stricaud/faup.git /srv/faup
cd /srv/faup cd /srv/faup
if [ ! -z ${LIBFAUP_COMMIT} ]; then if [ ! -z ${LIBFAUP_COMMIT} ]; then
git checkout ${LIBFAUP_COMMIT} git checkout ${LIBFAUP_COMMIT}
fi fi
cd /srv/faup/build
cmake -G "Ninja" ../
ninja
ninja install
cd /srv/faup/src/lib/bindings/python
pip3 wheel --no-cache-dir --no-dependencies -w /wheels/ .
rm -rf /srv/faup
EOF EOF
FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye" WORKDIR /srv/faup/build
RUN cmake -G "Ninja" ../
RUN ninja
RUN ninja install
WORKDIR /srv/faup/src/lib/bindings/python
RUN pip wheel --no-cache-dir --no-dependencies -w /wheels/ .
WORKDIR /srv/
RUN rm -rf /srv/faup
FROM "${DOCKER_HUB_PROXY}python:3.12-slim-bookworm"
ENV DEBIAN_FRONTEND noninteractive ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \ RUN apt-get update && apt-get install -y --no-install-recommends \
libglib2.0-0 \ libglib2.0-0 \
libzbar0 \
libxrender1 \
libxext6 \
libpoppler-cpp0v5 \ libpoppler-cpp0v5 \
libgl1 \ libgl1 \
libfuzzy2 \
libffi8 \
libxext6 \
libxml2 \
libxslt1.1 \
libzbar0 \
&& apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
COPY --from=python-build /wheels /wheels COPY --from=python-build /wheels /wheels
COPY --from=python-build /usr/local/lib/libfaupl* /usr/local/lib/ COPY --from=python-build /usr/local/lib/libfaupl* /usr/local/lib/
RUN pip3 install --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl; ldconfig && rm -rf /wheels RUN pip install --no-cache-dir --use-deprecated=legacy-resolver /wheels/*.whl; ldconfig && rm -rf /wheels
# Since we compile faup ourselves and lua is not required anymore, we can load our own library # Since we compile faup ourselves and lua is not required anymore, we can load our own library
# and skip the pre-compiled blob to improve compatibility with other architectures like ARM # and skip the pre-compiled blob to improve compatibility with other architectures like ARM
RUN sed -i s/LoadLibrary\(LOAD_LIB\)/LoadLibrary\(\"\\/usr\\/local\\/lib\\/libfaupl.so\"\)/ \ RUN sed -i s/LoadLibrary\(LOAD_LIB\)/LoadLibrary\(\"\\/usr\\/local\\/lib\\/libfaupl.so\"\)/ \
/usr/local/lib/python3.11/site-packages/pyfaup/__init__.py /usr/local/lib/python3.12/site-packages/pyfaup/__init__.py
# Disable (all) warnings raised when using 'future' # Disable (all) warnings raised when using 'future'
RUN sed -i '/import sys/a import warnings\nwarnings.warn = lambda *args, **kwargs: None' \ RUN sed -i '/import sys/a import warnings\nwarnings.warn = lambda *args, **kwargs: None' \

86
modules/files/Pipfile Normal file
View File

@ -0,0 +1,86 @@
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
nose = "*"
codecov = "*"
pytest = "*"
flake8 = "*"
[packages]
numpy = "<2.0.0"
matplotlib = "*"
sigmf = "*"
pysafebrowsing = "*"
dnspython = "*"
requests = { extras = ["security"], version = "*" }
urlarchiver = "*"
passivetotal = "*"
pypdns = "*"
pypssl = "*"
pyeupi = "*"
pymisp = { extras = ["fileobjects,openioc,pdfexport,email,url"], version = "*" }
pyonyphe = { git = "https://github.com/sebdraven/pyonyphe" }
pydnstrails = { git = "https://github.com/sebdraven/pydnstrails" }
pytesseract = "*"
pygeoip = "*"
beautifulsoup4 = "*"
oauth2 = "*"
yara-python = "==3.8.1"
sigmatools = "*"
stix2 = "*"
stix2-patterns = "*"
taxii2-client = "*"
maclookup = "*"
vulners = "*"
blockchain = "*"
reportlab = "*"
pyintel471 = { git = "https://github.com/MISP/PyIntel471.git" }
shodan = "*"
Pillow = ">=8.2.0"
Wand = "*"
SPARQLWrapper = "*"
domaintools_api = "*"
misp-modules = { path = "." }
pybgpranking = { git = "https://github.com/D4-project/BGP-Ranking.git/", subdirectory = "client", ref = "68de39f6c5196f796055c1ac34504054d688aa59" }
pyipasnhistory = { git = "https://github.com/D4-project/IPASN-History.git/", subdirectory = "client", ref = "a2853c39265cecdd0c0d16850bd34621c0551b87" }
backscatter = "*"
pyzbar = "*"
opencv-python = "*"
np = "*"
ODTReader = { git = "https://github.com/cartertemm/ODTReader.git/" }
python-pptx = "*"
python-docx = "*"
ezodf = "*"
pandas = "==1.3.5"
pandas_ods_reader = "==0.1.2"
pdftotext = "*"
lxml = "*"
xlrd = "*"
jbxapi = "*"
geoip2 = "*"
apiosintDS = "*"
assemblyline_client = "*"
vt-graph-api = "*"
trustar = { git = "https://github.com/SteveClement/trustar-python.git" }
markdownify = "==0.5.3"
socialscan = "==1.4"
pycountry = "==22.3.5"
dnsdb2 = "*"
clamd = "*"
aiohttp = ">=3.7.4"
tau-clients = "*"
vt-py = ">=0.7.1"
crowdstrike-falconpy = "0.9.0"
censys = "2.0.9"
mwdblib = "3.4.1"
ndjson = "0.3.1"
Jinja2 = "3.1.2"
mattermostdriver = "7.3.2"
openpyxl = "*"
slack-sdk = "3.27.1"
[requires]
python_version = "3.12"

3383
modules/files/Pipfile.lock generated Normal file

File diff suppressed because it is too large Load Diff