From 5c78c3492cc039d36547bb758a48c39846e1d2f0 Mon Sep 17 00:00:00 2001 From: Stefano Ortolani Date: Tue, 16 May 2023 15:26:20 +0100 Subject: [PATCH] Bump misp and misp-modules --- modules/Dockerfile | 93 ++++++++++++++++++---------------- server/Dockerfile | 36 +++++++------ server/files/configure_misp.sh | 7 ++- template.env | 4 +- 4 files changed, 72 insertions(+), 68 deletions(-) diff --git a/modules/Dockerfile b/modules/Dockerfile index 2fbcf95..702e84a 100644 --- a/modules/Dockerfile +++ b/modules/Dockerfile @@ -1,30 +1,32 @@ ARG DOCKER_HUB_PROXY="" FROM "${DOCKER_HUB_PROXY}python:3.9-slim-bullseye" as builder -ENV DEBIAN_FRONTEND noninteractive -ARG MODULES_TAG -ARG MODULES_COMMIT - -RUN apt-get update && apt-get install -y --no-install-recommends \ - cmake \ - git \ - python3-dev \ - python3-pip \ - python3-wheel \ - build-essential \ - pkg-config \ - libpoppler-cpp-dev \ - libfuzzy-dev \ - libssl-dev \ - && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* - -# Build MISP Modules + + ENV DEBIAN_FRONTEND noninteractive + ARG MODULES_TAG + ARG MODULES_COMMIT + + # Install development environment + RUN apt-get update && apt-get install -y --no-install-recommends \ + cmake \ + git \ + python3-dev \ + python3-pip \ + python3-wheel \ + build-essential \ + pkg-config \ + libpoppler-cpp-dev \ + libfuzzy-dev \ + libssl-dev \ + && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* + RUN mkdir /wheel WORKDIR /srv RUN if [ ! -z ${MODULES_COMMIT} ]; then \ git clone https://github.com/MISP/misp-modules.git /srv/misp-modules && cd /srv/misp-modules && git checkout ${MODULES_COMMIT}; \ else git clone --branch ${MODULES_TAG} --depth 1 https://github.com/MISP/misp-modules.git /srv/misp-modules; fi + RUN cd /srv/misp-modules; \ echo "pyeti" >> REQUIREMENTS; \ echo "git+https://github.com/abenassi/Google-Search-API" >> REQUIREMENTS; \ @@ -41,31 +43,36 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ FROM "${DOCKER_HUB_PROXY}python:3.9-slim-bullseye" - -RUN apt-get update && apt-get install -y --no-install-recommends \ - libglib2.0-0 \ - libzbar0 \ - libxrender1 \ - libxext6 \ - libpoppler-cpp0v5 \ - libgl1 \ + + # Install runtime environment + RUN apt-get update && apt-get install -y --no-install-recommends \ + libglib2.0-0 \ + libzbar0 \ + libxrender1 \ + libxext6 \ + libpoppler-cpp0v5 \ + libgl1 \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* + + COPY --from=builder /wheel /wheel + COPY --from=builder /usr/local/lib/libfaupl* /usr/local/lib/ + RUN pip install --use-deprecated=legacy-resolver /wheel/*.whl; ldconfig + + # Since we compile faup ourselves and lua is not required anymore, we can load our own library + # and skip the pre-compiled blob to improve compatibility with other architectures like ARM + RUN sed -i s/LoadLibrary\(LOAD_LIB\)/LoadLibrary\(\"\\/usr\\/local\\/lib\\/libfaupl.so\"\)/ \ + /usr/local/lib/python3.9/site-packages/pyfaup/__init__.py + + # Remove double logging configuration from 'apiosintDS' + RUN sed -i "/logging.basicConfig/d" \ + /usr/local/lib/python3.9/site-packages/apiosintDS/apiosintDS.py; \ + sed -i "/logging.basicConfig/d" \ + /usr/local/lib/python3.9/site-packages/apiosintDS/modules/dosearch.py; \ + sed -i "/logging.basicConfig/d" \ + /usr/local/lib/python3.9/site-packages/apiosintDS/modules/listutils.py -COPY --from=builder /wheel /wheel -COPY --from=builder /usr/local/lib/libfaupl* /usr/local/lib/ -RUN pip install --use-deprecated=legacy-resolver /wheel/*.whl; ldconfig + # Disable (all) warnings raised when using 'future' + RUN sed -i '/import sys/a import warnings\nwarnings.warn = lambda *args, **kwargs: None' \ + /usr/local/bin/misp-modules -# Since we compile faup ourselves and lua is not required anymore, we can load our own library -# and skip the pre-compiled blob to improve compatibility with other architectures like arm -RUN sed -i s/LoadLibrary\(LOAD_LIB\)/LoadLibrary\(\"\\/usr\\/local\\/lib\\/libfaupl.so\"\)/ \ - /usr/local/lib/python3.9/site-packages/pyfaup/__init__.py - -# Remove double logging -RUN sed -i "/logging.basicConfig/d" \ - /usr/local/lib/python3.9/site-packages/apiosintDS/apiosintDS.py; \ - sed -i "/logging.basicConfig/d" \ - /usr/local/lib/python3.9/site-packages/apiosintDS/modules/dosearch.py; \ - sed -i "/logging.basicConfig/d" \ - /usr/local/lib/python3.9/site-packages/apiosintDS/modules/listutils.py - -ENTRYPOINT [ "/usr/local/bin/misp-modules", "-l", "0.0.0.0"] + ENTRYPOINT [ "/usr/local/bin/misp-modules", "-l", "0.0.0.0"] diff --git a/server/Dockerfile b/server/Dockerfile index 839e2ca..f158a0d 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -73,6 +73,8 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build echo "-e ." > requirements.txt; pip3 wheel -r requirements.txt --no-cache-dir -w /wheels/ # install PyMISP + # note: when using poetry the missing wheels are installed at installation time + # which is something we might need to fix RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=/usr/local/ python3 - RUN git clone --depth 1 https://github.com/MISP/PyMISP.git; \ cd PyMISP || exit; poetry build -f wheel; cp dist/*.whl /wheels/ @@ -90,14 +92,13 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" -ENV DEBIAN_FRONTEND noninteractive -ARG MISP_TAG -ARG MISP_COMMIT -ARG PHP_VER + ENV DEBIAN_FRONTEND noninteractive + ARG MISP_TAG + ARG MISP_COMMIT + ARG PHP_VER -# OS Packages + # OS Packages RUN apt-get update; apt-get install -y --no-install-recommends \ - # Requirements: procps \ sudo \ nginx \ @@ -136,7 +137,6 @@ ARG PHP_VER curl jq \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* -# MISP code # Download MISP using git in the /var/www/ directory. RUN if [ ! -z ${MISP_COMMIT} ]; then \ git clone https://github.com/MISP/MISP.git /var/www/MISP && cd /var/www/MISP && git checkout ${MISP_COMMIT}; \ @@ -145,12 +145,11 @@ ARG PHP_VER # Remove some old and broken links that pollute the log files rm -rf /var/www/MISP/INSTALL/old -# Python Modules + # Python Modules COPY --from=python-build /wheels /wheels RUN pip3 install --no-cache-dir /wheels/*.whl && rm -rf /wheels -# PHP - # Install prebuilt libraries, latest composer, then install the app's PHP deps + # PHP: install prebuilt libraries, latest composer, then install the app's PHP deps COPY --from=php-build /usr/lib/php/${PHP_VER}/ssdeep.so /usr/lib/php/${PHP_VER}/ssdeep.so COPY --from=php-build /usr/lib/php/${PHP_VER}/rdkafka.so /usr/lib/php/${PHP_VER}/rdkafka.so COPY --from=php-build /usr/lib/php/${PHP_VER}/brotli.so /usr/lib/php/${PHP_VER}/brotli.so @@ -165,25 +164,24 @@ ARG PHP_VER RUN for dir in /etc/php/*; do echo "extension=simdjson.so" > "$dir/mods-available/simdjson.ini"; done; phpenmod simdjson RUN phpenmod redis -# nginx + # nginx RUN rm /etc/nginx/sites-enabled/*; mkdir /run/php /etc/nginx/certs COPY files/etc/nginx/misp /etc/nginx/sites-available/misp COPY files/etc/nginx/misp80 /etc/nginx/sites-available/misp80 -# Make a copy of the file store, so we can sync from it + # Make a copy of the file store, so we can sync from it RUN cp -R /var/www/MISP/app/files /var/www/MISP/app/files.dist - -# Make a copy of the configurations, so we can sync from it + # Make a copy of the configurations, so we can sync from it RUN cp -R /var/www/MISP/app/Config /var/www/MISP/app/Config.dist -# The spirit of the upstrem dockerization is to keep user and group aligned in terms of permissions + # The spirit of the upstrem dockerization is to keep user and group aligned in terms of permissions RUN find /var/www/MISP \( ! -user www-data -or ! -group www-data \) -exec chown www-data:www-data {} + -# Files are also executable and read only, because we have some rogue scripts like 'cake' and we can not do a full inventory + # Files are also executable and read only, because we have some rogue scripts like 'cake' and we can not do a full inventory RUN find /var/www/MISP -not -perm 550 -type f -exec chmod 0550 {} + -# Directories are also writable, because there seems to be a requirement to add new files every once in a while + # Directories are also writable, because there seems to be a requirement to add new files every once in a while RUN find /var/www/MISP -not -perm 770 -type d -exec chmod 0770 {} + -# Entrypoints + # Entrypoints COPY files/etc/supervisor/supervisor.conf /etc/supervisor/conf.d/10-supervisor.conf COPY files/etc/supervisor/workers.conf /etc/supervisor/conf.d/50-workers.conf COPY files/var/www/html/index.php /var/www/html/index.php @@ -195,5 +193,5 @@ ARG PHP_VER COPY files/entrypoint.sh / ENTRYPOINT [ "/entrypoint.sh" ] -# Change Workdirectory + # Change Workdirectory WORKDIR /var/www/MISP diff --git a/server/files/configure_misp.sh b/server/files/configure_misp.sh index 72aa7b0..8d168c2 100755 --- a/server/files/configure_misp.sh +++ b/server/files/configure_misp.sh @@ -117,7 +117,7 @@ init_user() { sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting -q "Security.password_policy_complexity" ${PASSWORD_POLICY} sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting -q "Security.password_policy_length" ${PASSWORD_LENGTH} else - echo "... leaving admin password as-is" + echo "... setting adming password skipped" fi echo 'UPDATE misp.users SET change_pw = 0 WHERE id = 1;' | ${MYSQLCMD} } @@ -199,15 +199,14 @@ create_sync_servers() { if [[ -z "$ORG_ID" ]]; then # Add remote organization if missing echo "... adding missing organization ${UUID}" - add_organization ${HOSTNAME} ${ADMIN_KEY} ${NAME} false ${UUID} + add_organization ${HOSTNAME} ${ADMIN_KEY} ${NAME} false ${UUID} > /dev/null ORG_ID=$(get_organization ${HOSTNAME} ${ADMIN_KEY} ${UUID}) fi # Add sync server echo "... adding new sync server ${NAME} with organization id ${ORG_ID}" JSON_DATA=$(echo "${!DATA}" | jq --arg org_id ${ORG_ID} 'del(.remote_org_uuid) | . + {remote_org_id: $org_id}') - echo "... adding new sync server ${JSON_DATA}" - add_server ${HOSTNAME} ${ADMIN_KEY} "$JSON_DATA" + add_server ${HOSTNAME} ${ADMIN_KEY} "$JSON_DATA" > /dev/null done } diff --git a/template.env b/template.env index 12f323e..508053d 100644 --- a/template.env +++ b/template.env @@ -1,5 +1,5 @@ -MISP_TAG=v2.4.170 -MODULES_TAG=v2.4.170 +MISP_TAG=v2.4.171 +MODULES_TAG=v2.4.171 PHP_VER=20190902 # MISP_COMMIT takes precedence over MISP_TAG # MISP_COMMIT=c56d537