Bump misp and misp-modules

modules/Dockerfile

@@ -1,30 +1,32 @@
 ARG DOCKER_HUB_PROXY=""
 
 FROM "${DOCKER_HUB_PROXY}python:3.9-slim-bullseye" as builder
-ENV DEBIAN_FRONTEND noninteractive
+    
-ARG MODULES_TAG
+    ENV DEBIAN_FRONTEND noninteractive
-ARG MODULES_COMMIT
+    ARG MODULES_TAG
-
+    ARG MODULES_COMMIT
-RUN apt-get update && apt-get install -y --no-install-recommends \
+    
-                cmake \
+    # Install development environment
-                git \
+    RUN apt-get update && apt-get install -y --no-install-recommends \
-                python3-dev \
+        cmake \
-                python3-pip \
+        git \
-                python3-wheel \
+        python3-dev \
-                build-essential \
+        python3-pip \
-                pkg-config \
+        python3-wheel \
-                libpoppler-cpp-dev \
+        build-essential \
-                libfuzzy-dev \
+        pkg-config \
-                libssl-dev \
+        libpoppler-cpp-dev \
-            && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
+        libfuzzy-dev \
-
+        libssl-dev \
-# Build MISP Modules
+        && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
+    
     RUN mkdir /wheel
     WORKDIR /srv
 
     RUN if [ ! -z ${MODULES_COMMIT} ]; then \
         git clone https://github.com/MISP/misp-modules.git /srv/misp-modules && cd /srv/misp-modules && git checkout ${MODULES_COMMIT}; \
         else git clone --branch ${MODULES_TAG} --depth 1 https://github.com/MISP/misp-modules.git /srv/misp-modules; fi
+
     RUN cd /srv/misp-modules; \
         echo "pyeti" >> REQUIREMENTS; \
         echo "git+https://github.com/abenassi/Google-Search-API" >> REQUIREMENTS; \
@@ -41,31 +43,36 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
 
 
 FROM "${DOCKER_HUB_PROXY}python:3.9-slim-bullseye"
-
+    
-RUN apt-get update && apt-get install -y --no-install-recommends \
+    # Install runtime environment
-            libglib2.0-0 \
+    RUN apt-get update && apt-get install -y --no-install-recommends \
-            libzbar0 \
+        libglib2.0-0 \
-            libxrender1 \
+        libzbar0 \
-            libxext6 \
+        libxrender1 \
-            libpoppler-cpp0v5 \
+        libxext6 \
-            libgl1 \
+        libpoppler-cpp0v5 \
+        libgl1 \
         && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
+    
+    COPY --from=builder /wheel /wheel
+    COPY --from=builder /usr/local/lib/libfaupl* /usr/local/lib/
+    RUN pip install --use-deprecated=legacy-resolver /wheel/*.whl; ldconfig
+    
+    # Since we compile faup ourselves and lua is not required anymore, we can load our own library 
+    #   and skip the pre-compiled blob to improve compatibility with other architectures like ARM
+    RUN sed -i s/LoadLibrary\(LOAD_LIB\)/LoadLibrary\(\"\\/usr\\/local\\/lib\\/libfaupl.so\"\)/ \
+        /usr/local/lib/python3.9/site-packages/pyfaup/__init__.py
+    
+    # Remove double logging configuration from 'apiosintDS'
+    RUN sed -i "/logging.basicConfig/d" \
+            /usr/local/lib/python3.9/site-packages/apiosintDS/apiosintDS.py; \
+        sed -i "/logging.basicConfig/d" \
+            /usr/local/lib/python3.9/site-packages/apiosintDS/modules/dosearch.py; \
+        sed -i "/logging.basicConfig/d" \
+            /usr/local/lib/python3.9/site-packages/apiosintDS/modules/listutils.py
 
-COPY --from=builder /wheel /wheel
+    # Disable (all) warnings raised when using 'future'
-COPY --from=builder /usr/local/lib/libfaupl* /usr/local/lib/
+    RUN sed -i '/import sys/a import warnings\nwarnings.warn = lambda *args, **kwargs: None' \
-RUN pip install --use-deprecated=legacy-resolver /wheel/*.whl; ldconfig
+            /usr/local/bin/misp-modules
 
-# Since we compile faup ourselves and lua is not required anymore, we can load our own library 
+    ENTRYPOINT [ "/usr/local/bin/misp-modules", "-l", "0.0.0.0"]
-#   and skip the pre-compiled blob to improve compatibility with other architectures like arm
-RUN sed -i s/LoadLibrary\(LOAD_LIB\)/LoadLibrary\(\"\\/usr\\/local\\/lib\\/libfaupl.so\"\)/ \
-    /usr/local/lib/python3.9/site-packages/pyfaup/__init__.py
-
-# Remove double logging
-RUN sed -i "/logging.basicConfig/d" \
-        /usr/local/lib/python3.9/site-packages/apiosintDS/apiosintDS.py; \
-    sed -i "/logging.basicConfig/d" \
-        /usr/local/lib/python3.9/site-packages/apiosintDS/modules/dosearch.py; \
-    sed -i "/logging.basicConfig/d" \
-        /usr/local/lib/python3.9/site-packages/apiosintDS/modules/listutils.py
-
-ENTRYPOINT [ "/usr/local/bin/misp-modules", "-l", "0.0.0.0"]

server/Dockerfile

@@ -73,6 +73,8 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build
         echo "-e ." > requirements.txt; pip3 wheel -r requirements.txt --no-cache-dir -w /wheels/
 
     # install PyMISP
+    # note: when using poetry the missing wheels are installed at installation time
+    #   which is something we might need to fix
     RUN curl -sSL https://install.python-poetry.org | POETRY_HOME=/usr/local/ python3 -
     RUN git clone --depth 1 https://github.com/MISP/PyMISP.git; \
         cd PyMISP || exit; poetry build -f wheel; cp dist/*.whl /wheels/
@@ -90,14 +92,13 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build
 
 
 FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim"
-ENV DEBIAN_FRONTEND noninteractive
+    ENV DEBIAN_FRONTEND noninteractive
-ARG MISP_TAG
+    ARG MISP_TAG
-ARG MISP_COMMIT
+    ARG MISP_COMMIT
-ARG PHP_VER
+    ARG PHP_VER
 
-# OS Packages
+    # OS Packages
     RUN apt-get update; apt-get install -y --no-install-recommends \
-        # Requirements:
         procps \
         sudo \
         nginx \
@@ -136,7 +137,6 @@ ARG PHP_VER
         curl jq \
         && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
-# MISP code
     # Download MISP using git in the /var/www/ directory.
     RUN if [ ! -z ${MISP_COMMIT} ]; then \
      git clone https://github.com/MISP/MISP.git /var/www/MISP && cd /var/www/MISP && git checkout ${MISP_COMMIT}; \
@@ -145,12 +145,11 @@ ARG PHP_VER
         # Remove some old and broken links that pollute the log files
         rm -rf /var/www/MISP/INSTALL/old
 
-# Python Modules
+    # Python Modules
     COPY --from=python-build /wheels /wheels
     RUN pip3 install --no-cache-dir /wheels/*.whl && rm -rf /wheels
 
-# PHP
+    # PHP: install prebuilt libraries, latest composer, then install the app's PHP deps
-    # Install prebuilt libraries, latest composer, then install the app's PHP deps
     COPY --from=php-build /usr/lib/php/${PHP_VER}/ssdeep.so /usr/lib/php/${PHP_VER}/ssdeep.so
     COPY --from=php-build /usr/lib/php/${PHP_VER}/rdkafka.so /usr/lib/php/${PHP_VER}/rdkafka.so
     COPY --from=php-build /usr/lib/php/${PHP_VER}/brotli.so /usr/lib/php/${PHP_VER}/brotli.so
@@ -165,25 +164,24 @@ ARG PHP_VER
     RUN for dir in /etc/php/*; do echo "extension=simdjson.so" > "$dir/mods-available/simdjson.ini"; done; phpenmod simdjson
     RUN phpenmod redis
 
-# nginx
+    # nginx
     RUN rm /etc/nginx/sites-enabled/*; mkdir /run/php /etc/nginx/certs
     COPY files/etc/nginx/misp /etc/nginx/sites-available/misp
     COPY files/etc/nginx/misp80 /etc/nginx/sites-available/misp80
 
-# Make a copy of the file store, so we can sync from it
+    # Make a copy of the file store, so we can sync from it
     RUN cp -R /var/www/MISP/app/files /var/www/MISP/app/files.dist
-
+    # Make a copy of the configurations, so we can sync from it
-# Make a copy of the configurations, so we can sync from it
     RUN cp -R /var/www/MISP/app/Config /var/www/MISP/app/Config.dist
 
-# The spirit of the upstrem dockerization is to keep user and group aligned in terms of permissions
+    # The spirit of the upstrem dockerization is to keep user and group aligned in terms of permissions
     RUN find /var/www/MISP \( ! -user www-data -or ! -group www-data \) -exec chown www-data:www-data {} +
-# Files are also executable and read only, because we have some rogue scripts like 'cake' and we can not do a full inventory
+    # Files are also executable and read only, because we have some rogue scripts like 'cake' and we can not do a full inventory
     RUN find /var/www/MISP -not -perm 550 -type f -exec chmod 0550 {} +
-# Directories are also writable, because there seems to be a requirement to add new files every once in a while
+    # Directories are also writable, because there seems to be a requirement to add new files every once in a while
     RUN find /var/www/MISP -not -perm 770 -type d -exec chmod 0770 {} +
 
-# Entrypoints
+    # Entrypoints
     COPY files/etc/supervisor/supervisor.conf /etc/supervisor/conf.d/10-supervisor.conf
     COPY files/etc/supervisor/workers.conf /etc/supervisor/conf.d/50-workers.conf
     COPY files/var/www/html/index.php /var/www/html/index.php
@@ -195,5 +193,5 @@ ARG PHP_VER
     COPY files/entrypoint.sh /
     ENTRYPOINT [ "/entrypoint.sh" ]
 
-# Change Workdirectory
+    # Change Workdirectory
     WORKDIR /var/www/MISP

server/files/configure_misp.sh

@@ -117,7 +117,7 @@ init_user() {
         sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting -q "Security.password_policy_complexity" ${PASSWORD_POLICY}
         sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting -q "Security.password_policy_length" ${PASSWORD_LENGTH}
     else
-        echo "... leaving admin password as-is"
+        echo "... setting adming password skipped"
     fi
     echo 'UPDATE misp.users SET change_pw = 0 WHERE id = 1;' | ${MYSQLCMD}
 }
@@ -199,15 +199,14 @@ create_sync_servers() {
         if [[ -z "$ORG_ID" ]]; then
             # Add remote organization if missing
             echo "... adding missing organization ${UUID}"
-            add_organization ${HOSTNAME} ${ADMIN_KEY} ${NAME} false ${UUID}
+            add_organization ${HOSTNAME} ${ADMIN_KEY} ${NAME} false ${UUID} > /dev/null
             ORG_ID=$(get_organization ${HOSTNAME} ${ADMIN_KEY} ${UUID})
         fi
 
         # Add sync server
         echo "... adding new sync server ${NAME} with organization id ${ORG_ID}"
         JSON_DATA=$(echo "${!DATA}" | jq --arg org_id ${ORG_ID} 'del(.remote_org_uuid) | . + {remote_org_id: $org_id}')
-        echo "... adding new sync server ${JSON_DATA}"
+        add_server ${HOSTNAME} ${ADMIN_KEY} "$JSON_DATA" > /dev/null
-        add_server ${HOSTNAME} ${ADMIN_KEY} "$JSON_DATA"
     done   
 }
 

template.env

@@ -1,5 +1,5 @@
-MISP_TAG=v2.4.170
+MISP_TAG=v2.4.171
-MODULES_TAG=v2.4.170
+MODULES_TAG=v2.4.171
 PHP_VER=20190902
 # MISP_COMMIT takes precedence over MISP_TAG
 # MISP_COMMIT=c56d537