Fix build arguments and pin pymisp (#28)

.github/workflows/test-build-latest.yml

@@ -28,4 +28,4 @@ jobs:
           # Create the environment
           sed -e '/^[[:space:]]*$/d' -e '/[#@]/d' -e 's/\"//g' -e 's/\(^[^=]*\)=\(.*\)/\1="\2"/' template.env > env.hcl
           # Build and Push
-          DOCKER_IMG_TAG=$DOCKER_IMG_TAG docker buildx bake -f docker-bake.hcl -f env.hcl
+          DOCKER_IMG_TAG=$DOCKER_IMG_TAG DOCKER_USERNAME=local docker buildx bake -f docker-bake.hcl -f env.hcl

README.md

@@ -57,8 +57,6 @@ GitHub builds the images automatically and pushes them to [Docker hub](https://h
     -   User: `admin@admin.test`
     -   Password: `admin`
 
--   Profit
-
 ### Using the image for development
 
 Pull the entire repository, you can build the images using `docker-compose build`
@@ -96,16 +94,8 @@ If you are interested in building the project from scratch - `git clone` or down
 
 ## Image file sizes
 
--   Core server(Saved: 2.5GB)
-    -   Original Image: 3.17GB
-    -   First attempt: 2.24GB
-    -   Remove chown: 1.56GB
-    -   PreBuild python modules, and only pull submodules we need: 800MB
-    -   PreBuild PHP modules: 664MB
-
--   Modules (Saved: 640MB)
-    -   Original: 1.36GB
-    -   Pre-build modules: 750MB
+-   Core server: 260MB
+-   Modules: 470MB
 
 ### Configuration
 

docker-bake.hcl

@@ -34,6 +34,10 @@ variable "PYPI_CYBOX_VERSION" {
   default = ""
 }
 
+variable "PYPI_PYMISP_VERSION" {
+  default = ""
+}
+
 variable "DOCKER_USERNAME" {
   default = null
 }
@@ -101,6 +105,7 @@ target "misp" {
     "PYPI_MAEC_VERSION": "${PYPI_MAEC_VERSION}",
     "PYPI_MIXBOX_VERSION": "${PYPI_MIXBOX_VERSION}",
     "PYPI_CYBOX_VERSION": "${PYPI_CYBOX_VERSION}",
+    "PYPI_PYMISP_VERSION": "${PYPI_PYMISP_VERSION}",
   }
   platforms = "${PLATFORMS}"
 }

docker-compose.yml

@@ -36,6 +36,15 @@ services:
           - MISP_TAG=${MISP_TAG}
           - MISP_COMMIT=${MISP_COMMIT}
           - PHP_VER=${PHP_VER}
+          - PYPI_REDIS_VERSION=${PYPI_REDIS_VERSION}
+          - PYPI_LIEF_VERSION=${PYPI_LIEF_VERSION}
+          - PYPI_PYDEEP2_VERSION=${PYPI_PYDEEP2_VERSION}
+          - PYPI_PYTHON_MAGIC_VERSION=${PYPI_PYTHON_MAGIC_VERSION}
+          - PYPI_MISP_LIB_STIX2_VERSION=${PYPI_MISP_LIB_STIX2_VERSION}
+          - PYPI_MAEC_VERSION=${PYPI_MAEC_VERSION}
+          - PYPI_MIXBOX_VERSION=${PYPI_MIXBOX_VERSION}
+          - PYPI_CYBOX_VERSION=${PYPI_CYBOX_VERSION}
+          - PYPI_PYMISP_VERSION=${PYPI_PYMISP_VERSION}
     depends_on:
       - redis
       - db
@@ -78,6 +87,7 @@ services:
       args:
         - MODULES_TAG=${MODULES_TAG}
         - MODULES_COMMIT=${MODULES_COMMIT}
+        - LIBFAUP_COMMIT=${LIBFAUP_COMMIT}
     environment:
       - "REDIS_BACKEND=redis"
     depends_on:

modules/Dockerfile

@@ -5,6 +5,7 @@ FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye" as python-build
     ARG MODULES_TAG
     ARG MODULES_COMMIT
     ARG LIBFAUP_COMMIT
+
     RUN apt-get update && apt-get install -y --no-install-recommends \
         cmake \
         git \
@@ -59,6 +60,7 @@ EOF
 
 FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye"
     ENV DEBIAN_FRONTEND noninteractive
+
     RUN apt-get update && apt-get install -y --no-install-recommends \
         libglib2.0-0 \
         libzbar0 \

server/Dockerfile

@@ -1,11 +1,11 @@
 ARG DOCKER_HUB_PROXY=""
 
 FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build
-
     ENV DEBIAN_FRONTEND noninteractive
     ENV COMPOSER_ALLOW_SUPERUSER 1
     ARG MISP_TAG
     ARG MISP_COMMIT
+
     RUN apt-get update; apt-get install -y --no-install-recommends \
         ca-certificates \
         php \
@@ -42,6 +42,7 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build
 FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as php-build
     ENV DEBIAN_FRONTEND noninteractive
     ENV TZ Etc/UTC
+
     RUN apt-get update; apt-get install -y --no-install-recommends \
         gcc \
         g++ \
@@ -56,16 +57,30 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as php-build
         libsimdjson-dev \
         git \
         && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
+
     RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib; pecl channel-update pecl.php.net && pecl install ssdeep && pecl install rdkafka && pecl install simdjson
     RUN git clone --recursive --depth=1 https://github.com/kjdev/php-ext-brotli.git && \
         cd php-ext-brotli && phpize && ./configure && make && make install
 
 FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build
     ENV DEBIAN_FRONTEND noninteractive   
-
-    RUN apt-get update; apt-get install -y --no-install-recommends python3-pip git && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
     ARG MISP_TAG
     ARG MISP_COMMIT
+    ARG PYPI_REDIS_VERSION
+    ARG PYPI_LIEF_VERSION
+    ARG PYPI_PYDEEP2_VERSION
+    ARG PYPI_PYTHON_MAGIC_VERSION
+    ARG PYPI_MISP_LIB_STIX2_VERSION
+    ARG PYPI_MAEC_VERSION
+    ARG PYPI_MIXBOX_VERSION
+    ARG PYPI_CYBOX_VERSION
+    ARG PYPI_PYMISP_VERSION
+
+    RUN apt-get update; apt-get install -y --no-install-recommends \
+        python3-pip \
+        git \
+        && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
+    
     # Download MISP using git in the /var/www/ directory. Remove unnecessary items.
     RUN <<-EOF
         if [ ! -z "${MISP_COMMIT}" ]; then
@@ -75,15 +90,6 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build
         fi
 
         cd /var/www/MISP || exit; git submodule update --init --recursive .
-
-        # Until MISP project specifies its required PyMISP version, we grab this from the submodule
-        pymispversion=$(head -n1 PyMISP/pymisp/__init__.py | awk '{print $3}' | tr -d \')
-
-        if [ -n "$pymispversion" ]; then
-            sed -i "s/pymisp$/pymisp==$pymispversion/" requirements.txt
-            cat requirements.txt
-        fi
-        # End of pymisp version fixing hack
 EOF
 
     RUN <<-EOF
@@ -104,7 +110,7 @@ EOF
         # 2. If missing, add it (with optional version from env (defaults to empty string))
         # 3. If present, replace with our specified version if it exists, otherwise leave
         #    the upstream version alone.
-        set -- "redis" "lief" "pydeep2" "python-magic" "misp-lib-stix2" "maec" "mixbox" "cybox"
+        set -- "redis" "lief" "pydeep2" "python-magic" "misp-lib-stix2" "maec" "mixbox" "cybox" "pymisp"
         for mod in "$@"; do
             mod_version_var=$(echo "PYPI_${mod}_VERSION" | tr '[:lower:]' '[:upper:]' | tr '-' '_')
             mod_version=$(eval "echo \"\$$mod_version_var\"")
@@ -134,11 +140,11 @@ EOF
 EOF
 
 FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim"
-
     ENV DEBIAN_FRONTEND noninteractive
     ARG MISP_TAG
     ARG MISP_COMMIT
     ARG PHP_VER
+
     RUN apt-get update; apt-get install -y --no-install-recommends \
         procps \
         sudo \

template.env

@@ -11,6 +11,7 @@ PYPI_MISP_LIB_STIX2_VERSION="==3.0.*"
 PYPI_MAEC_VERSION="==4.1.*"
 PYPI_MIXBOX_VERSION="==1.0.*"
 PYPI_CYBOX_VERSION="==2.1.*"
+PYPI_PYMISP_VERSION="==2.4.176"
 
 # MISP_COMMIT takes precedence over MISP_TAG
 # MISP_COMMIT=c56d537