diff --git a/.github/workflows/test-build-latest.yml b/.github/workflows/test-build-latest.yml index ae7d8fe..c5b1b45 100644 --- a/.github/workflows/test-build-latest.yml +++ b/.github/workflows/test-build-latest.yml @@ -28,4 +28,4 @@ jobs: # Create the environment sed -e '/^[[:space:]]*$/d' -e '/[#@]/d' -e 's/\"//g' -e 's/\(^[^=]*\)=\(.*\)/\1="\2"/' template.env > env.hcl # Build and Push - DOCKER_IMG_TAG=$DOCKER_IMG_TAG docker buildx bake -f docker-bake.hcl -f env.hcl + DOCKER_IMG_TAG=$DOCKER_IMG_TAG DOCKER_USERNAME=local docker buildx bake -f docker-bake.hcl -f env.hcl diff --git a/README.md b/README.md index f9c45f9..3575980 100644 --- a/README.md +++ b/README.md @@ -57,8 +57,6 @@ GitHub builds the images automatically and pushes them to [Docker hub](https://h - User: `admin@admin.test` - Password: `admin` -- Profit - ### Using the image for development Pull the entire repository, you can build the images using `docker-compose build` @@ -96,16 +94,8 @@ If you are interested in building the project from scratch - `git clone` or down ## Image file sizes -- Core server(Saved: 2.5GB) - - Original Image: 3.17GB - - First attempt: 2.24GB - - Remove chown: 1.56GB - - PreBuild python modules, and only pull submodules we need: 800MB - - PreBuild PHP modules: 664MB - -- Modules (Saved: 640MB) - - Original: 1.36GB - - Pre-build modules: 750MB +- Core server: 260MB +- Modules: 470MB ### Configuration diff --git a/docker-bake.hcl b/docker-bake.hcl index a760827..1a26f4c 100644 --- a/docker-bake.hcl +++ b/docker-bake.hcl @@ -34,6 +34,10 @@ variable "PYPI_CYBOX_VERSION" { default = "" } +variable "PYPI_PYMISP_VERSION" { + default = "" +} + variable "DOCKER_USERNAME" { default = null } @@ -101,6 +105,7 @@ target "misp" { "PYPI_MAEC_VERSION": "${PYPI_MAEC_VERSION}", "PYPI_MIXBOX_VERSION": "${PYPI_MIXBOX_VERSION}", "PYPI_CYBOX_VERSION": "${PYPI_CYBOX_VERSION}", + "PYPI_PYMISP_VERSION": "${PYPI_PYMISP_VERSION}", } platforms = "${PLATFORMS}" } diff --git a/docker-compose.yml b/docker-compose.yml index b5b2c42..d44817c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -36,6 +36,15 @@ services: - MISP_TAG=${MISP_TAG} - MISP_COMMIT=${MISP_COMMIT} - PHP_VER=${PHP_VER} + - PYPI_REDIS_VERSION=${PYPI_REDIS_VERSION} + - PYPI_LIEF_VERSION=${PYPI_LIEF_VERSION} + - PYPI_PYDEEP2_VERSION=${PYPI_PYDEEP2_VERSION} + - PYPI_PYTHON_MAGIC_VERSION=${PYPI_PYTHON_MAGIC_VERSION} + - PYPI_MISP_LIB_STIX2_VERSION=${PYPI_MISP_LIB_STIX2_VERSION} + - PYPI_MAEC_VERSION=${PYPI_MAEC_VERSION} + - PYPI_MIXBOX_VERSION=${PYPI_MIXBOX_VERSION} + - PYPI_CYBOX_VERSION=${PYPI_CYBOX_VERSION} + - PYPI_PYMISP_VERSION=${PYPI_PYMISP_VERSION} depends_on: - redis - db @@ -78,6 +87,7 @@ services: args: - MODULES_TAG=${MODULES_TAG} - MODULES_COMMIT=${MODULES_COMMIT} + - LIBFAUP_COMMIT=${LIBFAUP_COMMIT} environment: - "REDIS_BACKEND=redis" depends_on: diff --git a/modules/Dockerfile b/modules/Dockerfile index 5aa8fe3..829da39 100644 --- a/modules/Dockerfile +++ b/modules/Dockerfile @@ -5,6 +5,7 @@ FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye" as python-build ARG MODULES_TAG ARG MODULES_COMMIT ARG LIBFAUP_COMMIT + RUN apt-get update && apt-get install -y --no-install-recommends \ cmake \ git \ @@ -59,6 +60,7 @@ EOF FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye" ENV DEBIAN_FRONTEND noninteractive + RUN apt-get update && apt-get install -y --no-install-recommends \ libglib2.0-0 \ libzbar0 \ diff --git a/server/Dockerfile b/server/Dockerfile index 8098d59..52d2864 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -1,11 +1,11 @@ ARG DOCKER_HUB_PROXY="" FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build - ENV DEBIAN_FRONTEND noninteractive ENV COMPOSER_ALLOW_SUPERUSER 1 ARG MISP_TAG ARG MISP_COMMIT + RUN apt-get update; apt-get install -y --no-install-recommends \ ca-certificates \ php \ @@ -22,7 +22,7 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build php-zip \ unzip \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* - + WORKDIR /tmp ADD https://raw.githubusercontent.com/MISP/MISP/${MISP_COMMIT:-${MISP_TAG}}/app/composer.json /tmp COPY --from=composer:latest /usr/bin/composer /usr/bin/composer @@ -42,6 +42,7 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as php-build ENV DEBIAN_FRONTEND noninteractive ENV TZ Etc/UTC + RUN apt-get update; apt-get install -y --no-install-recommends \ gcc \ g++ \ @@ -56,16 +57,30 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as php-build libsimdjson-dev \ git \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* + RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib; pecl channel-update pecl.php.net && pecl install ssdeep && pecl install rdkafka && pecl install simdjson RUN git clone --recursive --depth=1 https://github.com/kjdev/php-ext-brotli.git && \ cd php-ext-brotli && phpize && ./configure && make && make install FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build - ENV DEBIAN_FRONTEND noninteractive - - RUN apt-get update; apt-get install -y --no-install-recommends python3-pip git && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* + ENV DEBIAN_FRONTEND noninteractive ARG MISP_TAG ARG MISP_COMMIT + ARG PYPI_REDIS_VERSION + ARG PYPI_LIEF_VERSION + ARG PYPI_PYDEEP2_VERSION + ARG PYPI_PYTHON_MAGIC_VERSION + ARG PYPI_MISP_LIB_STIX2_VERSION + ARG PYPI_MAEC_VERSION + ARG PYPI_MIXBOX_VERSION + ARG PYPI_CYBOX_VERSION + ARG PYPI_PYMISP_VERSION + + RUN apt-get update; apt-get install -y --no-install-recommends \ + python3-pip \ + git \ + && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* + # Download MISP using git in the /var/www/ directory. Remove unnecessary items. RUN <<-EOF if [ ! -z "${MISP_COMMIT}" ]; then @@ -75,15 +90,6 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build fi cd /var/www/MISP || exit; git submodule update --init --recursive . - - # Until MISP project specifies its required PyMISP version, we grab this from the submodule - pymispversion=$(head -n1 PyMISP/pymisp/__init__.py | awk '{print $3}' | tr -d \') - - if [ -n "$pymispversion" ]; then - sed -i "s/pymisp$/pymisp==$pymispversion/" requirements.txt - cat requirements.txt - fi - # End of pymisp version fixing hack EOF RUN <<-EOF @@ -104,7 +110,7 @@ EOF # 2. If missing, add it (with optional version from env (defaults to empty string)) # 3. If present, replace with our specified version if it exists, otherwise leave # the upstream version alone. - set -- "redis" "lief" "pydeep2" "python-magic" "misp-lib-stix2" "maec" "mixbox" "cybox" + set -- "redis" "lief" "pydeep2" "python-magic" "misp-lib-stix2" "maec" "mixbox" "cybox" "pymisp" for mod in "$@"; do mod_version_var=$(echo "PYPI_${mod}_VERSION" | tr '[:lower:]' '[:upper:]' | tr '-' '_') mod_version=$(eval "echo \"\$$mod_version_var\"") @@ -134,11 +140,11 @@ EOF EOF FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" - ENV DEBIAN_FRONTEND noninteractive ARG MISP_TAG ARG MISP_COMMIT ARG PHP_VER + RUN apt-get update; apt-get install -y --no-install-recommends \ procps \ sudo \ diff --git a/template.env b/template.env index 16ed9db..8aac10b 100644 --- a/template.env +++ b/template.env @@ -11,6 +11,7 @@ PYPI_MISP_LIB_STIX2_VERSION="==3.0.*" PYPI_MAEC_VERSION="==4.1.*" PYPI_MIXBOX_VERSION="==1.0.*" PYPI_CYBOX_VERSION="==2.1.*" +PYPI_PYMISP_VERSION="==2.4.176" # MISP_COMMIT takes precedence over MISP_TAG # MISP_COMMIT=c56d537