Fix build arguments and pin pymisp (#28)

pull/1/head
Stefano Ortolani 2023-09-17 12:37:02 +01:00 committed by GitHub
parent a92d08b324
commit 969c131c84
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 43 additions and 29 deletions

View File

@ -28,4 +28,4 @@ jobs:
# Create the environment
sed -e '/^[[:space:]]*$/d' -e '/[#@]/d' -e 's/\"//g' -e 's/\(^[^=]*\)=\(.*\)/\1="\2"/' template.env > env.hcl
# Build and Push
DOCKER_IMG_TAG=$DOCKER_IMG_TAG docker buildx bake -f docker-bake.hcl -f env.hcl
DOCKER_IMG_TAG=$DOCKER_IMG_TAG DOCKER_USERNAME=local docker buildx bake -f docker-bake.hcl -f env.hcl

View File

@ -57,8 +57,6 @@ GitHub builds the images automatically and pushes them to [Docker hub](https://h
- User: `admin@admin.test`
- Password: `admin`
- Profit
### Using the image for development
Pull the entire repository, you can build the images using `docker-compose build`
@ -96,16 +94,8 @@ If you are interested in building the project from scratch - `git clone` or down
## Image file sizes
- Core server(Saved: 2.5GB)
- Original Image: 3.17GB
- First attempt: 2.24GB
- Remove chown: 1.56GB
- PreBuild python modules, and only pull submodules we need: 800MB
- PreBuild PHP modules: 664MB
- Modules (Saved: 640MB)
- Original: 1.36GB
- Pre-build modules: 750MB
- Core server: 260MB
- Modules: 470MB
### Configuration

View File

@ -34,6 +34,10 @@ variable "PYPI_CYBOX_VERSION" {
default = ""
}
variable "PYPI_PYMISP_VERSION" {
default = ""
}
variable "DOCKER_USERNAME" {
default = null
}
@ -101,6 +105,7 @@ target "misp" {
"PYPI_MAEC_VERSION": "${PYPI_MAEC_VERSION}",
"PYPI_MIXBOX_VERSION": "${PYPI_MIXBOX_VERSION}",
"PYPI_CYBOX_VERSION": "${PYPI_CYBOX_VERSION}",
"PYPI_PYMISP_VERSION": "${PYPI_PYMISP_VERSION}",
}
platforms = "${PLATFORMS}"
}

View File

@ -36,6 +36,15 @@ services:
- MISP_TAG=${MISP_TAG}
- MISP_COMMIT=${MISP_COMMIT}
- PHP_VER=${PHP_VER}
- PYPI_REDIS_VERSION=${PYPI_REDIS_VERSION}
- PYPI_LIEF_VERSION=${PYPI_LIEF_VERSION}
- PYPI_PYDEEP2_VERSION=${PYPI_PYDEEP2_VERSION}
- PYPI_PYTHON_MAGIC_VERSION=${PYPI_PYTHON_MAGIC_VERSION}
- PYPI_MISP_LIB_STIX2_VERSION=${PYPI_MISP_LIB_STIX2_VERSION}
- PYPI_MAEC_VERSION=${PYPI_MAEC_VERSION}
- PYPI_MIXBOX_VERSION=${PYPI_MIXBOX_VERSION}
- PYPI_CYBOX_VERSION=${PYPI_CYBOX_VERSION}
- PYPI_PYMISP_VERSION=${PYPI_PYMISP_VERSION}
depends_on:
- redis
- db
@ -78,6 +87,7 @@ services:
args:
- MODULES_TAG=${MODULES_TAG}
- MODULES_COMMIT=${MODULES_COMMIT}
- LIBFAUP_COMMIT=${LIBFAUP_COMMIT}
environment:
- "REDIS_BACKEND=redis"
depends_on:

View File

@ -5,6 +5,7 @@ FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye" as python-build
ARG MODULES_TAG
ARG MODULES_COMMIT
ARG LIBFAUP_COMMIT
RUN apt-get update && apt-get install -y --no-install-recommends \
cmake \
git \
@ -59,6 +60,7 @@ EOF
FROM "${DOCKER_HUB_PROXY}python:3.11-slim-bullseye"
ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update && apt-get install -y --no-install-recommends \
libglib2.0-0 \
libzbar0 \

View File

@ -1,11 +1,11 @@
ARG DOCKER_HUB_PROXY=""
FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build
ENV DEBIAN_FRONTEND noninteractive
ENV COMPOSER_ALLOW_SUPERUSER 1
ARG MISP_TAG
ARG MISP_COMMIT
RUN apt-get update; apt-get install -y --no-install-recommends \
ca-certificates \
php \
@ -42,6 +42,7 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as composer-build
FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as php-build
ENV DEBIAN_FRONTEND noninteractive
ENV TZ Etc/UTC
RUN apt-get update; apt-get install -y --no-install-recommends \
gcc \
g++ \
@ -56,16 +57,30 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as php-build
libsimdjson-dev \
git \
&& apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
RUN cp "/usr/lib/$(gcc -dumpmachine)"/libfuzzy.* /usr/lib; pecl channel-update pecl.php.net && pecl install ssdeep && pecl install rdkafka && pecl install simdjson
RUN git clone --recursive --depth=1 https://github.com/kjdev/php-ext-brotli.git && \
cd php-ext-brotli && phpize && ./configure && make && make install
FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build
ENV DEBIAN_FRONTEND noninteractive
RUN apt-get update; apt-get install -y --no-install-recommends python3-pip git && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
ARG MISP_TAG
ARG MISP_COMMIT
ARG PYPI_REDIS_VERSION
ARG PYPI_LIEF_VERSION
ARG PYPI_PYDEEP2_VERSION
ARG PYPI_PYTHON_MAGIC_VERSION
ARG PYPI_MISP_LIB_STIX2_VERSION
ARG PYPI_MAEC_VERSION
ARG PYPI_MIXBOX_VERSION
ARG PYPI_CYBOX_VERSION
ARG PYPI_PYMISP_VERSION
RUN apt-get update; apt-get install -y --no-install-recommends \
python3-pip \
git \
&& apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
# Download MISP using git in the /var/www/ directory. Remove unnecessary items.
RUN <<-EOF
if [ ! -z "${MISP_COMMIT}" ]; then
@ -75,15 +90,6 @@ FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim" as python-build
fi
cd /var/www/MISP || exit; git submodule update --init --recursive .
# Until MISP project specifies its required PyMISP version, we grab this from the submodule
pymispversion=$(head -n1 PyMISP/pymisp/__init__.py | awk '{print $3}' | tr -d \')
if [ -n "$pymispversion" ]; then
sed -i "s/pymisp$/pymisp==$pymispversion/" requirements.txt
cat requirements.txt
fi
# End of pymisp version fixing hack
EOF
RUN <<-EOF
@ -104,7 +110,7 @@ EOF
# 2. If missing, add it (with optional version from env (defaults to empty string))
# 3. If present, replace with our specified version if it exists, otherwise leave
# the upstream version alone.
set -- "redis" "lief" "pydeep2" "python-magic" "misp-lib-stix2" "maec" "mixbox" "cybox"
set -- "redis" "lief" "pydeep2" "python-magic" "misp-lib-stix2" "maec" "mixbox" "cybox" "pymisp"
for mod in "$@"; do
mod_version_var=$(echo "PYPI_${mod}_VERSION" | tr '[:lower:]' '[:upper:]' | tr '-' '_')
mod_version=$(eval "echo \"\$$mod_version_var\"")
@ -134,11 +140,11 @@ EOF
EOF
FROM "${DOCKER_HUB_PROXY}debian:bullseye-slim"
ENV DEBIAN_FRONTEND noninteractive
ARG MISP_TAG
ARG MISP_COMMIT
ARG PHP_VER
RUN apt-get update; apt-get install -y --no-install-recommends \
procps \
sudo \

View File

@ -11,6 +11,7 @@ PYPI_MISP_LIB_STIX2_VERSION="==3.0.*"
PYPI_MAEC_VERSION="==4.1.*"
PYPI_MIXBOX_VERSION="==1.0.*"
PYPI_CYBOX_VERSION="==2.1.*"
PYPI_PYMISP_VERSION="==2.4.176"
# MISP_COMMIT takes precedence over MISP_TAG
# MISP_COMMIT=c56d537