Instantiate custom entrypoint

.env

@@ -1,3 +0,0 @@
-MISP_TAG=v2.4.164
-MODULES_TAG=v2.4.163
-PHP_VER=20190902

.gitignore

@@ -1,4 +1,6 @@
 /logs/
 /files/
 /ssl/
-/server-configs/
+/configs/
+.gnupg
+.env

README.md

@@ -29,6 +29,8 @@ This is based on some of the work from the DSCO docker build, nearly all of the
 
 ## Getting Started
 
+-   Copy the `template.env` to `.env` and fill the missing configuration variables
+
 ### Development/Test
 
 -   Grab the `docker-compose.yml` and `server-configs/email.php` files (Keep directory structure)

build-docker-compose.yml

@@ -6,6 +6,7 @@ services:
         context: server/.
         args:
             - MISP_TAG=${MISP_TAG}
+            - MISP_COMMIT=${MISP_COMMIT}
             - PHP_VER=${PHP_VER}
 
   misp-modules:

docker-compose.yml

@@ -1,16 +1,22 @@
 version: '3'
 services:
-  # This is capible to relay via gmail, Amazon SES, or generic relays
-  # See: https://hub.docker.com/r/namshi/smtp
+  # This is capable to relay via gmail, Amazon SES, or generic relays
+  # See: https://hub.docker.com/r/ixdotai/smtp
   mail:
-    image: namshi/smtp
+    image: ixdotai/smtp
+    environment:
+      - "SMARTHOST_ADDRESS=${SMARTHOST_ADDRESS}"
+      - "SMARTHOST_PORT=${SMARTHOST_PORT}"
+      - "SMARTHOST_USER=${SMARTHOST_USER}"
+      - "SMARTHOST_PASSWORD=${SMARTHOST_PASSWORD}"
+      - "SMARTHOST_ALIASES=${SMARTHOST_ALIASES}"
 
   redis:
     image: redis:5.0.6
 
   db:
-    image: mysql:8.0.19
-    command: --default-authentication-plugin=mysql_native_password
+    # We use MariaDB because it supports ARM and has the expected collations
+    image: mariadb:10.8.2
     restart: always
     environment:
       - "MYSQL_USER=misp"
@@ -31,11 +37,12 @@ services:
       - "80:80"
       - "443:443"
     volumes:
-      - "./server-configs/:/var/www/MISP/app/Config/"
+      - "./configs/:/var/www/MISP/app/Config/"
       - "./logs/:/var/www/MISP/app/tmp/logs/"
       - "./files/:/var/www/MISP/app/files"
       - "./ssl/:/etc/nginx/certs"
 #      - "./examples/custom-entrypoint.sh:/custom-entrypoint.sh" # Use the example custom-entrypoint.sh
+      - "./.gnupg:/var/www/MISP/.gnupg"
     environment:
       - "HOSTNAME=https://localhost"
       - "REDIS_FQDN=redis"
@@ -53,7 +60,20 @@ services:
 #      - "CERTAUTH=optional" # Can be set to optional or on - Step 2 of https://github.com/MISP/MISP/tree/2.4/app/Plugin/CertAuth is still required
 #      - "SECURESSL=true" # Enable higher security SSL in nginx
 #      - "MISP_MODULES_FQDN=http://misp-modules" # Set the MISP Modules FQDN, used for Enrichment_services_url/Import_services_url/Export_services_url
-#      - "WORKERS=1" #If set to a value larger than 1 this will increase the number of parallel worker processes
+#      - "WORKERS=1" # Legacy variable controlling the number of parallel workers (use variables below instead)
+#      - "NUM_WORKERS_DEFAULT=5"    # To set the number of default workers
+#      - "NUM_WORKERS_PRIO=5"       # To set the number of prio workers
+#      - "NUM_WORKERS_EMAIL=5"      # To set the number of email workers
+#      - "NUM_WORKERS_UPDATE=1"     # To set the number of update workers
+#      - "NUM_WORKERS_CACHE=5"      # To set the number of cache workers
+      # Custom Settings
+      - "ADMIN_EMAIL=${ADMIN_EMAIL}"
+      - "ADMIN_KEY=${ADMIN_KEY}"
+      - "ADMIN_ORG=${ADMIN_ORG}"
+      - "GPG_PASSPHRASE=${GPG_PASSPHRASE}"
+      - "NSX_ANALYSIS_API_TOKEN=${NSX_ANALYSIS_API_TOKEN}"
+      - "NSX_ANALYSIS_KEY=${NSX_ANALYSIS_KEY}"
+      - "VIRUSTOTAL_KEY=${VIRUSTOTAL_KEY}"
   misp-modules:
     image: coolacid/misp-docker:modules-latest
     environment:

modules/Dockerfile

@@ -1,6 +1,7 @@
 FROM python:3.9-slim-bullseye  as builder
 ENV DEBIAN_FRONTEND noninteractive
 ARG MODULES_TAG
+ARG MODULES_COMMIT
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
                 cmake \
@@ -19,7 +20,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     RUN mkdir /wheel
     WORKDIR /srv
 
-    RUN git clone --branch ${MODULES_TAG} --depth 1  https://github.com/MISP/misp-modules.git /srv/misp-modules; \
+    RUN git clone --branch ${MODULES_TAG} --depth 1  https://github.com/MISP/misp-modules.git /srv/misp-modules
+    RUN if [ ! -z ${MODULES_COMMIT} ]; then cd /srv/misp-modules && git checkout ${MODULES_COMMIT}; fi; \
+        # Fixed in master - even though __init__.py is not mandatory in py3, wheels skip packages without it
+        touch /srv/misp-modules/misp_modules/modules/action_mod/_utils/__init__.py; \
+        # Fixed in master - missing dependencies
+        echo "Jinja2==3.1.2\nmattermostdriver==7.3.2" >> /srv/misp-modules/REQUIREMENTS; \
         cd /srv/misp-modules || exit; sed -i 's/-e //g' REQUIREMENTS; pip3 wheel -r REQUIREMENTS --no-cache-dir -w /wheel/
 
     RUN git clone --depth 1 https://github.com/stricaud/faup.git /srv/faup; \
@@ -39,10 +45,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
             libxrender1 \
             libxext6 \
             libpoppler-cpp0v5 \
+            libgl1 \
         && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/*
 
 COPY --from=builder /wheel /wheel
 COPY --from=builder /usr/local/lib/libfaupl* /usr/local/lib/
 RUN pip install --use-deprecated=legacy-resolver /wheel/*.whl; ldconfig
 
+# Since we compile faup ourselves and lua is not required anymore, we can load our own library 
+#   and skip the pre-compiled blob to improve compatibility with other architectures like arm
+RUN sed -i s/LoadLibrary\(LOAD_LIB\)/LoadLibrary\(\"\\/usr\\/local\\/lib\\/libfaupl.so\"\)/ \
+    /usr/local/lib/python3.9/site-packages/pyfaup/__init__.py
+
 ENTRYPOINT [ "/usr/local/bin/misp-modules", "-l", "0.0.0.0"]

server-configs/email.php

@@ -1,106 +0,0 @@
-<?php
-/**
- * This is email configuration file.
- *
- * Use it to configure email transports of Cake.
- *
- * PHP 5
- *
- * CakePHP(tm) : Rapid Development Framework (http://cakephp.org)
- * Copyright 2005-2012, Cake Software Foundation, Inc. (http://cakefoundation.org)
- *
- * Licensed under The MIT License
- * Redistributions of files must retain the above copyright notice.
- *
- * @copyright     Copyright 2005-2012, Cake Software Foundation, Inc. (http://cakefoundation.org)
- * @link          http://cakephp.org CakePHP(tm) Project
- * @package       app.Config
- * @since         CakePHP(tm) v 2.0.0
- * @license       MIT License (http://www.opensource.org/licenses/mit-license.php)
- */
-/**
- * In this file you set up your send email details.
- *
- * @package       cake.config
- */
-
-/**
- * Email configuration class.
- * You can specify multiple configurations for production, development and testing.
- *
- * transport => The name of a supported transport; valid options are as follows:
- *        Mail        - Send using PHP mail function
- *        Smtp        - Send using SMTP
- *        Debug        - Do not send the email, just return the result
- *
- * You can add custom transports (or override existing transports) by adding the
- * appropriate file to app/Network/Email.  Transports should be named 'YourTransport.php',
- * where 'Your' is the name of the transport.
- *
- * from =>
- * The origin email. See CakeEmail::from() about the valid values
- *
- */
-class EmailConfig {
-
-	// to set the return-path header, simply uncomment the line below and change you@localhost to the desired e-mail address
-	public $default = array(
-		'transport'     => 'Smtp',
-		'from'          => array('misp-dev@admin.test' => 'Misp DEV'),
-		'host'          => 'mail',
-		'port'          => 25,
-		'timeout'       => 30,
-//		'username'      => 'user',
-//		'password'      => 'secret',
-		'client'        => null,
-		'log'           => false,
-		//'charset'       => 'utf-8',
-		//'headerCharset' => 'utf-8',
-	);
-
-	public $smtp = array(
-		'transport'     => 'Smtp',
-		'from'          => array('misp-dev@admin.test' => 'Misp DEV'),
-		'host'          => 'mail',
-		'port'          => 25,
-		'timeout'       => 30,
-//		'username'      => 'user',
-//		'password'      => 'secret',
-		'client'        => null,
-		'log'           => false,
-		//'charset'       => 'utf-8',
-		//'headerCharset' => 'utf-8',
-	);
-
-	public $fast = array(
-		'from'          => 'misp-dev@admin.test',
-		'sender'        => null,
-		'to'            => null,
-		'cc'            => null,
-		'bcc'           => null,
-		'replyTo'       => null,
-		'readReceipt'   => null,
-		'returnPath'    => null,
-		'messageId'     => true,
-		'subject'       => null,
-		'message'       => null,
-		'headers'       => null,
-		'viewRender'    => null,
-		'template'      => false,
-		'layout'        => false,
-		'viewVars'      => null,
-		'attachments'   => null,
-		'emailFormat'   => null,
-		'transport'     => 'Smtp',
-		'host'          => 'mail',
-		'port'          => 25,
-		'timeout'       => 30,
-//		'username'      => 'user',
-//		'password'      => 'secret',
-		'client'        => null,
-		'log'           => true,
-		//'charset'       => 'utf-8',
-		//'headerCharset' => 'utf-8',
-	);
-
-}

server/Dockerfile

@@ -3,7 +3,9 @@ FROM composer:2.1.14 as composer-build
     WORKDIR /tmp
     ADD https://raw.githubusercontent.com/MISP/MISP/${MISP_TAG}/app/composer.json /tmp
     RUN composer install --ignore-platform-reqs && \
-     composer require jakub-onderka/openid-connect-php:1.0.0-rc1 --ignore-platform-reqs
+     composer require jakub-onderka/openid-connect-php:1.0.0-rc1 --ignore-platform-reqs && \
+     composer require --with-all-dependencies supervisorphp/supervisor:^4.0 \
+      guzzlehttp/guzzle php-http/message lstrojny/fxmlrpc --ignore-platform-reqs
 
 FROM debian:bullseye-slim as php-build
     RUN apt-get update; apt-get install -y --no-install-recommends \
@@ -84,6 +86,7 @@ FROM debian:bullseye-slim as python-build
 FROM debian:bullseye-slim
 ENV DEBIAN_FRONTEND noninteractive
 ARG MISP_TAG
+ARG MISP_COMMIT
 ARG PHP_VER
 
 # OS Packages
@@ -126,7 +129,8 @@ ARG PHP_VER
 
 # MISP code
     # Download MISP using git in the /var/www/ directory.
-    RUN git clone --branch ${MISP_TAG} --depth 1 https://github.com/MISP/MISP.git /var/www/MISP; \
+    RUN git clone --branch ${MISP_TAG} --depth 1 https://github.com/MISP/MISP.git /var/www/MISP
+    RUN if [ ! -z ${MISP_COMMIT} ]; then cd /var/www/MISP && git checkout ${MISP_COMMIT}; fi; \
         # We build the MISP modules outside, so we don't need to grab those submodules
         cd /var/www/MISP/app || exit; git submodule update --init --recursive .;
 
@@ -148,12 +152,8 @@ ARG PHP_VER
 
     RUN for dir in /etc/php/*; do echo "extension=ssdeep.so" > "$dir/mods-available/ssdeep.ini"; done \
             ;phpenmod redis \
-    # Enable CakeResque with php-gnupgp
-        ;phpenmod gnupg \
     # Enable ssdeep we build earlier
-        ;phpenmod ssdeep \
-    # To use the scheduler worker for scheduled tasks, do the following:
-        ;cp -fa /var/www/MISP/INSTALL/setup/config.php /var/www/MISP/app/Plugin/CakeResque/Config/config.php
+        ;phpenmod ssdeep
 
 # nginx
     RUN rm /etc/nginx/sites-enabled/*; mkdir /run/php /etc/nginx/certs
@@ -169,11 +169,12 @@ ARG PHP_VER
     RUN cp -R /var/www/MISP/app/Config /var/www/MISP/app/Config.dist
 
 # Entrypoints
-    COPY files/etc/supervisor/supervisor.conf /etc/supervisor/conf.d/supervisord.conf
+    COPY files/etc/supervisor/supervisor.conf /etc/supervisor/conf.d/10-supervisor.conf
+    COPY files/etc/supervisor/workers.conf /etc/supervisor/conf.d/50-workers.conf
+    COPY files/entrypoint_internal.sh /
     COPY files/entrypoint_fpm.sh /
     COPY files/entrypoint_nginx.sh /
     COPY files/entrypoint_cron.sh /
-    COPY files/entrypoint_workers.sh /
     COPY files/entrypoint.sh /
     ENTRYPOINT [ "/entrypoint.sh" ]
 

server/files/entrypoint.sh

@@ -1,3 +1,11 @@
 #!/bin/bash
-# start supervisord
-/usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf
+
+# export env variables again so they are not mandatory in docker-compose.yml in a backward compatible manner
+export NUM_WORKERS_DEFAULT=${NUM_WORKERS_DEFAULT:-${WORKERS:-5}}
+export NUM_WORKERS_PRIO=${NUM_WORKERS_PRIO:-${WORKERS:-5}}
+export NUM_WORKERS_EMAIL=${NUM_WORKERS_EMAIL:-${WORKERS:-5}}
+export NUM_WORKERS_UPDATE=${NUM_WORKERS_UPDATE:-${WORKERS:-1}}
+export NUM_WORKERS_CACHE=${NUM_WORKERS_CACHE:-${WORKERS:-5}}
+
+# start supervisord using the main configuration file so we have a socket interface
+/usr/bin/supervisord -c /etc/supervisor/supervisord.conf

server/files/entrypoint_cron.sh

@@ -9,7 +9,7 @@ cat << EOF > /etc/cron.d/misp
 10 3 * * * www-data /var/www/MISP/app/Console/cake Admin updateTaxonomies >/tmp/cronlog 2>/tmp/cronlog
 20 3 * * * www-data /var/www/MISP/app/Console/cake Admin updateWarningLists >/tmp/cronlog 2>/tmp/cronlog
 30 3 * * * www-data /var/www/MISP/app/Console/cake Admin updateNoticeLists >/tmp/cronlog 2>/tmp/cronlog
-45 3 * * * www-data /var/www/MISP/app/Console/cake Admin updateObjectTemplates >/tmp/cronlog 2>/tmp/cronlog
+45 3 * * * www-data /var/www/MISP/app/Console/cake Admin updateObjectTemplates "$CRON_USER_ID" >/tmp/cronlog 2>/tmp/cronlog
 
 EOF
 

server/files/entrypoint_internal.sh

@@ -0,0 +1,195 @@
+#!/bin/bash
+
+[ -z "$ADMIN_EMAIL" ] && ADMIN_EMAIL="admin@admin.test"
+[ -z "$GPG_PASSPHRASE" ] && GPG_PASSPHRASE="passphrase"
+
+init_gnupg() {
+    GPG_DIR=/var/www/MISP/.gnupg
+    GPG_ASC=/var/www/MISP/app/webroot/gpg.asc
+    GPG_TMP=/tmp/gpg.tmp
+
+    if [ ! -d ${GPG_DIR} ]; then
+        echo "Generating GPG key ... (please be patient, we need some entropy)"
+        cat >${GPG_TMP} <<GPGEOF
+%echo Generating a basic OpenPGP key
+Key-Type: RSA
+Key-Length: 3072
+Name-Real: MISP Admin
+Name-Email: $ADMIN_EMAIL
+Expire-Date: 0
+Passphrase: $GPG_PASSPHRASE
+%commit
+%echo Done
+GPGEOF
+        mkdir ${GPG_DIR}
+        gpg --homedir ${GPG_DIR} --gen-key --batch ${GPG_TMP}
+        chown -R www-data:www-data ${GPG_DIR}
+        rm -f ${GPG_TMP}
+    else
+        echo "Using pre-generated GPG key in ${GPG_DIR}"
+    fi
+
+    if [ ! -f ${GPG_ASC} ]; then
+        echo "Exporting GPG key ..."
+        sudo -u www-data gpg --homedir ${GPG_DIR} --export --armor ${ADMIN_EMAIL} > ${GPG_ASC}
+    else
+        echo "Found exported key ${GPG_ASC}"
+    fi
+
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "GnuPG.email" "${ADMIN_EMAIL}"
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "GnuPG.homedir" "${GPG_DIR}"
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "GnuPG.password" "${GPG_PASSPHRASE}"
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "GnuPG.obscure_subject" false
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "GnuPG.binary" "$(which gpg)"
+}
+
+apply_updates() {
+    # Disable weird default
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" false
+    # Run updates
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin runUpdates
+}
+
+init_user() {
+    # Create the main user if it is not there already
+    sudo -u www-data /var/www/MISP/app/Console/cake userInit -q
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "MISP.email" ${ADMIN_EMAIL}
+    echo 'UPDATE misp.users SET change_pw = 0 WHERE id = 1;' | ${MYSQLCMD}
+    echo "UPDATE misp.users SET email = \"${ADMIN_EMAIL}\" WHERE id = 1;" | ${MYSQLCMD}
+    if [ ! -z "$ADMIN_ORG" ]; then
+        echo "UPDATE misp.organisations SET name = \"${ADMIN_ORG}\" where id = 1;" | ${MYSQLCMD}
+    fi
+    if [ ! -z "$ADMIN_KEY" ]; then
+        echo "Customize MISP | Setting admin key to '${ADMIN_KEY}'"
+        CHANGE_CMD=(sudo -u www-data /var/www/MISP/app/Console/cake User change_authkey 1 "${ADMIN_KEY}")
+    else
+        echo "Customize MISP | Regenerating admin key"
+        CHANGE_CMD=(sudo -u www-data /var/www/MISP/app/Console/cake User change_authkey 1)
+    fi
+    ADMIN_KEY=`${CHANGE_CMD[@]} | awk 'END {print $NF; exit}'`
+    echo "Customize MISP | Admin user key set to '${ADMIN_KEY}'"
+}
+
+apply_critical_fixes() {
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "MISP.external_baseurl" "${HOSTNAME}"
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "MISP.host_org_id" 1
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "Plugin.Action_services_enable" false
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "Plugin.Enrichment_hover_enable" true
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "Plugin.Enrichment_hover_popover_only" false
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "Security.csp_enforce" true
+}
+
+apply_custom_settings() {
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting --force "MISP.welcome_text_top" ""
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting --force "MISP.welcome_text_bottom" ""
+    
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "MISP.contact" "${ADMIN_EMAIL}"
+    # This is not necessary because we update the DB directly
+    # sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "MISP.org" "${ADMIN_ORG}"
+    
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "MISP.log_client_ip" true
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "MISP.log_user_ips" true
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "MISP.log_user_ips_authkeys" true
+
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "Plugin.Enrichment_timeout" 30
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "Plugin.Enrichment_hover_timeout" 5
+}
+
+configure_plugins() {
+    if [ ! -z "$VIRUSTOTAL_KEY" ]; then
+        echo "Customize MISP | Enabling 'virustotal' module ..."
+        sudo -u www-data php /var/www/MISP/tests/modify_config.php modify "{
+            \"Plugin\": {
+                \"Enrichment_virustotal_enabled\": true,
+                \"Enrichment_virustotal_apikey\": \"${VIRUSTOTAL_KEY}\"
+            }
+        }" > /dev/null
+    fi
+
+    if [ ! -z "$VIRUSTOTAL_KEY" ] && [ ! -z "$NSX_ANALYSIS_KEY" ] && [ ! -z "$NSX_ANALYSIS_API_TOKEN" ] && [ ! -z "$ADMIN_KEY" ]; then
+        echo "Customize MISP | Enabling 'vmware_nsx' module ..."
+        sudo -u www-data php /var/www/MISP/tests/modify_config.php modify "{
+            \"Plugin\": {
+                \"Enrichment_vmware_nsx_enabled\": true,
+                \"Enrichment_vmware_nsx_analysis_verify_ssl\": \"True\",
+                \"Enrichment_vmware_nsx_analysis_key\": \"${NSX_ANALYSIS_KEY}\",
+                \"Enrichment_vmware_nsx_analysis_api_token\": \"${NSX_ANALYSIS_API_TOKEN}\",
+                \"Enrichment_vmware_nsx_vt_key\": \"${VIRUSTOTAL_KEY}\",
+                \"Enrichment_vmware_nsx_misp_url\": \"${HOSTNAME}\",
+                \"Enrichment_vmware_nsx_misp_verify_ssl\": \"False\",
+                \"Enrichment_vmware_nsx_misp_key\": \"${ADMIN_KEY}\"
+            }
+        }" > /dev/null
+    fi
+}
+
+configure_email() {
+    sudo -u www-data tee /var/www/MISP/app/Config/email.php > /dev/null <<EOT
+<?php
+class EmailConfig {
+    public \$default = array(
+        'transport'     => 'Smtp',
+        'from'          => array('misp-dev@admin.test' => 'Misp DEV'),
+        'host'          => 'mail',
+        'port'          => 25,
+        'timeout'       => 30,
+        'client'        => null,
+        'log'           => false,
+    );
+    public \$smtp = array(
+        'transport'     => 'Smtp',
+        'from'          => array('misp-dev@admin.test' => 'Misp DEV'),
+        'host'          => 'mail',
+        'port'          => 25,
+        'timeout'       => 30,
+        'client'        => null,
+        'log'           => false,
+    );
+    public \$fast = array(
+        'from'          => 'misp-dev@admin.test',
+        'sender'        => null,
+        'to'            => null,
+        'cc'            => null,
+        'bcc'           => null,
+        'replyTo'       => null,
+        'readReceipt'   => null,
+        'returnPath'    => null,
+        'messageId'     => true,
+        'subject'       => null,
+        'message'       => null,
+        'headers'       => null,
+        'viewRender'    => null,
+        'template'      => false,
+        'layout'        => false,
+        'viewVars'      => null,
+        'attachments'   => null,
+        'emailFormat'   => null,
+        'transport'     => 'Smtp',
+        'host'          => 'mail',
+        'port'          => 25,
+        'timeout'       => 30,
+        'client'        => null,
+        'log'           => true,
+    );
+}
+EOT
+}
+
+
+echo "Customize MISP | Configure email ..." && configure_email
+
+echo "Customize MISP | Configure GPG key ..." && init_gnupg
+
+echo "Customize MISP | Running updates ..." && apply_updates
+
+echo "Customize MISP | Init default user and organization ..." && init_user
+
+echo "Customize MISP | Resolve critical issues ..." && apply_critical_fixes
+
+echo "Customize MISP | Customize installation ..." && apply_custom_settings
+
+# This item last so we had a chance to create the ADMIN_KEY if not specified
+echo "Customize MISP | Configure plugins ..." && configure_plugins
+
+# Make the instance live
+sudo -u www-data /var/www/MISP/app/Console/cake Admin live 1

server/files/entrypoint_nginx.sh

@@ -13,11 +13,6 @@ MISP_APP_CONFIG_PATH=/var/www/MISP/app/Config
 ENTRYPOINT_PID_FILE="/entrypoint_apache.install"
 [ ! -f $ENTRYPOINT_PID_FILE ] && touch $ENTRYPOINT_PID_FILE
 
-setup_cake_config(){
-    sed -i "s/'host' => 'localhost'.*/'host' => '$REDIS_FQDN',          \/\/ Redis server hostname/" "/var/www/MISP/app/Plugin/CakeResque/Config/config.php"
-    sed -i "s/'host' => '127.0.0.1'.*/'host' => '$REDIS_FQDN',          \/\/ Redis server hostname/" "/var/www/MISP/app/Plugin/CakeResque/Config/config.php"
-}
-
 init_misp_config(){
     [ -f $MISP_APP_CONFIG_PATH/bootstrap.php ] || cp $MISP_APP_CONFIG_PATH.dist/bootstrap.default.php $MISP_APP_CONFIG_PATH/bootstrap.php
     [ -f $MISP_APP_CONFIG_PATH/database.php ] || cp $MISP_APP_CONFIG_PATH.dist/database.default.php $MISP_APP_CONFIG_PATH/database.php
@@ -50,11 +45,20 @@ init_misp_config(){
     /var/www/MISP/app/Console/cake Admin setSetting "Plugin.Export_services_url" "$MISP_MODULES_FQDN"
 
     /var/www/MISP/app/Console/cake Admin setSetting "Plugin.Cortex_services_enable" false
+}
 
-    echo Change number of workers
-    if [ ! -z "$WORKERS" ] && [ "$WORKERS" -gt "1" ]; then
-        sed -i "s/start --interval/start -n $WORKERS --interval/" /var/www/MISP/app/Console/worker/start.sh
-    fi
+init_misp_workers(){
+    # Note that we are doing this after enforcing permissions, so we need to use the www-data user for this
+    echo "Configuring background workers"
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "SimpleBackgroundJobs.enabled" true
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "SimpleBackgroundJobs.supervisor_host" "127.0.0.1"
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "SimpleBackgroundJobs.supervisor_port" 9001
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "SimpleBackgroundJobs.supervisor_password" "supervisor"
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "SimpleBackgroundJobs.supervisor_user" "supervisor"
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "SimpleBackgroundJobs.redis_host" "$REDIS_FQDN"
+
+    echo "Starting background workers"
+    supervisorctl start misp-workers:*
 }
 
 init_misp_files(){
@@ -127,22 +131,22 @@ if [[ "$INIT" == true ]]; then
     echo "Ensure SSL certs exist..." && init_ssl
 fi
 
-# Things that should ALWAYS happen
-echo "Configure Cake | Change Redis host to $REDIS_FQDN ... " && setup_cake_config
-
 # Things we should do if we're configuring MISP via ENV
 echo "Configure MISP | Initialize misp base config..." && init_misp_config
 
 echo "Configure MISP | Sync app files..." && sync_files
 
 echo "Configure MISP | Enforce permissions ..."
-echo "... chown -R www-data.www-data /var/www/MISP ..." && find /var/www/MISP -not -user www-data -exec chown www-data.www-data {} +
+echo "... chown -R www-data:www-data /var/www/MISP ..." && find /var/www/MISP -not -user www-data -exec chown www-data:www-data {} +
 echo "... chmod -R 0750 /var/www/MISP ..." && find /var/www/MISP -perm 550 -type f -exec chmod 0550 {} + && find /var/www/MISP -perm 770 -type d -exec chmod 0770 {} +
 echo "... chmod -R g+ws /var/www/MISP/app/tmp ..." && chmod -R g+ws /var/www/MISP/app/tmp
 echo "... chmod -R g+ws /var/www/MISP/app/files ..." && chmod -R g+ws /var/www/MISP/app/files
 echo "... chmod -R g+ws /var/www/MISP/app/files/scripts/tmp ..." && chmod -R g+ws /var/www/MISP/app/files/scripts/tmp
 echo "... chmod 600 /var/www/MISP/app/Config/config.php /var/www/MISP/app/Config/database.php /var/www/MISP/app/Config/email.php ... " && chmod 600 /var/www/MISP/app/Config/config.php /var/www/MISP/app/Config/database.php /var/www/MISP/app/Config/email.php
 
+# Workers are set to NOT auto start so we have time to enforce permissions on the cache first
+echo "Configure MISP | Starting workers ..." && init_misp_workers
+
 # Work around https://github.com/MISP/MISP/issues/5608
 if [[ ! -f /var/www/MISP/PyMISP/pymisp/data/describeTypes.json ]]; then
     mkdir -p /var/www/MISP/PyMISP/pymisp/data/
@@ -195,6 +199,18 @@ if [[ -x /custom-entrypoint.sh ]]; then
     /custom-entrypoint.sh
 fi
 
+if [[ -x /entrypoint_internal.sh ]]; then
+    export ADMIN_EMAIL=${ADMIN_EMAIL}
+    export ADMIN_ORG=${ADMIN_ORG}
+    export GPG_PASSPHRASE=${GPG_PASSPHRASE}
+    export HOSTNAME=${HOSTNAME}
+    export MYSQLCMD=${MYSQLCMD}
+    export NSX_ANALYSIS_API_TOKEN=${NSX_ANALYSIS_API_TOKEN}
+    export NSX_ANALYSIS_KEY=${NSX_ANALYSIS_KEY}
+    export VIRUSTOTAL_KEY=${VIRUSTOTAL_KEY}
+    /entrypoint_internal.sh
+fi
+
 # delete pid file
 [ -f $ENTRYPOINT_PID_FILE ] && rm $ENTRYPOINT_PID_FILE
 

server/files/entrypoint_workers.sh

@@ -1,17 +0,0 @@
-#!/bin/bash
-
-# Wait until entrypoint apache is ready
-while (true)
-do
-    sleep 2
-    [ -f /entrypoint_apache.install ] && continue
-    break
-done
-
-while true
-do
-    echo "Start Workers..."
-    sudo -u www-data /var/www/MISP/app/Console/worker/start.sh
-    echo "Start Workers...finished"
-    sleep 3600
-done

server/files/etc/supervisor/supervisor.conf

@@ -6,6 +6,11 @@ stdout_logfile_maxbytes=0
 stderr_logfile=/dev/stderr
 stderr_logfile_maxbytes=0
 
+[inet_http_server]
+port=127.0.0.1:9001
+username=supervisor
+password=supervisor
+
 [program:nginx]
 command=/entrypoint_nginx.sh
 autorestart=true
@@ -25,16 +30,7 @@ redirect_stderr=true
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0
 
-[program:workers]
-command=/entrypoint_workers.sh
-stdout_logfile=/dev/stdout
-stdout_logfile_maxbytes=0
-stderr_logfile=/dev/stderr
-stderr_logfile_maxbytes=0
-autostart=true
-
 [program:cron]
-environment=CRON_USER_ID=%(ENV_CRON_USER_ID)s
 command=/entrypoint_cron.sh
 stdout_logfile=/dev/stdout
 stdout_logfile_maxbytes=0

server/files/etc/supervisor/workers.conf

@@ -0,0 +1,68 @@
+# Workers are set to NOT auto start so we have time to enforce permissions on the cache first
+
+[group:misp-workers]
+programs=default,email,cache,prio,update
+
+[program:default]
+directory=/var/www/MISP
+command=/var/www/MISP/app/Console/cake start_worker default
+process_name=%(program_name)s_%(process_num)02d
+numprocs=%(ENV_NUM_WORKERS_DEFAULT)s
+autostart=false
+autorestart=true
+redirect_stderr=false
+stderr_logfile=/var/www/MISP/app/tmp/logs/misp-workers-errors.log
+stdout_logfile=/var/www/MISP/app/tmp/logs/misp-workers.log
+directory=/var/www/MISP
+user=www-data
+
+[program:prio]
+directory=/var/www/MISP
+command=/var/www/MISP/app/Console/cake start_worker prio
+process_name=%(program_name)s_%(process_num)02d
+numprocs=%(ENV_NUM_WORKERS_PRIO)s
+autostart=false
+autorestart=true
+redirect_stderr=false
+stderr_logfile=/var/www/MISP/app/tmp/logs/misp-workers-errors.log
+stdout_logfile=/var/www/MISP/app/tmp/logs/misp-workers.log
+directory=/var/www/MISP
+user=www-data
+
+[program:email]
+directory=/var/www/MISP
+command=/var/www/MISP/app/Console/cake start_worker email
+process_name=%(program_name)s_%(process_num)02d
+numprocs=%(ENV_NUM_WORKERS_EMAIL)s
+autostart=false
+autorestart=true
+redirect_stderr=false
+stderr_logfile=/var/www/MISP/app/tmp/logs/misp-workers-errors.log
+stdout_logfile=/var/www/MISP/app/tmp/logs/misp-workers.log
+directory=/var/www/MISP
+user=www-data
+
+[program:update]
+directory=/var/www/MISP
+command=/var/www/MISP/app/Console/cake start_worker update
+process_name=%(program_name)s_%(process_num)02d
+numprocs=%(ENV_NUM_WORKERS_UPDATE)s
+autostart=false
+autorestart=true
+redirect_stderr=false
+stderr_logfile=/var/www/MISP/app/tmp/logs/misp-workers-errors.log
+stdout_logfile=/var/www/MISP/app/tmp/logs/misp-workers.log
+directory=/var/www/MISP
+user=www-data
+
+[program:cache]
+directory=/var/www/MISP
+command=/var/www/MISP/app/Console/cake start_worker cache
+process_name=%(program_name)s_%(process_num)02d
+numprocs=%(ENV_NUM_WORKERS_CACHE)s
+autostart=false
+autorestart=true
+redirect_stderr=false
+stderr_logfile=/var/www/MISP/app/tmp/logs/misp-workers-errors.log
+stdout_logfile=/var/www/MISP/app/tmp/logs/misp-workers.log
+user=www-data

template.env

@@ -0,0 +1,21 @@
+MISP_TAG=v2.4.161
+MODULES_TAG=v2.4.160
+PHP_VER=20190902
+# MISP_COMMIT takes precedence over MISP_TAG
+# MISP_COMMIT=c56d537
+# MODULES_COMMIT takes precedence over MODULES_TAG
+# MODULES_COMMIT=de69ae3
+
+ADMIN_EMAIL=
+ADMIN_ORG=
+ADMIN_KEY=
+GPG_PASSPHRASE=
+NSX_ANALYSIS_API_TOKEN=
+NSX_ANALYSIS_KEY=
+VIRUSTOTAL_KEY=
+
+SMARTHOST_ADDRESS=
+SMARTHOST_PORT=
+SMARTHOST_USER=
+SMARTHOST_PASSWORD=
+SMARTHOST_ALIASES=