diff --git a/.env b/.env deleted file mode 100644 index 838de33..0000000 --- a/.env +++ /dev/null @@ -1,3 +0,0 @@ -MISP_TAG=v2.4.164 -MODULES_TAG=v2.4.163 -PHP_VER=20190902 diff --git a/.gitignore b/.gitignore index 07c7cdd..6fe426c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,6 @@ /logs/ /files/ /ssl/ -/server-configs/ \ No newline at end of file +/configs/ +.gnupg +.env diff --git a/README.md b/README.md index 3bf44c1..53266bf 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,8 @@ This is based on some of the work from the DSCO docker build, nearly all of the ## Getting Started +- Copy the `template.env` to `.env` and fill the missing configuration variables + ### Development/Test - Grab the `docker-compose.yml` and `server-configs/email.php` files (Keep directory structure) diff --git a/build-docker-compose.yml b/build-docker-compose.yml index 87ae843..b44ad2c 100644 --- a/build-docker-compose.yml +++ b/build-docker-compose.yml @@ -6,6 +6,7 @@ services: context: server/. args: - MISP_TAG=${MISP_TAG} + - MISP_COMMIT=${MISP_COMMIT} - PHP_VER=${PHP_VER} misp-modules: diff --git a/docker-compose.yml b/docker-compose.yml index 5af92b7..9649e7b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,16 +1,22 @@ version: '3' services: - # This is capible to relay via gmail, Amazon SES, or generic relays - # See: https://hub.docker.com/r/namshi/smtp + # This is capable to relay via gmail, Amazon SES, or generic relays + # See: https://hub.docker.com/r/ixdotai/smtp mail: - image: namshi/smtp + image: ixdotai/smtp + environment: + - "SMARTHOST_ADDRESS=${SMARTHOST_ADDRESS}" + - "SMARTHOST_PORT=${SMARTHOST_PORT}" + - "SMARTHOST_USER=${SMARTHOST_USER}" + - "SMARTHOST_PASSWORD=${SMARTHOST_PASSWORD}" + - "SMARTHOST_ALIASES=${SMARTHOST_ALIASES}" redis: image: redis:5.0.6 db: - image: mysql:8.0.19 - command: --default-authentication-plugin=mysql_native_password + # We use MariaDB because it supports ARM and has the expected collations + image: mariadb:10.8.2 restart: always environment: - "MYSQL_USER=misp" @@ -31,11 +37,12 @@ services: - "80:80" - "443:443" volumes: - - "./server-configs/:/var/www/MISP/app/Config/" + - "./configs/:/var/www/MISP/app/Config/" - "./logs/:/var/www/MISP/app/tmp/logs/" - "./files/:/var/www/MISP/app/files" - "./ssl/:/etc/nginx/certs" # - "./examples/custom-entrypoint.sh:/custom-entrypoint.sh" # Use the example custom-entrypoint.sh + - "./.gnupg:/var/www/MISP/.gnupg" environment: - "HOSTNAME=https://localhost" - "REDIS_FQDN=redis" @@ -53,7 +60,20 @@ services: # - "CERTAUTH=optional" # Can be set to optional or on - Step 2 of https://github.com/MISP/MISP/tree/2.4/app/Plugin/CertAuth is still required # - "SECURESSL=true" # Enable higher security SSL in nginx # - "MISP_MODULES_FQDN=http://misp-modules" # Set the MISP Modules FQDN, used for Enrichment_services_url/Import_services_url/Export_services_url -# - "WORKERS=1" #If set to a value larger than 1 this will increase the number of parallel worker processes +# - "WORKERS=1" # Legacy variable controlling the number of parallel workers (use variables below instead) +# - "NUM_WORKERS_DEFAULT=5" # To set the number of default workers +# - "NUM_WORKERS_PRIO=5" # To set the number of prio workers +# - "NUM_WORKERS_EMAIL=5" # To set the number of email workers +# - "NUM_WORKERS_UPDATE=1" # To set the number of update workers +# - "NUM_WORKERS_CACHE=5" # To set the number of cache workers + # Custom Settings + - "ADMIN_EMAIL=${ADMIN_EMAIL}" + - "ADMIN_KEY=${ADMIN_KEY}" + - "ADMIN_ORG=${ADMIN_ORG}" + - "GPG_PASSPHRASE=${GPG_PASSPHRASE}" + - "NSX_ANALYSIS_API_TOKEN=${NSX_ANALYSIS_API_TOKEN}" + - "NSX_ANALYSIS_KEY=${NSX_ANALYSIS_KEY}" + - "VIRUSTOTAL_KEY=${VIRUSTOTAL_KEY}" misp-modules: image: coolacid/misp-docker:modules-latest environment: diff --git a/modules/Dockerfile b/modules/Dockerfile index 9683708..e0cc25d 100644 --- a/modules/Dockerfile +++ b/modules/Dockerfile @@ -1,6 +1,7 @@ FROM python:3.9-slim-bullseye as builder ENV DEBIAN_FRONTEND noninteractive ARG MODULES_TAG +ARG MODULES_COMMIT RUN apt-get update && apt-get install -y --no-install-recommends \ cmake \ @@ -19,7 +20,12 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ RUN mkdir /wheel WORKDIR /srv - RUN git clone --branch ${MODULES_TAG} --depth 1 https://github.com/MISP/misp-modules.git /srv/misp-modules; \ + RUN git clone --branch ${MODULES_TAG} --depth 1 https://github.com/MISP/misp-modules.git /srv/misp-modules + RUN if [ ! -z ${MODULES_COMMIT} ]; then cd /srv/misp-modules && git checkout ${MODULES_COMMIT}; fi; \ + # Fixed in master - even though __init__.py is not mandatory in py3, wheels skip packages without it + touch /srv/misp-modules/misp_modules/modules/action_mod/_utils/__init__.py; \ + # Fixed in master - missing dependencies + echo "Jinja2==3.1.2\nmattermostdriver==7.3.2" >> /srv/misp-modules/REQUIREMENTS; \ cd /srv/misp-modules || exit; sed -i 's/-e //g' REQUIREMENTS; pip3 wheel -r REQUIREMENTS --no-cache-dir -w /wheel/ RUN git clone --depth 1 https://github.com/stricaud/faup.git /srv/faup; \ @@ -39,10 +45,16 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ libxrender1 \ libxext6 \ libpoppler-cpp0v5 \ + libgl1 \ && apt-get autoremove -y && apt-get clean -y && rm -rf /var/lib/apt/lists/* COPY --from=builder /wheel /wheel COPY --from=builder /usr/local/lib/libfaupl* /usr/local/lib/ RUN pip install --use-deprecated=legacy-resolver /wheel/*.whl; ldconfig +# Since we compile faup ourselves and lua is not required anymore, we can load our own library +# and skip the pre-compiled blob to improve compatibility with other architectures like arm +RUN sed -i s/LoadLibrary\(LOAD_LIB\)/LoadLibrary\(\"\\/usr\\/local\\/lib\\/libfaupl.so\"\)/ \ + /usr/local/lib/python3.9/site-packages/pyfaup/__init__.py + ENTRYPOINT [ "/usr/local/bin/misp-modules", "-l", "0.0.0.0"] diff --git a/server-configs/email.php b/server-configs/email.php deleted file mode 100644 index bf460cb..0000000 --- a/server-configs/email.php +++ /dev/null @@ -1,106 +0,0 @@ - The name of a supported transport; valid options are as follows: - * Mail - Send using PHP mail function - * Smtp - Send using SMTP - * Debug - Do not send the email, just return the result - * - * You can add custom transports (or override existing transports) by adding the - * appropriate file to app/Network/Email. Transports should be named 'YourTransport.php', - * where 'Your' is the name of the transport. - * - * from => - * The origin email. See CakeEmail::from() about the valid values - * - */ -class EmailConfig { - - // to set the return-path header, simply uncomment the line below and change you@localhost to the desired e-mail address - public $default = array( - 'transport' => 'Smtp', - 'from' => array('misp-dev@admin.test' => 'Misp DEV'), - 'host' => 'mail', - 'port' => 25, - 'timeout' => 30, -// 'username' => 'user', -// 'password' => 'secret', - 'client' => null, - 'log' => false, - //'charset' => 'utf-8', - //'headerCharset' => 'utf-8', - ); - - public $smtp = array( - 'transport' => 'Smtp', - 'from' => array('misp-dev@admin.test' => 'Misp DEV'), - 'host' => 'mail', - 'port' => 25, - 'timeout' => 30, -// 'username' => 'user', -// 'password' => 'secret', - 'client' => null, - 'log' => false, - //'charset' => 'utf-8', - //'headerCharset' => 'utf-8', - ); - - public $fast = array( - 'from' => 'misp-dev@admin.test', - 'sender' => null, - 'to' => null, - 'cc' => null, - 'bcc' => null, - 'replyTo' => null, - 'readReceipt' => null, - 'returnPath' => null, - 'messageId' => true, - 'subject' => null, - 'message' => null, - 'headers' => null, - 'viewRender' => null, - 'template' => false, - 'layout' => false, - 'viewVars' => null, - 'attachments' => null, - 'emailFormat' => null, - 'transport' => 'Smtp', - 'host' => 'mail', - 'port' => 25, - 'timeout' => 30, -// 'username' => 'user', -// 'password' => 'secret', - 'client' => null, - 'log' => true, - //'charset' => 'utf-8', - //'headerCharset' => 'utf-8', - ); - -} diff --git a/server/Dockerfile b/server/Dockerfile index 9e8d56a..f4c83ea 100644 --- a/server/Dockerfile +++ b/server/Dockerfile @@ -3,7 +3,9 @@ FROM composer:2.1.14 as composer-build WORKDIR /tmp ADD https://raw.githubusercontent.com/MISP/MISP/${MISP_TAG}/app/composer.json /tmp RUN composer install --ignore-platform-reqs && \ - composer require jakub-onderka/openid-connect-php:1.0.0-rc1 --ignore-platform-reqs + composer require jakub-onderka/openid-connect-php:1.0.0-rc1 --ignore-platform-reqs && \ + composer require --with-all-dependencies supervisorphp/supervisor:^4.0 \ + guzzlehttp/guzzle php-http/message lstrojny/fxmlrpc --ignore-platform-reqs FROM debian:bullseye-slim as php-build RUN apt-get update; apt-get install -y --no-install-recommends \ @@ -84,6 +86,7 @@ FROM debian:bullseye-slim as python-build FROM debian:bullseye-slim ENV DEBIAN_FRONTEND noninteractive ARG MISP_TAG +ARG MISP_COMMIT ARG PHP_VER # OS Packages @@ -126,7 +129,8 @@ ARG PHP_VER # MISP code # Download MISP using git in the /var/www/ directory. - RUN git clone --branch ${MISP_TAG} --depth 1 https://github.com/MISP/MISP.git /var/www/MISP; \ + RUN git clone --branch ${MISP_TAG} --depth 1 https://github.com/MISP/MISP.git /var/www/MISP + RUN if [ ! -z ${MISP_COMMIT} ]; then cd /var/www/MISP && git checkout ${MISP_COMMIT}; fi; \ # We build the MISP modules outside, so we don't need to grab those submodules cd /var/www/MISP/app || exit; git submodule update --init --recursive .; @@ -148,12 +152,8 @@ ARG PHP_VER RUN for dir in /etc/php/*; do echo "extension=ssdeep.so" > "$dir/mods-available/ssdeep.ini"; done \ ;phpenmod redis \ - # Enable CakeResque with php-gnupgp - ;phpenmod gnupg \ # Enable ssdeep we build earlier - ;phpenmod ssdeep \ - # To use the scheduler worker for scheduled tasks, do the following: - ;cp -fa /var/www/MISP/INSTALL/setup/config.php /var/www/MISP/app/Plugin/CakeResque/Config/config.php + ;phpenmod ssdeep # nginx RUN rm /etc/nginx/sites-enabled/*; mkdir /run/php /etc/nginx/certs @@ -169,11 +169,12 @@ ARG PHP_VER RUN cp -R /var/www/MISP/app/Config /var/www/MISP/app/Config.dist # Entrypoints - COPY files/etc/supervisor/supervisor.conf /etc/supervisor/conf.d/supervisord.conf + COPY files/etc/supervisor/supervisor.conf /etc/supervisor/conf.d/10-supervisor.conf + COPY files/etc/supervisor/workers.conf /etc/supervisor/conf.d/50-workers.conf + COPY files/entrypoint_internal.sh / COPY files/entrypoint_fpm.sh / COPY files/entrypoint_nginx.sh / COPY files/entrypoint_cron.sh / - COPY files/entrypoint_workers.sh / COPY files/entrypoint.sh / ENTRYPOINT [ "/entrypoint.sh" ] diff --git a/server/files/entrypoint.sh b/server/files/entrypoint.sh index 5a7aa71..61c9d85 100755 --- a/server/files/entrypoint.sh +++ b/server/files/entrypoint.sh @@ -1,3 +1,11 @@ #!/bin/bash -# start supervisord -/usr/bin/supervisord -c /etc/supervisor/conf.d/supervisord.conf + +# export env variables again so they are not mandatory in docker-compose.yml in a backward compatible manner +export NUM_WORKERS_DEFAULT=${NUM_WORKERS_DEFAULT:-${WORKERS:-5}} +export NUM_WORKERS_PRIO=${NUM_WORKERS_PRIO:-${WORKERS:-5}} +export NUM_WORKERS_EMAIL=${NUM_WORKERS_EMAIL:-${WORKERS:-5}} +export NUM_WORKERS_UPDATE=${NUM_WORKERS_UPDATE:-${WORKERS:-1}} +export NUM_WORKERS_CACHE=${NUM_WORKERS_CACHE:-${WORKERS:-5}} + +# start supervisord using the main configuration file so we have a socket interface +/usr/bin/supervisord -c /etc/supervisor/supervisord.conf diff --git a/server/files/entrypoint_cron.sh b/server/files/entrypoint_cron.sh index 98977b7..c38e6fc 100755 --- a/server/files/entrypoint_cron.sh +++ b/server/files/entrypoint_cron.sh @@ -9,7 +9,7 @@ cat << EOF > /etc/cron.d/misp 10 3 * * * www-data /var/www/MISP/app/Console/cake Admin updateTaxonomies >/tmp/cronlog 2>/tmp/cronlog 20 3 * * * www-data /var/www/MISP/app/Console/cake Admin updateWarningLists >/tmp/cronlog 2>/tmp/cronlog 30 3 * * * www-data /var/www/MISP/app/Console/cake Admin updateNoticeLists >/tmp/cronlog 2>/tmp/cronlog -45 3 * * * www-data /var/www/MISP/app/Console/cake Admin updateObjectTemplates >/tmp/cronlog 2>/tmp/cronlog +45 3 * * * www-data /var/www/MISP/app/Console/cake Admin updateObjectTemplates "$CRON_USER_ID" >/tmp/cronlog 2>/tmp/cronlog EOF diff --git a/server/files/entrypoint_internal.sh b/server/files/entrypoint_internal.sh new file mode 100755 index 0000000..14c6b20 --- /dev/null +++ b/server/files/entrypoint_internal.sh @@ -0,0 +1,195 @@ +#!/bin/bash + +[ -z "$ADMIN_EMAIL" ] && ADMIN_EMAIL="admin@admin.test" +[ -z "$GPG_PASSPHRASE" ] && GPG_PASSPHRASE="passphrase" + +init_gnupg() { + GPG_DIR=/var/www/MISP/.gnupg + GPG_ASC=/var/www/MISP/app/webroot/gpg.asc + GPG_TMP=/tmp/gpg.tmp + + if [ ! -d ${GPG_DIR} ]; then + echo "Generating GPG key ... (please be patient, we need some entropy)" + cat >${GPG_TMP} < ${GPG_ASC} + else + echo "Found exported key ${GPG_ASC}" + fi + + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "GnuPG.email" "${ADMIN_EMAIL}" + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "GnuPG.homedir" "${GPG_DIR}" + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "GnuPG.password" "${GPG_PASSPHRASE}" + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "GnuPG.obscure_subject" false + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "GnuPG.binary" "$(which gpg)" +} + +apply_updates() { + # Disable weird default + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" false + # Run updates + sudo -u www-data /var/www/MISP/app/Console/cake Admin runUpdates +} + +init_user() { + # Create the main user if it is not there already + sudo -u www-data /var/www/MISP/app/Console/cake userInit -q + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "MISP.email" ${ADMIN_EMAIL} + echo 'UPDATE misp.users SET change_pw = 0 WHERE id = 1;' | ${MYSQLCMD} + echo "UPDATE misp.users SET email = \"${ADMIN_EMAIL}\" WHERE id = 1;" | ${MYSQLCMD} + if [ ! -z "$ADMIN_ORG" ]; then + echo "UPDATE misp.organisations SET name = \"${ADMIN_ORG}\" where id = 1;" | ${MYSQLCMD} + fi + if [ ! -z "$ADMIN_KEY" ]; then + echo "Customize MISP | Setting admin key to '${ADMIN_KEY}'" + CHANGE_CMD=(sudo -u www-data /var/www/MISP/app/Console/cake User change_authkey 1 "${ADMIN_KEY}") + else + echo "Customize MISP | Regenerating admin key" + CHANGE_CMD=(sudo -u www-data /var/www/MISP/app/Console/cake User change_authkey 1) + fi + ADMIN_KEY=`${CHANGE_CMD[@]} | awk 'END {print $NF; exit}'` + echo "Customize MISP | Admin user key set to '${ADMIN_KEY}'" +} + +apply_critical_fixes() { + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "MISP.external_baseurl" "${HOSTNAME}" + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "MISP.host_org_id" 1 + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "Plugin.Action_services_enable" false + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "Plugin.Enrichment_hover_enable" true + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "Plugin.Enrichment_hover_popover_only" false + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "Security.csp_enforce" true +} + +apply_custom_settings() { + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting --force "MISP.welcome_text_top" "" + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting --force "MISP.welcome_text_bottom" "" + + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "MISP.contact" "${ADMIN_EMAIL}" + # This is not necessary because we update the DB directly + # sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "MISP.org" "${ADMIN_ORG}" + + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "MISP.log_client_ip" true + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "MISP.log_user_ips" true + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "MISP.log_user_ips_authkeys" true + + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "Plugin.Enrichment_timeout" 30 + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "Plugin.Enrichment_hover_timeout" 5 +} + +configure_plugins() { + if [ ! -z "$VIRUSTOTAL_KEY" ]; then + echo "Customize MISP | Enabling 'virustotal' module ..." + sudo -u www-data php /var/www/MISP/tests/modify_config.php modify "{ + \"Plugin\": { + \"Enrichment_virustotal_enabled\": true, + \"Enrichment_virustotal_apikey\": \"${VIRUSTOTAL_KEY}\" + } + }" > /dev/null + fi + + if [ ! -z "$VIRUSTOTAL_KEY" ] && [ ! -z "$NSX_ANALYSIS_KEY" ] && [ ! -z "$NSX_ANALYSIS_API_TOKEN" ] && [ ! -z "$ADMIN_KEY" ]; then + echo "Customize MISP | Enabling 'vmware_nsx' module ..." + sudo -u www-data php /var/www/MISP/tests/modify_config.php modify "{ + \"Plugin\": { + \"Enrichment_vmware_nsx_enabled\": true, + \"Enrichment_vmware_nsx_analysis_verify_ssl\": \"True\", + \"Enrichment_vmware_nsx_analysis_key\": \"${NSX_ANALYSIS_KEY}\", + \"Enrichment_vmware_nsx_analysis_api_token\": \"${NSX_ANALYSIS_API_TOKEN}\", + \"Enrichment_vmware_nsx_vt_key\": \"${VIRUSTOTAL_KEY}\", + \"Enrichment_vmware_nsx_misp_url\": \"${HOSTNAME}\", + \"Enrichment_vmware_nsx_misp_verify_ssl\": \"False\", + \"Enrichment_vmware_nsx_misp_key\": \"${ADMIN_KEY}\" + } + }" > /dev/null + fi +} + +configure_email() { + sudo -u www-data tee /var/www/MISP/app/Config/email.php > /dev/null < 'Smtp', + 'from' => array('misp-dev@admin.test' => 'Misp DEV'), + 'host' => 'mail', + 'port' => 25, + 'timeout' => 30, + 'client' => null, + 'log' => false, + ); + public \$smtp = array( + 'transport' => 'Smtp', + 'from' => array('misp-dev@admin.test' => 'Misp DEV'), + 'host' => 'mail', + 'port' => 25, + 'timeout' => 30, + 'client' => null, + 'log' => false, + ); + public \$fast = array( + 'from' => 'misp-dev@admin.test', + 'sender' => null, + 'to' => null, + 'cc' => null, + 'bcc' => null, + 'replyTo' => null, + 'readReceipt' => null, + 'returnPath' => null, + 'messageId' => true, + 'subject' => null, + 'message' => null, + 'headers' => null, + 'viewRender' => null, + 'template' => false, + 'layout' => false, + 'viewVars' => null, + 'attachments' => null, + 'emailFormat' => null, + 'transport' => 'Smtp', + 'host' => 'mail', + 'port' => 25, + 'timeout' => 30, + 'client' => null, + 'log' => true, + ); +} +EOT +} + + +echo "Customize MISP | Configure email ..." && configure_email + +echo "Customize MISP | Configure GPG key ..." && init_gnupg + +echo "Customize MISP | Running updates ..." && apply_updates + +echo "Customize MISP | Init default user and organization ..." && init_user + +echo "Customize MISP | Resolve critical issues ..." && apply_critical_fixes + +echo "Customize MISP | Customize installation ..." && apply_custom_settings + +# This item last so we had a chance to create the ADMIN_KEY if not specified +echo "Customize MISP | Configure plugins ..." && configure_plugins + +# Make the instance live +sudo -u www-data /var/www/MISP/app/Console/cake Admin live 1 diff --git a/server/files/entrypoint_nginx.sh b/server/files/entrypoint_nginx.sh index 1eb0f46..a69b954 100755 --- a/server/files/entrypoint_nginx.sh +++ b/server/files/entrypoint_nginx.sh @@ -13,11 +13,6 @@ MISP_APP_CONFIG_PATH=/var/www/MISP/app/Config ENTRYPOINT_PID_FILE="/entrypoint_apache.install" [ ! -f $ENTRYPOINT_PID_FILE ] && touch $ENTRYPOINT_PID_FILE -setup_cake_config(){ - sed -i "s/'host' => 'localhost'.*/'host' => '$REDIS_FQDN', \/\/ Redis server hostname/" "/var/www/MISP/app/Plugin/CakeResque/Config/config.php" - sed -i "s/'host' => '127.0.0.1'.*/'host' => '$REDIS_FQDN', \/\/ Redis server hostname/" "/var/www/MISP/app/Plugin/CakeResque/Config/config.php" -} - init_misp_config(){ [ -f $MISP_APP_CONFIG_PATH/bootstrap.php ] || cp $MISP_APP_CONFIG_PATH.dist/bootstrap.default.php $MISP_APP_CONFIG_PATH/bootstrap.php [ -f $MISP_APP_CONFIG_PATH/database.php ] || cp $MISP_APP_CONFIG_PATH.dist/database.default.php $MISP_APP_CONFIG_PATH/database.php @@ -50,11 +45,20 @@ init_misp_config(){ /var/www/MISP/app/Console/cake Admin setSetting "Plugin.Export_services_url" "$MISP_MODULES_FQDN" /var/www/MISP/app/Console/cake Admin setSetting "Plugin.Cortex_services_enable" false +} - echo Change number of workers - if [ ! -z "$WORKERS" ] && [ "$WORKERS" -gt "1" ]; then - sed -i "s/start --interval/start -n $WORKERS --interval/" /var/www/MISP/app/Console/worker/start.sh - fi +init_misp_workers(){ + # Note that we are doing this after enforcing permissions, so we need to use the www-data user for this + echo "Configuring background workers" + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "SimpleBackgroundJobs.enabled" true + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "SimpleBackgroundJobs.supervisor_host" "127.0.0.1" + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "SimpleBackgroundJobs.supervisor_port" 9001 + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "SimpleBackgroundJobs.supervisor_password" "supervisor" + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "SimpleBackgroundJobs.supervisor_user" "supervisor" + sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting "SimpleBackgroundJobs.redis_host" "$REDIS_FQDN" + + echo "Starting background workers" + supervisorctl start misp-workers:* } init_misp_files(){ @@ -127,22 +131,22 @@ if [[ "$INIT" == true ]]; then echo "Ensure SSL certs exist..." && init_ssl fi -# Things that should ALWAYS happen -echo "Configure Cake | Change Redis host to $REDIS_FQDN ... " && setup_cake_config - # Things we should do if we're configuring MISP via ENV echo "Configure MISP | Initialize misp base config..." && init_misp_config echo "Configure MISP | Sync app files..." && sync_files echo "Configure MISP | Enforce permissions ..." -echo "... chown -R www-data.www-data /var/www/MISP ..." && find /var/www/MISP -not -user www-data -exec chown www-data.www-data {} + +echo "... chown -R www-data:www-data /var/www/MISP ..." && find /var/www/MISP -not -user www-data -exec chown www-data:www-data {} + echo "... chmod -R 0750 /var/www/MISP ..." && find /var/www/MISP -perm 550 -type f -exec chmod 0550 {} + && find /var/www/MISP -perm 770 -type d -exec chmod 0770 {} + echo "... chmod -R g+ws /var/www/MISP/app/tmp ..." && chmod -R g+ws /var/www/MISP/app/tmp echo "... chmod -R g+ws /var/www/MISP/app/files ..." && chmod -R g+ws /var/www/MISP/app/files echo "... chmod -R g+ws /var/www/MISP/app/files/scripts/tmp ..." && chmod -R g+ws /var/www/MISP/app/files/scripts/tmp echo "... chmod 600 /var/www/MISP/app/Config/config.php /var/www/MISP/app/Config/database.php /var/www/MISP/app/Config/email.php ... " && chmod 600 /var/www/MISP/app/Config/config.php /var/www/MISP/app/Config/database.php /var/www/MISP/app/Config/email.php +# Workers are set to NOT auto start so we have time to enforce permissions on the cache first +echo "Configure MISP | Starting workers ..." && init_misp_workers + # Work around https://github.com/MISP/MISP/issues/5608 if [[ ! -f /var/www/MISP/PyMISP/pymisp/data/describeTypes.json ]]; then mkdir -p /var/www/MISP/PyMISP/pymisp/data/ @@ -195,6 +199,18 @@ if [[ -x /custom-entrypoint.sh ]]; then /custom-entrypoint.sh fi +if [[ -x /entrypoint_internal.sh ]]; then + export ADMIN_EMAIL=${ADMIN_EMAIL} + export ADMIN_ORG=${ADMIN_ORG} + export GPG_PASSPHRASE=${GPG_PASSPHRASE} + export HOSTNAME=${HOSTNAME} + export MYSQLCMD=${MYSQLCMD} + export NSX_ANALYSIS_API_TOKEN=${NSX_ANALYSIS_API_TOKEN} + export NSX_ANALYSIS_KEY=${NSX_ANALYSIS_KEY} + export VIRUSTOTAL_KEY=${VIRUSTOTAL_KEY} + /entrypoint_internal.sh +fi + # delete pid file [ -f $ENTRYPOINT_PID_FILE ] && rm $ENTRYPOINT_PID_FILE diff --git a/server/files/entrypoint_workers.sh b/server/files/entrypoint_workers.sh deleted file mode 100755 index 2e23fbb..0000000 --- a/server/files/entrypoint_workers.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -# Wait until entrypoint apache is ready -while (true) -do - sleep 2 - [ -f /entrypoint_apache.install ] && continue - break -done - -while true -do - echo "Start Workers..." - sudo -u www-data /var/www/MISP/app/Console/worker/start.sh - echo "Start Workers...finished" - sleep 3600 -done diff --git a/server/files/etc/supervisor/supervisor.conf b/server/files/etc/supervisor/supervisor.conf index ed23d98..0ed47f1 100644 --- a/server/files/etc/supervisor/supervisor.conf +++ b/server/files/etc/supervisor/supervisor.conf @@ -6,6 +6,11 @@ stdout_logfile_maxbytes=0 stderr_logfile=/dev/stderr stderr_logfile_maxbytes=0 +[inet_http_server] +port=127.0.0.1:9001 +username=supervisor +password=supervisor + [program:nginx] command=/entrypoint_nginx.sh autorestart=true @@ -25,16 +30,7 @@ redirect_stderr=true stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 -[program:workers] -command=/entrypoint_workers.sh -stdout_logfile=/dev/stdout -stdout_logfile_maxbytes=0 -stderr_logfile=/dev/stderr -stderr_logfile_maxbytes=0 -autostart=true - [program:cron] -environment=CRON_USER_ID=%(ENV_CRON_USER_ID)s command=/entrypoint_cron.sh stdout_logfile=/dev/stdout stdout_logfile_maxbytes=0 diff --git a/server/files/etc/supervisor/workers.conf b/server/files/etc/supervisor/workers.conf new file mode 100644 index 0000000..e42013e --- /dev/null +++ b/server/files/etc/supervisor/workers.conf @@ -0,0 +1,68 @@ +# Workers are set to NOT auto start so we have time to enforce permissions on the cache first + +[group:misp-workers] +programs=default,email,cache,prio,update + +[program:default] +directory=/var/www/MISP +command=/var/www/MISP/app/Console/cake start_worker default +process_name=%(program_name)s_%(process_num)02d +numprocs=%(ENV_NUM_WORKERS_DEFAULT)s +autostart=false +autorestart=true +redirect_stderr=false +stderr_logfile=/var/www/MISP/app/tmp/logs/misp-workers-errors.log +stdout_logfile=/var/www/MISP/app/tmp/logs/misp-workers.log +directory=/var/www/MISP +user=www-data + +[program:prio] +directory=/var/www/MISP +command=/var/www/MISP/app/Console/cake start_worker prio +process_name=%(program_name)s_%(process_num)02d +numprocs=%(ENV_NUM_WORKERS_PRIO)s +autostart=false +autorestart=true +redirect_stderr=false +stderr_logfile=/var/www/MISP/app/tmp/logs/misp-workers-errors.log +stdout_logfile=/var/www/MISP/app/tmp/logs/misp-workers.log +directory=/var/www/MISP +user=www-data + +[program:email] +directory=/var/www/MISP +command=/var/www/MISP/app/Console/cake start_worker email +process_name=%(program_name)s_%(process_num)02d +numprocs=%(ENV_NUM_WORKERS_EMAIL)s +autostart=false +autorestart=true +redirect_stderr=false +stderr_logfile=/var/www/MISP/app/tmp/logs/misp-workers-errors.log +stdout_logfile=/var/www/MISP/app/tmp/logs/misp-workers.log +directory=/var/www/MISP +user=www-data + +[program:update] +directory=/var/www/MISP +command=/var/www/MISP/app/Console/cake start_worker update +process_name=%(program_name)s_%(process_num)02d +numprocs=%(ENV_NUM_WORKERS_UPDATE)s +autostart=false +autorestart=true +redirect_stderr=false +stderr_logfile=/var/www/MISP/app/tmp/logs/misp-workers-errors.log +stdout_logfile=/var/www/MISP/app/tmp/logs/misp-workers.log +directory=/var/www/MISP +user=www-data + +[program:cache] +directory=/var/www/MISP +command=/var/www/MISP/app/Console/cake start_worker cache +process_name=%(program_name)s_%(process_num)02d +numprocs=%(ENV_NUM_WORKERS_CACHE)s +autostart=false +autorestart=true +redirect_stderr=false +stderr_logfile=/var/www/MISP/app/tmp/logs/misp-workers-errors.log +stdout_logfile=/var/www/MISP/app/tmp/logs/misp-workers.log +user=www-data diff --git a/template.env b/template.env new file mode 100644 index 0000000..14128aa --- /dev/null +++ b/template.env @@ -0,0 +1,21 @@ +MISP_TAG=v2.4.161 +MODULES_TAG=v2.4.160 +PHP_VER=20190902 +# MISP_COMMIT takes precedence over MISP_TAG +# MISP_COMMIT=c56d537 +# MODULES_COMMIT takes precedence over MODULES_TAG +# MODULES_COMMIT=de69ae3 + +ADMIN_EMAIL= +ADMIN_ORG= +ADMIN_KEY= +GPG_PASSPHRASE= +NSX_ANALYSIS_API_TOKEN= +NSX_ANALYSIS_KEY= +VIRUSTOTAL_KEY= + +SMARTHOST_ADDRESS= +SMARTHOST_PORT= +SMARTHOST_USER= +SMARTHOST_PASSWORD= +SMARTHOST_ALIASES=