From ae52b9d6ae3d914d7b450c76d096372a545e33d7 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Thu, 23 Dec 2021 11:55:55 +0100 Subject: [PATCH] fix: broken stix install and other minor fixes --- docker-compose.yml | 2 + web/Dockerfile | 3 +- web/INSTALL_NODB.sh | 20 +++- web/INSTALL_NODB.sh.sha1 | 2 +- web/INSTALL_NODB.sh.sha256 | 2 +- web/INSTALL_NODB.sh.sha384 | 2 +- web/INSTALL_NODB.sh.sha512 | 2 +- web/wait-for-it.sh | 186 +++++++++++++++++++++++++++++++++++++ 8 files changed, 210 insertions(+), 9 deletions(-) create mode 100755 web/wait-for-it.sh diff --git a/docker-compose.yml b/docker-compose.yml index 2d87ef2..e5d118b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,6 +12,7 @@ services: - "80:80" - "443:443" volumes: + - ./web/wait-for-it.sh:/usr/local/bin/wait-for-it.sh:ro - /dev/urandom:/dev/random - ${DATA_DIR}/web:/var/www/MISP environment: @@ -24,6 +25,7 @@ services: - MISP_BASEURL=${MISP_BASEURL} - POSTFIX_RELAY_HOST=${POSTFIX_RELAY_HOST} - TIMEZONE=${TIMEZONE} + entrypoint: "wait-for-it.sh -t 0 -h db -p 3306 -- /run.sh" db: container_name: misp_db diff --git a/web/Dockerfile b/web/Dockerfile index 9730811..6c98335 100644 --- a/web/Dockerfile +++ b/web/Dockerfile @@ -50,19 +50,18 @@ RUN ( \ echo ''; \ echo '[program:redis-server]'; \ echo 'command=redis-server /etc/redis/redis.conf'; \ + echo 'user=redis'; \ echo ''; \ echo '[program:apache2]'; \ echo 'command=/bin/bash -c "source /etc/apache2/envvars && exec /usr/sbin/apache2 -D FOREGROUND"'; \ echo ''; \ echo '[program:resque]'; \ echo 'command=/bin/bash /var/www/MISP/app/Console/worker/start.sh'; \ - echo 'user = www-data'; \ echo 'startsecs = 0'; \ echo 'autorestart = false'; \ echo ''; \ echo '[program:misp-modules]'; \ echo 'command=/bin/bash -c "/var/www/MISP/venv/bin/misp-modules -l 127.0.0.1 -s"'; \ - echo 'user = www-data'; \ echo 'startsecs = 0'; \ echo 'autorestart = false'; \ ) >> /etc/supervisor/conf.d/supervisord.conf diff --git a/web/INSTALL_NODB.sh b/web/INSTALL_NODB.sh index a90ff12..c3e7fcd 100755 --- a/web/INSTALL_NODB.sh +++ b/web/INSTALL_NODB.sh @@ -1334,6 +1334,11 @@ installCore () { sudo mkdir /var/www/.cache/ sudo chown ${WWW_USER}:${WWW_USER} /var/www/.cache + # install python-stix dependencies + ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ordered-set python-dateutil six weakrefmethod + debug "Install misp-stix" + ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/app/files/scripts/misp-stix + debug "Install PyMISP" ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/PyMISP @@ -1470,6 +1475,9 @@ coreCAKE () { $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Session.timeout" 600 $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Session.cookieTimeout" 3600 + # Set the default temp dir + ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.tmpdir" "${PATH_TO_MISP}/app/tmp" + # Change base url, either with this CLI command or in the UI $SUDO_WWW $RUN_PHP -- $CAKE Baseurl $MISP_BASEURL # example: 'baseurl' => 'https://', @@ -1489,7 +1497,7 @@ coreCAKE () { # Enable installer org and tune some configurables $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.host_org_id" 1 $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.email" "info@admin.test" - $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.disable_emailing" true + $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.disable_emailing" true --force $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.contact" "info@admin.test" $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.disablerestalert" true $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.showCorrelationsOnIndex" true @@ -1500,7 +1508,7 @@ coreCAKE () { $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Cortex_services_url" "http://127.0.0.1" $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Cortex_services_port" 9000 $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Cortex_timeout" 120 - $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Cortex_authkey" "" + $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Cortex_authkey" false $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Cortex_ssl_verify_peer" false $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Cortex_ssl_verify_host" false $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.Cortex_ssl_allow_self_signed" true @@ -1525,7 +1533,7 @@ coreCAKE () { $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.RPZ_minimum_ttl" "1h" $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.RPZ_ttl" "1w" $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.RPZ_ns" "localhost." - $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.RPZ_ns_alt" "" + $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.RPZ_ns_alt" false $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "Plugin.RPZ_email" "root.localhost" # Force defaults to make MISP Server Settings less RED @@ -1557,6 +1565,9 @@ coreCAKE () { $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.block_old_event_alert" false $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.block_old_event_alert_age" "" $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.block_old_event_alert_by_date" "" + $SUDO_WWW $RUN_PHP -- ${CAKE} Admin setSetting "MISP.event_alert_republish_ban" false + $SUDO_WWW $RUN_PHP -- ${CAKE} Admin setSetting "MISP.event_alert_republish_ban_threshold" 5 + $SUDO_WWW $RUN_PHP -- ${CAKE} Admin setSetting "MISP.event_alert_republish_ban_refresh_on_retry" false $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.incoming_tags_disabled_by_default" false $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.maintenance_message" "Great things are happening! MISP is undergoing maintenance, but will return shortly. You can contact the administration at \$email." $SUDO_WWW $RUN_PHP -- $CAKE Admin setSetting "MISP.footermidleft" "This is an initial install" @@ -2058,6 +2069,9 @@ installCoreRHEL () { UMASK=$(umask) umask 0022 + # install python-stix dependencies + $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install ordered-set python-dateutil six weakrefmethod + # install zmq $SUDO_WWW $PATH_TO_MISP/venv/bin/pip install -U zmq diff --git a/web/INSTALL_NODB.sh.sha1 b/web/INSTALL_NODB.sh.sha1 index 83b7771..1e74d1e 100644 --- a/web/INSTALL_NODB.sh.sha1 +++ b/web/INSTALL_NODB.sh.sha1 @@ -1 +1 @@ -79e122609f2fe35feaeadbb1c59bc7aac790a31b INSTALL_NODB.sh +3ee3d8d50c3a765f6f7838a3c3e433cd20b30e14 INSTALL_NODB.sh diff --git a/web/INSTALL_NODB.sh.sha256 b/web/INSTALL_NODB.sh.sha256 index 468bacd..863f25e 100644 --- a/web/INSTALL_NODB.sh.sha256 +++ b/web/INSTALL_NODB.sh.sha256 @@ -1 +1 @@ -fc624837cfba356c19d5db1b6a1f2249600c5573096ab5d8b8886b21fe89bbff INSTALL_NODB.sh +bcc523d529a9415876b1b9b8eead0d223a329d4a420711a8a506fda09b6949a8 INSTALL_NODB.sh diff --git a/web/INSTALL_NODB.sh.sha384 b/web/INSTALL_NODB.sh.sha384 index cbaa083..024f550 100644 --- a/web/INSTALL_NODB.sh.sha384 +++ b/web/INSTALL_NODB.sh.sha384 @@ -1 +1 @@ -a542aa41ff5d31e40dc0cbbf72601feda622aacc8e6bbee327d362aa0528547db2c38090e3a74df7be2470e2609ad1f7 INSTALL_NODB.sh +15c6e8bf7f3c8f49975f25e74ec39f63409ab5c7539eb934a8c6aa69a29225bb337437822b5049c222405590584302d7 INSTALL_NODB.sh diff --git a/web/INSTALL_NODB.sh.sha512 b/web/INSTALL_NODB.sh.sha512 index 47ac9d0..e449b71 100644 --- a/web/INSTALL_NODB.sh.sha512 +++ b/web/INSTALL_NODB.sh.sha512 @@ -1 +1 @@ -8ac0e4194ad2dc187bf06c89dac4bab972c517ac02e7d67f3ef6e87ce8039de550d34fad80429e011ff50650906b303c86367e58d6d61f1f910bfdabcd570309 INSTALL_NODB.sh +fde55c9f924f0481ab51b30df328bf254638b6dd1b87d2e0ee181b074f4c5ff359185cd60321fb1bc0928ad46722ac345db315e448585fa16be85059eac82808 INSTALL_NODB.sh diff --git a/web/wait-for-it.sh b/web/wait-for-it.sh new file mode 100755 index 0000000..65b7f1f --- /dev/null +++ b/web/wait-for-it.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +# The MIT License (MIT) +# Copyright (c) 2016 Giles Hall +# Source: https://github.com/vishnubob/wait-for-it + +WAITFORIT_cmdname=${0##*/} + +echoerr() { if [[ $WAITFORIT_QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $WAITFORIT_cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + echoerr "$WAITFORIT_cmdname: waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + else + echoerr "$WAITFORIT_cmdname: waiting for $WAITFORIT_HOST:$WAITFORIT_PORT without a timeout" + fi + WAITFORIT_start_ts=$(date +%s) + while : + do + if [[ $WAITFORIT_ISBUSY -eq 1 ]]; then + nc -z $WAITFORIT_HOST $WAITFORIT_PORT + WAITFORIT_result=$? + else + (echo -n > /dev/tcp/$WAITFORIT_HOST/$WAITFORIT_PORT) >/dev/null 2>&1 + WAITFORIT_result=$? + fi + if [[ $WAITFORIT_result -eq 0 ]]; then + WAITFORIT_end_ts=$(date +%s) + echoerr "$WAITFORIT_cmdname: $WAITFORIT_HOST:$WAITFORIT_PORT is available after $((WAITFORIT_end_ts - WAITFORIT_start_ts)) seconds" + break + fi + sleep 1 + done + return $WAITFORIT_result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $WAITFORIT_QUIET -eq 1 ]]; then + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --quiet --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + else + timeout $WAITFORIT_BUSYTIMEFLAG $WAITFORIT_TIMEOUT $0 --child --host=$WAITFORIT_HOST --port=$WAITFORIT_PORT --timeout=$WAITFORIT_TIMEOUT & + fi + WAITFORIT_PID=$! + trap "kill -INT -$WAITFORIT_PID" INT + wait $WAITFORIT_PID + WAITFORIT_RESULT=$? + if [[ $WAITFORIT_RESULT -ne 0 ]]; then + echoerr "$WAITFORIT_cmdname: timeout occurred after waiting $WAITFORIT_TIMEOUT seconds for $WAITFORIT_HOST:$WAITFORIT_PORT" + fi + return $WAITFORIT_RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + WAITFORIT_hostport=(${1//:/ }) + WAITFORIT_HOST=${WAITFORIT_hostport[0]} + WAITFORIT_PORT=${WAITFORIT_hostport[1]} + shift 1 + ;; + --child) + WAITFORIT_CHILD=1 + shift 1 + ;; + -q | --quiet) + WAITFORIT_QUIET=1 + shift 1 + ;; + -s | --strict) + WAITFORIT_STRICT=1 + shift 1 + ;; + -h) + WAITFORIT_HOST="$2" + if [[ $WAITFORIT_HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + WAITFORIT_HOST="${1#*=}" + shift 1 + ;; + -p) + WAITFORIT_PORT="$2" + if [[ $WAITFORIT_PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + WAITFORIT_PORT="${1#*=}" + shift 1 + ;; + -t) + WAITFORIT_TIMEOUT="$2" + if [[ $WAITFORIT_TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + WAITFORIT_TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + WAITFORIT_CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$WAITFORIT_HOST" == "" || "$WAITFORIT_PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +WAITFORIT_TIMEOUT=${WAITFORIT_TIMEOUT:-15} +WAITFORIT_STRICT=${WAITFORIT_STRICT:-0} +WAITFORIT_CHILD=${WAITFORIT_CHILD:-0} +WAITFORIT_QUIET=${WAITFORIT_QUIET:-0} + +# Check to see if timeout is from busybox? +WAITFORIT_TIMEOUT_PATH=$(type -p timeout) +WAITFORIT_TIMEOUT_PATH=$(realpath $WAITFORIT_TIMEOUT_PATH 2>/dev/null || readlink -f $WAITFORIT_TIMEOUT_PATH) + +WAITFORIT_BUSYTIMEFLAG="" +if [[ $WAITFORIT_TIMEOUT_PATH =~ "busybox" ]]; then + WAITFORIT_ISBUSY=1 + # Check if busybox timeout uses -t flag + # (recent Alpine versions don't support -t anymore) + if timeout &>/dev/stdout | grep -q -e '-t '; then + WAITFORIT_BUSYTIMEFLAG="-t" + fi +else + WAITFORIT_ISBUSY=0 +fi + +if [[ $WAITFORIT_CHILD -gt 0 ]]; then + wait_for + WAITFORIT_RESULT=$? + exit $WAITFORIT_RESULT +else + if [[ $WAITFORIT_TIMEOUT -gt 0 ]]; then + wait_for_wrapper + WAITFORIT_RESULT=$? + else + wait_for + WAITFORIT_RESULT=$? + fi +fi + +if [[ $WAITFORIT_CLI != "" ]]; then + if [[ $WAITFORIT_RESULT -ne 0 && $WAITFORIT_STRICT -eq 1 ]]; then + echoerr "$WAITFORIT_cmdname: strict mode, refusing to execute subprocess" + exit $WAITFORIT_RESULT + fi + exec "${WAITFORIT_CLI[@]}" +else + exit $WAITFORIT_RESULT +fi