Bring forward when database updates occur (#76)

Remove await_settings_db() entirely

Linebuffer some outputs so they look nicer

Move redis specific config items to minimum_config*json

Add start_interval to docker-compose.yml to avoid runUpdates race condition caused by health check which could lead to bad db updates, which seems to have been an issue for quite a while but is very hard to reproduce

core/files/configure_misp.sh

@@ -23,25 +23,20 @@ export GPG_BINARY="$(which gpg)"
 export SETTING_CONTACT="${MISP_CONTACT-$ADMIN_EMAIL}"
 export SETTING_EMAIL="${MISP_EMAIL-$ADMIN_EMAIL}"
 
-init_cli_only_config() {
-    # I think no matter what we do, we should wait for this table to turn up.
-    # Only really impacts us on first run, and on my machine only takes a few seconds to turn up.
-    # TODO: this is not the right solution because `system_settings` is not part of the original dump
-    # await_system_settings_table
-    # Temporarily disable DB to apply cli_only settings, since these MUST be in the config.php file (by design or otherwise)
-    # This will reenable upon init_settings "db_enable" below if it is indeed enabled
+init_minimum_config() {
+    # Temporarily disable DB to apply config file settings, reenable after if needed 
     sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting -q "MISP.system_setting_db" false
-    init_settings "cli_only"
-    init_settings "db_enable"
+    init_settings "minimum_config"
 }
 
 init_configuration() {
+    init_settings "db_enable"
     init_settings "initialisation"
 }
 
 init_workers() {
     echo "... starting background workers"
-    supervisorctl start misp-workers:*
+    stdbuf -oL supervisorctl start misp-workers:*
 }
 
 configure_gnupg() {
@@ -215,16 +210,16 @@ set_up_proxy() {
 
 apply_updates() {
     # Disable 'ZeroMQ_enable' to get better logs when applying updates
-    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting -q "Plugin.ZeroMQ_enable" false
+#    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting -q "Plugin.ZeroMQ_enable" false
     # Run updates (strip colors since output might end up in a log)
-    sudo -u www-data /var/www/MISP/app/Console/cake Admin runUpdates | sed -r "s/[[:cntrl:]]\[[0-9]{1,3}m//g"
+    sudo -u www-data /var/www/MISP/app/Console/cake Admin runUpdates | stdbuf -oL sed -r "s/[[:cntrl:]]\[[0-9]{1,3}m//g"
     # Re-enable 'ZeroMQ_enable'
-    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting -q "Plugin.ZeroMQ_enable" true
+#    sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting -q "Plugin.ZeroMQ_enable" true
 }
 
 init_user() {
     # Create the main user if it is not there already
-    sudo -u www-data /var/www/MISP/app/Console/cake user init -q 2>&1 > /dev/null
+    sudo -u www-data /var/www/MISP/app/Console/cake user init -q > /dev/null 2>&1
 
     echo "UPDATE misp.users SET email = \"${ADMIN_EMAIL}\" WHERE id = 1;" | ${MYSQLCMD}
 
@@ -250,7 +245,7 @@ init_user() {
     if [ ! -z "$ADMIN_PASSWORD" ]; then
         echo "... setting admin password to '${ADMIN_PASSWORD}'"
         PASSWORD_POLICY=$(sudo -u www-data /var/www/MISP/app/Console/cake Admin getSetting "Security.password_policy_complexity" | jq ".value" -r)
-        PASSWORD_LENGTH=$(sudo -u www-data /var/www/MISP/app/Console/cake Admin getSetting "Security.password_policy_length" | jq ".value")
+        PASSWORD_LENGTH=$(sudo -u www-data /var/www/MISP/app/Console/cake Admin getSetting "Security.password_policy_length" | jq ".value" -r)
         sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting -q "Security.password_policy_length" 1
         sudo -u www-data /var/www/MISP/app/Console/cake Admin setSetting -q "Security.password_policy_complexity" '/.*/'
         sudo -u www-data /var/www/MISP/app/Console/cake User change_pw "${ADMIN_EMAIL}" "${ADMIN_PASSWORD}"
@@ -366,13 +361,6 @@ init_settings() {
     fi
 }
 
-await_system_settings_table() {
-    until [[ $(echo "SELECT EXISTS(SELECT 1 FROM information_schema.tables WHERE table_schema = '$MYSQL_DATABASE' and table_name = 'system_settings');" | ${MYSQLCMD}) -eq 1 ]]; do
-        echo "... awaiting availability of system_settings table"
-        sleep 2
-    done
-}
-
 update_components() {
     sudo -u www-data /var/www/MISP/app/Console/cake Admin updateGalaxies
     sudo -u www-data /var/www/MISP/app/Console/cake Admin updateTaxonomies
@@ -440,7 +428,9 @@ create_sync_servers() {
 
 echo "MISP | Update CA certificates ..." && update_ca_certificates
 
-echo "MISP | CLI_only configuration directives ..." && init_cli_only_config
+echo "MISP | Apply minimum configuration directives ..." && init_minimum_config
+
+echo "MISP | Apply DB updates ..." && apply_updates
 
 echo "MISP | Initialize configuration ..." && init_configuration
 
@@ -448,8 +438,6 @@ echo "MISP | Initialize workers ..." && init_workers
 
 echo "MISP | Configure GPG key ..." && configure_gnupg
 
-echo "MISP | Apply updates ..." && apply_updates
-
 echo "MISP | Init default user and organization ..." && init_user
 
 echo "MISP | Resolve critical issues ..." && apply_critical_fixes

core/files/etc/misp-docker/initialisation.defaults.json

@@ -2,15 +2,6 @@
   "MISP.ca_path": {
     "default_value": "/var/www/MISP/app/Lib/cakephp/lib/Cake/Config/cacert.pem"
   },
-  "MISP.redis_port": {
-    "default_value": 6379
-  },
-  "MISP.redis_database": {
-    "default_value": 13
-  },
-  "MISP.redis_password": {
-    "default_value": ""
-  },
   "MISP.language": {
     "default_value": "eng"
   },
@@ -113,6 +104,10 @@
   "Security.check_sec_fetch_site_header": {
     "default_value": true
   },
+  "Security.encryption_key": {
+    "default_value": "",
+    "command_args": "-f"
+  },
   "Security.username_in_response_header": {
     "default_value": true
   },

core/files/etc/misp-docker/initialisation.envars.json

@@ -12,9 +12,6 @@
   "MISP.contact": {
     "default_value": "${SETTING_CONTACT}"
   },
-  "MISP.redis_host": {
-    "default_value": "${REDIS_FQDN}"
-  },
   "Plugin.ZeroMQ_redis_host": {
     "default_value": "${REDIS_FQDN}"
   },

core/files/etc/misp-docker/minimum_config.defaults.json

@@ -24,6 +24,15 @@
     "default_value": "/etc/ssl/certs/ca-certificates.crt",
     "command_args": "-f"
   },
+  "MISP.redis_port": {
+    "default_value": 6379
+  },
+  "MISP.redis_database": {
+    "default_value": 13
+  },
+  "MISP.redis_password": {
+    "default_value": ""
+  },
   "MISP.menu_custom_right_link": {
     "default_value": ""
   },

core/files/etc/misp-docker/minimum_config.envars.json

@@ -2,6 +2,9 @@
   "MISP.python_bin": {
     "default_value": "${PYTHON_BIN}"
   },
+  "MISP.redis_host": {
+    "default_value": "${REDIS_FQDN}"
+  },
   "GnuPG.binary": {
     "default_value": "${GPG_BINARY}"
   },

docker-compose.yml

@@ -68,6 +68,7 @@ services:
       timeout: 1s
       retries: 3
       start_period: 30s
+      start_interval: 30s
     ports:
       - "80:80"
       - "443:443"