From 5b623198ca327f70d0714c02b43aa4360d872bf9 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Thu, 9 Sep 2021 14:17:57 +0200
Subject: [PATCH 01/45] chg: [internal] Deprecate Org::getUUIDs endpoint

---
 app/Controller/OrganisationsController.php    | 27 +++++++++----------
 .../Organisations/ajax/fetch_orgs_for_sg.ctp  |  4 +--
 2 files changed, 15 insertions(+), 16 deletions(-)

diff --git a/app/Controller/OrganisationsController.php b/app/Controller/OrganisationsController.php
index 705b2736f..78ca7dc79 100644
--- a/app/Controller/OrganisationsController.php
+++ b/app/Controller/OrganisationsController.php
@@ -372,19 +372,15 @@ class OrganisationsController extends AppController
         }
         $idList = json_decode($idList, true);
         $id_exclusion_list = array_merge($idList, array($this->Auth->user('Organisation')['id']));
-        $temp = $this->Organisation->find('all', array(
-                'conditions' => array(
-                        'local' => $local,
-                        'id !=' => $id_exclusion_list,
-                ),
-                'recursive' => -1,
-                'fields' => array('id', 'name'),
-                'order' => array('lower(name) ASC')
+        $orgs = $this->Organisation->find('list', array(
+            'conditions' => array(
+                'local' => $local,
+                'id !=' => $id_exclusion_list,
+            ),
+            'recursive' => -1,
+            'fields' => array('id', 'name'),
+            'order' => array('lower(name) ASC')
         ));
-        $orgs = array();
-        foreach ($temp as $org) {
-            $orgs[] = array('id' => $org['Organisation']['id'], 'name' => $org['Organisation']['name']);
-        }
         $this->set('local', $local);
         $this->layout = false;
         $this->autoRender = false;
@@ -402,10 +398,13 @@ class OrganisationsController extends AppController
         $this->render('ajax/sg_org_row_empty');
     }
 
+    /**
+     * @deprecated Probably not used anywhere.
+     */
     public function getUUIDs()
     {
-        if (!$this->Auth->user('Role')['perm_sync']) {
-            throw new MethodNotAllowedException(__('This action is restricted to sync users'));
+        if (Configure::read('Security.hide_organisation_index_from_users')) {
+            throw new MethodNotAllowedException(__('This action is not enabled on this instance.'));
         }
         $temp = $this->Organisation->find('all', array(
                 'recursive' => -1,
diff --git a/app/View/Organisations/ajax/fetch_orgs_for_sg.ctp b/app/View/Organisations/ajax/fetch_orgs_for_sg.ctp
index 8516215ca..e3124eda6 100644
--- a/app/View/Organisations/ajax/fetch_orgs_for_sg.ctp
+++ b/app/View/Organisations/ajax/fetch_orgs_for_sg.ctp
@@ -7,8 +7,8 @@
                     <p><?php echo __('Available Organisations');?></p>
                     <select id="leftValues" size="5" multiple style="width:285px;">
                         <?php
-                            foreach ($orgs as $org) {
-                                echo '<option value="' . h($org['id']) . '" selected>' . h($org['name']) . '</option>';
+                            foreach ($orgs as $orgId => $orgName) {
+                                echo '<option value="' . intval($orgId) . '" selected>' . h($orgName) . '</option>';
                             }
                         ?>
                     </select>

From 665c96035e98785581d8c2a280011c88ab94e652 Mon Sep 17 00:00:00 2001
From: Steve Clement <steve@localhost.lu>
Date: Tue, 2 Nov 2021 12:22:33 +0100
Subject: [PATCH 02/45] new: [doc] Initial php8.0 and Ubuntu 22.04

---
 docs/INSTALL.ubuntu2204.md | 527 +++++++++++++++++++++++++++++++++++++
 1 file changed, 527 insertions(+)
 create mode 100644 docs/INSTALL.ubuntu2204.md

diff --git a/docs/INSTALL.ubuntu2204.md b/docs/INSTALL.ubuntu2204.md
new file mode 100644
index 000000000..04f198088
--- /dev/null
+++ b/docs/INSTALL.ubuntu2204.md
@@ -0,0 +1,527 @@
+# INSTALLATION INSTRUCTIONS
+## for Ubuntu 22.04-server
+
+{!generic/manual-install-notes.md!}
+
+### -1/ Installer and Manual install instructions
+
+Make sure you are reading the parsed version of this Document. When in doubt [click here](https://misp.github.io/MISP/INSTALL.ubuntu2004/).
+
+### 0/ MISP Ubuntu 20.04-server install - status
+-------------------------
+!!! notice
+    Installer tested working by [@SteveClement](https://twitter.com/SteveClement) on 20211002
+
+!!! notice
+    If the next line is `[!generic/core.md!]()` [click here](https://misp.github.io/MISP/INSTALL.ubuntu2204/).
+
+{!generic/core.md!}
+
+### 1/ Minimal Ubuntu install
+-------------------------
+
+#### Install a minimal Ubuntu 20.04-server system with the software:
+- OpenSSH server
+- This guide assumes a user name of 'misp' with sudo working but can be overwritten by setting the environment variable: *${MISP_USER}*
+
+#### Make sure your system is up2date
+```bash
+# <snippet-begin 0_apt-upgrade.sh>
+aptUpgrade () {
+  debug "Upgrading system"
+  checkAptLock
+
+  # If we run in non-interactive mode, make sure we do not stop all of a sudden
+  if [[ "${PACKER}" == "1" || "${UNATTENDED}" == "1" ]]; then
+    export DEBIAN_FRONTEND=noninteractive
+    export DEBIAN_PRIORITY=critical
+    sudo -E apt-get -qy -o "Dpkg::Options::=--force-confdef" -o "Dpkg::Options::=--force-confold" upgrade
+    sudo -E apt-get -qy autoclean
+  else
+    sudo apt-get upgrade -qy
+  fi
+}
+# <snippet-end 0_apt-upgrade.sh>
+```
+
+{!generic/sudo_etckeeper.md!}
+
+{!generic/ethX.md!}
+
+#### install postfix, there will be some questions.
+```bash
+# <snippet-begin postfix.sh>
+sudo apt-get install postfix dialog -qy
+# <snippet-end postfix.sh>
+```
+
+!!! notice
+    Postfix Configuration: Satellite system<br />
+    change the relay server later with:
+    ```bash
+    sudo postconf -e 'relayhost = example.com'
+    sudo postfix reload
+    ```
+
+{!generic/globalVariables.md!}
+
+### 2/ Install LAMP & dependencies
+------------------------------
+Once the system is installed you can perform the following steps.
+```bash
+# <snippet-begin 0_installCoreDeps.sh>
+installCoreDeps () {
+  debug "Installing core dependencies"
+  # Install the dependencies: (some might already be installed)
+  sudo apt-get install curl gcc git gpg-agent make python python3 openssl redis-server sudo vim zip unzip virtualenv libfuzzy-dev sqlite3 moreutils -qy
+
+  # Install MariaDB (a MySQL fork/alternative)
+  sudo apt-get install mariadb-client mariadb-server -qy
+
+  # Install Apache2
+  sudo apt-get install apache2 apache2-doc apache2-utils -qy
+
+  # install Mitre's STIX and its dependencies by running the following commands:
+  sudo apt-get install python3-dev python3-pip libxml2-dev libxslt1-dev zlib1g-dev python-setuptools -qy
+}
+# <snippet-end 0_installCoreDeps.sh>
+
+# <snippet-begin 0_installDepsPhp80.sh>
+# Install Php 8.0 dependencies
+installDepsPhp80 () {
+  debug "Installing PHP 8.0 dependencies"
+  PHP_ETC_BASE=/etc/php/8.0
+  PHP_INI=${PHP_ETC_BASE}/apache2/php.ini
+  checkAptLock
+  sudo apt install -qy \
+  libapache2-mod-php8.0 \
+  php8.0 php8.0-cli \
+  php8.0-dev \
+  php-json php8.0-xml php8.0-mysql php8.0-opcache php8.0-readline php8.0-mbstring php8.0-zip \
+  php8.0-redis php-gnupg \
+  php8.0-intl php8.0-bcmath \
+  php8.0-gd
+
+  for key in upload_max_filesize post_max_size max_execution_time max_input_time memory_limit
+  do
+      sudo sed -i "s/^\($key\).*/\1 = $(eval echo \${$key})/" $PHP_INI
+  done
+  sudo sed -i "s/^\(session.sid_length\).*/\1 = $(eval echo \${session0sid_length})/" $PHP_INI
+  sudo sed -i "s/^\(session.use_strict_mode\).*/\1 = $(eval echo \${session0use_strict_mode})/" $PHP_INI
+}
+# <snippet-end 0_installDepsPhp80.sh>
+```
+
+### 3/ MISP code
+------------
+```bash
+# <snippet-begin 1_mispCoreInstall.sh>
+installCore () {
+  debug "Installing ${LBLUE}MISP${NC} core"
+  # Download MISP using git in the /var/www/ directory.
+  if [[ ! -d ${PATH_TO_MISP} ]]; then
+    sudo mkdir ${PATH_TO_MISP}
+    sudo chown ${WWW_USER}:${WWW_USER} ${PATH_TO_MISP}
+    false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git clone https://github.com/MISP/MISP.git ${PATH_TO_MISP}; done
+    false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} git -C ${PATH_TO_MISP} submodule update --progress --init --recursive; done
+    # Make git ignore filesystem permission differences for submodules
+    ${SUDO_WWW} git -C ${PATH_TO_MISP} submodule foreach --recursive git config core.filemode false
+
+    # Make git ignore filesystem permission differences
+    ${SUDO_WWW} git -C ${PATH_TO_MISP} config core.filemode false
+
+    # Create a python3 virtualenv
+    ${SUDO_WWW} virtualenv -p python3 ${PATH_TO_MISP}/venv
+
+    # make pip happy
+    sudo mkdir /var/www/.cache/
+    sudo chown ${WWW_USER}:${WWW_USER} /var/www/.cache
+
+    # install python-stix dependencies
+    ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ordered-set python-dateutil six weakrefmethod
+
+    debug "Install PyMISP"
+    ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install ${PATH_TO_MISP}/PyMISP
+
+    # FIXME: Remove libfaup etc once the egg has the library baked-in
+    sudo apt-get install cmake libcaca-dev liblua5.3-dev -y
+    cd /tmp
+    false; while [[ $? -ne 0 ]]; do [[ ! -d "faup" ]] && ${SUDO_CMD} git clone https://github.com/stricaud/faup.git faup; done
+    false; while [[ $? -ne 0 ]]; do [[ ! -d "gtcaca" ]] && ${SUDO_CMD} git clone https://github.com/stricaud/gtcaca.git gtcaca; done
+    sudo chown -R ${MISP_USER}:${MISP_USER} faup gtcaca
+    cd gtcaca
+    ${SUDO_CMD} mkdir -p build
+    cd build
+    ${SUDO_CMD} cmake .. && ${SUDO_CMD} make
+    sudo make install
+    cd ../../faup
+    ${SUDO_CMD} mkdir -p build
+    cd build
+    ${SUDO_CMD} cmake .. && ${SUDO_CMD} make
+    sudo make install
+    sudo ldconfig
+
+    # install pydeep
+    false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install git+https://github.com/kbandla/pydeep.git; done
+
+    # install lief
+    ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install lief
+
+    # install zmq needed by mispzmq
+    ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install zmq redis
+
+    # install python-magic
+    ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install python-magic
+
+    # install plyara
+    ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install plyara
+  else
+    debug "Trying to git pull existing install"
+    ${SUDO_WWW} git pull -C ${PATH_TO_MISP}
+    false; while [[ $? -ne 0 ]]; do ${SUDO_WWW} git -C ${PATH_TO_MISP} submodule update --progress --init --recursive; done
+
+    ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U ${PATH_TO_MISP}/PyMISP
+    false; while [[ $? -ne 0 ]]; do checkAptLock; ${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install -U git+https://github.com/kbandla/pydeep.git; done
+fi
+}
+# <snippet-end 1_mispCoreInstall.sh>
+```
+
+### 4/ CakePHP
+-----------
+
+```bash
+# <snippet-begin 1_installCake.sh>
+installCake () {
+  debug "Installing CakePHP"
+  # Make composer cache happy
+  # /!\ composer on Ubuntu when invoked with sudo -u doesn't set $HOME to /var/www but keeps it /home/misp \!/
+  sudo mkdir -p /var/www/.composer ; sudo chown ${WWW_USER}:${WWW_USER} /var/www/.composer
+  ${SUDO_WWW} sh -c "cd ${PATH_TO_MISP}/app ;php composer.phar install --no-dev"
+
+  # Enable CakeResque with php-redis
+  sudo phpenmod redis
+  sudo phpenmod gnupg
+
+  # To use the scheduler worker for scheduled tasks, do the following:
+  ${SUDO_WWW} cp -fa ${PATH_TO_MISP}/INSTALL/setup/config.php ${PATH_TO_MISP}/app/Plugin/CakeResque/Config/config.php
+
+  # If you have multiple MISP instances on the same system, don't forget to have a different Redis per MISP instance for the CakeResque workers
+  # The default Redis port can be updated in Plugin/CakeResque/Config/config.php
+}
+# <snippet-end 1_installCake.sh>
+```
+
+### 5/ Set the permissions
+----------------------
+
+```bash
+# <snippet-begin 2_permissions.sh>
+# Main function to fix permissions to something sane
+permissions () {
+  debug "Setting permissions"
+  sudo chown -R ${WWW_USER}:${WWW_USER} ${PATH_TO_MISP}
+  sudo chmod -R 750 ${PATH_TO_MISP}
+  sudo chmod -R g+ws ${PATH_TO_MISP}/app/tmp
+  sudo chmod -R g+ws ${PATH_TO_MISP}/app/files
+  sudo chmod -R g+ws ${PATH_TO_MISP}/app/files/scripts/tmp
+}
+# <snippet-end 2_permissions.sh>
+```
+
+### 6/ Create a database and user
+-----------------------------
+
+#### Set-up DB, User and import empty MISP DB
+
+```bash
+# <snippet-begin 1_prepareDB.sh>
+prepareDB () {
+  if sudo test ! -e "/var/lib/mysql/mysql/"; then
+    #Make sure initial tables are created in MySQL
+    debug "Install mysql tables"
+    sudo mysql_install_db --user=mysql --basedir=/usr --datadir=/var/lib/mysql
+    sudo service mysql start
+  fi
+
+  if sudo test ! -e "/var/lib/mysql/misp/"; then
+    debug "Start mysql"
+    sudo service mysql start
+
+    debug "Setting up database"
+    # Kill the anonymous users
+    sudo mysql -h $DBHOST -e "DROP USER IF EXISTS ''@'localhost'"
+    # Because our hostname varies we'll use some Bash magic here.
+    sudo mysql -h $DBHOST -e "DROP USER IF EXISTS ''@'$(hostname)'"
+    # Kill off the demo database
+    sudo mysql -h $DBHOST -e "DROP DATABASE IF EXISTS test"
+    # No root remote logins
+    sudo mysql -h $DBHOST -e "DELETE FROM mysql.user WHERE User='root' AND Host NOT IN ('localhost', '127.0.0.1', '::1')"
+    # Make sure that NOBODY can access the server without a password
+    sudo mysqladmin -h $DBHOST -u "${DBUSER_ADMIN}" password "${DBPASSWORD_ADMIN}"
+    # Make our changes take effect
+    sudo mysql -h $DBHOST -e "FLUSH PRIVILEGES"
+
+    sudo mysql -h $DBHOST -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "CREATE DATABASE ${DBNAME};"
+    sudo mysql -h $DBHOST -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "CREATE USER '${DBUSER_MISP}'@'localhost' IDENTIFIED BY '${DBPASSWORD_MISP}';"
+    sudo mysql -h $DBHOST -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "GRANT USAGE ON *.* to '${DBUSER_MISP}'@'localhost';"
+    sudo mysql -h $DBHOST -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "GRANT ALL PRIVILEGES on ${DBNAME}.* to '${DBUSER_MISP}'@'localhost';"
+    sudo mysql -h $DBHOST -u "${DBUSER_ADMIN}" -p"${DBPASSWORD_ADMIN}" -e "FLUSH PRIVILEGES;"
+    # Import the empty MISP database from MYSQL.sql
+    ${SUDO_WWW} cat ${PATH_TO_MISP}/INSTALL/MYSQL.sql | mysql -h $DBHOST -u "${DBUSER_MISP}" -p"${DBPASSWORD_MISP}" ${DBNAME}
+  fi
+}
+# <snippet-end 1_prepareDB.sh>
+```
+
+### 7/ Apache configuration
+-----------------------
+Now configure your Apache webserver with the DocumentRoot ${PATH_TO_MISP}/app/webroot/
+
+#### Apache version 2.4 config:
+
+!!! notice
+    Be aware that the configuration files for apache 2.4 and up have changed.
+    The configuration file has to have the .conf extension in the sites-available directory
+    For more information, visit http://httpd.apache.org/docs/2.4/upgrading.html
+
+```bash
+# <snippet-begin 1_apacheConfig.sh>
+apacheConfig () {
+  debug "Generating Apache config, if this hangs, make sure you have enough entropy (install: haveged or wait)"
+  sudo cp ${PATH_TO_MISP}/INSTALL/apache.24.misp.ssl /etc/apache2/sites-available/misp-ssl.conf
+
+  if [[ ! -z ${MISP_BASEURL} ]] && [[ "$(echo $MISP_BASEURL|cut -f 1 -d :)" == "http" || "$(echo $MISP_BASEURL|cut -f 1 -d :)" == "https" ]]; then
+
+    echo "Potentially replacing misp.local with $MISP_BASEURL in misp-ssl.conf"
+
+  fi
+
+  # If a valid SSL certificate is not already created for the server,
+  # create a self-signed certificate:
+  sudo openssl req -newkey rsa:4096 -days 365 -nodes -x509 \
+  -subj "/C=${OPENSSL_C}/ST=${OPENSSL_ST}/L=${OPENSSL_L}/O=${OPENSSL_O}/OU=${OPENSSL_OU}/CN=${OPENSSL_CN}/emailAddress=${OPENSSL_EMAILADDRESS}" \
+  -keyout /etc/ssl/private/misp.local.key -out /etc/ssl/private/misp.local.crt
+
+  # Enable modules, settings, and default of SSL in Apache
+  sudo a2dismod status
+  sudo a2enmod ssl
+  sudo a2enmod rewrite
+  sudo a2enmod headers
+  sudo a2dissite 000-default
+  sudo a2ensite default-ssl
+
+  # Apply all changes
+  sudo systemctl restart apache2
+  # activate new vhost
+  sudo a2dissite default-ssl
+  sudo a2ensite misp-ssl
+
+  # Restart apache
+  sudo systemctl restart apache2
+}
+# <snippet-end 1_apacheConfig.sh>
+```
+
+!!! notice
+    Please find a sample conf file for an SSL enabled conf file in-line below (alternatively use one of the samples provided in /var/www/MISP/INSTALL).<br />
+    Also remember to verify the SSLCertificateChainFile property in your config file.<br />
+    This is usually commented out for the self-generated certificate in the sample configurations, such as the one pasted below.<br />
+    Otherwise, copy the SSLCertificateFile, SSLCertificateKeyFile, and SSLCertificateChainFile to /etc/ssl/private/. (Modify path and config to fit your environment)
+
+```
+============================================= Begin sample working SSL config for MISP
+<VirtualHost <IP, FQDN, or *>:80>
+        ServerName <your.FQDN.here>
+
+        Redirect permanent / https://<your.FQDN.here>
+
+        LogLevel warn
+        ErrorLog /var/log/apache2/misp.local_error.log
+        CustomLog /var/log/apache2/misp.local_access.log combined
+        ServerSignature Off
+</VirtualHost>
+
+<VirtualHost <IP, FQDN, or *>:443>
+        ServerAdmin admin@<your.FQDN.here>
+        ServerName <your.FQDN.here>
+        DocumentRoot /var/www/MISP/app/webroot
+        <Directory /var/www/MISP/app/webroot>
+                Options -Indexes
+                AllowOverride all
+                Order allow,deny
+                allow from all
+        </Directory>
+
+        SSLEngine On
+        SSLCertificateFile /etc/ssl/private/misp.local.crt
+        SSLCertificateKeyFile /etc/ssl/private/misp.local.key
+#        SSLCertificateChainFile /etc/ssl/private/misp-chain.crt
+
+        LogLevel warn
+        ErrorLog /var/log/apache2/misp.local_error.log
+        CustomLog /var/log/apache2/misp.local_access.log combined
+        ServerSignature Off
+</VirtualHost>
+============================================= End sample working SSL config for MISP
+```
+
+### 8/ Log rotation
+---------------
+```bash
+# <snippet-begin 2_logRotation.sh>
+logRotation () {
+  # MISP saves the stdout and stderr of its workers in ${PATH_TO_MISP}/app/tmp/logs
+  # To rotate these logs install the supplied logrotate script:
+  sudo cp ${PATH_TO_MISP}/INSTALL/misp.logrotate /etc/logrotate.d/misp
+  sudo chmod 0640 /etc/logrotate.d/misp
+}
+# <snippet-end 2_logRotation.sh>
+```
+
+### 9/ MISP configuration
+---------------------
+```bash
+# <snippet-begin 2_configMISP.sh>
+configMISP () {
+  debug "Generating ${LBLUE}MISP${NC} config files"
+  # There are 4 sample configuration files in ${PATH_TO_MISP}/app/Config that need to be copied
+  ${SUDO_WWW} cp -a ${PATH_TO_MISP}/app/Config/bootstrap.default.php ${PATH_TO_MISP}/app/Config/bootstrap.php
+  ${SUDO_WWW} cp -a ${PATH_TO_MISP}/app/Config/database.default.php ${PATH_TO_MISP}/app/Config/database.php
+  ${SUDO_WWW} cp -a ${PATH_TO_MISP}/app/Config/core.default.php ${PATH_TO_MISP}/app/Config/core.php
+  ${SUDO_WWW} cp -a ${PATH_TO_MISP}/app/Config/config.default.php ${PATH_TO_MISP}/app/Config/config.php
+
+  echo "<?php
+  class DATABASE_CONFIG {
+          public \$default = array(
+                  'datasource' => 'Database/Mysql',
+                  //'datasource' => 'Database/Postgres',
+                  'persistent' => false,
+                  'host' => '$DBHOST',
+                  'login' => '$DBUSER_MISP',
+                  'port' => 3306, // MySQL & MariaDB
+                  //'port' => 5432, // PostgreSQL
+                  'password' => '$DBPASSWORD_MISP',
+                  'database' => '$DBNAME',
+                  'prefix' => '',
+                  'encoding' => 'utf8',
+          );
+  }" | ${SUDO_WWW} tee ${PATH_TO_MISP}/app/Config/database.php
+
+  # Important! Change the salt key in ${PATH_TO_MISP}/app/Config/config.php
+  # The salt key must be a string at least 32 bytes long.
+  # The admin user account will be generated on the first login, make sure that the salt is changed before you create that user
+  # If you forget to do this step, and you are still dealing with a fresh installation, just alter the salt,
+  # delete the user from mysql and log in again using the default admin credentials (admin@admin.test / admin)
+
+  # and make sure the file permissions are still OK
+  sudo chown -R ${WWW_USER}:${WWW_USER} ${PATH_TO_MISP}/app/Config
+  sudo chmod -R 750 ${PATH_TO_MISP}/app/Config
+}
+# <snippet-end 2_configMISP.sh>
+```
+
+{!generic/gnupg.md!}
+
+!!! notice
+    If entropy is not high enough, you can install havegd and then start the service
+    ```bash
+    sudo apt install haveged -qy
+    sudo service haveged start
+    ```
+
+```bash
+# <snippet-begin 2_backgroundWorkers.sh>
+backgroundWorkers () {
+  debug "Setting up background workers"
+  # To make the background workers start on boot
+  sudo chmod +x ${PATH_TO_MISP}/app/Console/worker/start.sh
+
+  if [ ! -e /etc/rc.local ]
+  then
+      echo '#!/bin/sh -e' | sudo tee -a /etc/rc.local
+      echo 'exit 0' | sudo tee -a /etc/rc.local
+      sudo chmod u+x /etc/rc.local
+  fi
+
+  echo "[Unit]
+Description=MISP background workers
+After=network.target
+
+[Service]
+Type=forking
+User=${WWW_USER}
+Group=${WWW_USER}
+ExecStart=${PATH_TO_MISP}/app/Console/worker/start.sh
+Restart=always
+RestartSec=10
+
+[Install]
+WantedBy=multi-user.target" | sudo tee /etc/systemd/system/misp-workers.service
+
+  sudo systemctl daemon-reload
+  sudo systemctl enable --now misp-workers
+
+  # Add the following lines before the last line (exit 0). Make sure that you replace www-data with your apache user:
+  sudo sed -i -e '$i \echo never > /sys/kernel/mm/transparent_hugepage/enabled\n' /etc/rc.local
+  sudo sed -i -e '$i \echo 1024 > /proc/sys/net/core/somaxconn\n' /etc/rc.local
+  sudo sed -i -e '$i \sysctl vm.overcommit_memory=1\n' /etc/rc.local
+}
+# <snippet-end 2_backgroundWorkers.sh>
+```
+
+```bash
+echo "Admin (root) DB Password: $DBPASSWORD_ADMIN"
+echo "User  (misp) DB Password: $DBPASSWORD_MISP"
+```
+
+{!generic/MISP_CAKE_init.md!}
+
+{!generic/misp-modules-debian.md!}
+
+{!generic/misp-modules-cake.md!}
+
+{!generic/INSTALL.done.md!}
+
+{!generic/recommended.actions.md!}
+
+### Optional features
+-----------------
+#### MISP has a new pub/sub feature, using ZeroMQ. To enable it, simply run the following command
+```bash
+${SUDO_WWW} ${PATH_TO_MISP}/venv/bin/pip install pyzmq
+```
+
+#### MISP has a feature for publishing events to Kafka. To enable it, simply run the following commands
+```bash
+# <snippet-begin 4_kafka.sh>
+installKafka () {
+  sudo apt-get install librdkafka-dev php-dev -y
+  sudo pecl channel-update pecl.php.net
+  sudo pecl install rdkafka
+  echo "extension=rdkafka.so" | sudo tee ${PHP_ETC_BASE}/mods-available/rdkafka.ini
+  sudo phpenmod rdkafka
+  sudo service apache2 restart
+}
+# <snippet-end 4_kafka.sh>
+```
+
+{!generic/misp-dashboard-debian.md!}
+
+{!generic/misp-dashboard-cake.md!}
+
+{!generic/viper-debian.md!}
+
+{!generic/ssdeep-debian.md!}
+
+{!generic/mail_to_misp-debian.md!}
+
+{!generic/hardening.md!}
+
+# INSTALL.sh
+
+!!! notice
+    The following section is an administrative section that is used by the "[INSTALL.sh](https://raw.githubusercontent.com/MISP/MISP/2.4/INSTALL/INSTALL.sh)" script.
+    Please ignore.
+
+{!generic/supportFunctions.md!}

From 8135ee7ccd383f22223a6ee54f77727c67b1696c Mon Sep 17 00:00:00 2001
From: StefanKelm <StefanKelm@users.noreply.github.com>
Date: Tue, 2 Nov 2021 12:56:33 +0100
Subject: [PATCH 03/45] Update openapi.yaml

tiny typo...
---
 app/webroot/doc/openapi.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/webroot/doc/openapi.yaml b/app/webroot/doc/openapi.yaml
index 4a643aede..0d9dbd8d3 100644
--- a/app/webroot/doc/openapi.yaml
+++ b/app/webroot/doc/openapi.yaml
@@ -6082,7 +6082,7 @@ components:
                 format: date
                 example: "2021-03-05"
               org:
-                description: "Filter events by matching an the creator organisation name"
+                description: "Filter events by matching the creator organisation name"
                 type: string
                 nullable: true
                 example: "CIRCL"

From 98234d827f695ff7bb3fecca958925a829c185c8 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Thu, 4 Nov 2021 11:04:30 +0100
Subject: [PATCH 04/45] chg: [server settings] allow empty baseurl to be saved

---
 app/Model/Server.php | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/app/Model/Server.php b/app/Model/Server.php
index 6e7b6425e..306607391 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -1668,11 +1668,14 @@ class Server extends AppModel
             }
             return true;
         }
+        if (empty($value)) {
+            return true;
+        }
         if ($this->testForEmpty($value) !== true) {
             return $this->testForEmpty($value);
         }
         $regex = "/^(?<proto>https?):\/\/(?<host>([\w,\-,\.]+))(?::(?<port>[0-9]+))?(?<base>\/[a-z0-9_\-\.]+)?$/i";
-	if (
+        if (
             !preg_match($regex, $value, $matches) ||
             strtolower($matches['proto']) != strtolower($this->getProto()) ||
             (
@@ -4505,6 +4508,7 @@ class Server extends AppModel
                     'errorMessage' => __('The currently set baseurl does not match the URL through which you have accessed the page. Disregard this if you are accessing the page via an alternate URL (for example via IP address).'),
                     'test' => 'testBaseURL',
                     'type' => 'string',
+                    'null' => true
                 ),
                 'external_baseurl' => array(
                     'level' => 0,

From eddf95355a4bd32eab6ccbccea5849f152e672b0 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Thu, 4 Nov 2021 11:04:56 +0100
Subject: [PATCH 05/45] fix: [database] upgrade script using mb4 defaulted to
 255 key length

- default should be 191
---
 app/Model/AppModel.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php
index e70fd97e4..057568e93 100644
--- a/app/Model/AppModel.php
+++ b/app/Model/AppModel.php
@@ -1534,7 +1534,7 @@ class AppModel extends Model
                     `value` text NOT NULL,
                     `from_json` tinyint(1) default 0,
                     PRIMARY KEY (`id`),
-                    UNIQUE INDEX `value` (`value`(255))
+                    UNIQUE INDEX `value` (`value`(191))
                 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;";
                 break;
             case 66:

From e25efe2d17383893e960990bdd431f21aedb8bce Mon Sep 17 00:00:00 2001
From: Steve Clement <steve@localhost.lu>
Date: Thu, 4 Nov 2021 15:25:54 +0100
Subject: [PATCH 06/45] chg: [doc] Varios small fixes

---
 INSTALL/INSTALL.tpl.sh                        |  4 ----
 docs/generic/MISP_CAKE_init.md                | 11 ++++++----
 docs/generic/globalVariables.md               |  3 ++-
 docs/generic/misp-modules-debian.md           |  1 +
 ...L.ubuntu2204.md => xINSTALL.ubuntu2204.md} | 21 +++++++++++--------
 mkdocs.yml                                    |  1 +
 6 files changed, 23 insertions(+), 18 deletions(-)
 rename docs/{INSTALL.ubuntu2204.md => xINSTALL.ubuntu2204.md} (96%)

diff --git a/INSTALL/INSTALL.tpl.sh b/INSTALL/INSTALL.tpl.sh
index 8369623c8..9a970ce84 100755
--- a/INSTALL/INSTALL.tpl.sh
+++ b/INSTALL/INSTALL.tpl.sh
@@ -260,10 +260,6 @@ installSupported () {
   echo "Proceeding with the installation of MISP core"
   space
 
-  # Set Base URL - functionLocation('generic/supportFunctions.md')
-  [[ -n $CORE ]]   || [[ -n $ALL ]] && setBaseURL
-  progress 4
-
   # Check if sudo is installed and etckeeper - functionLocation('generic/sudo_etckeeper.md')
   [[ -n $CORE ]]   || [[ -n $ALL ]] && checkSudoKeeper
   [[ ! -z ${MISP_USER} ]] && [[ ! -f /etc/sudoers.d/misp ]] && echo "%${MISP_USER} ALL=(ALL:ALL) NOPASSWD:ALL" |sudo tee /etc/sudoers.d/misp
diff --git a/docs/generic/MISP_CAKE_init.md b/docs/generic/MISP_CAKE_init.md
index b6fd02a23..cecf4fe65 100644
--- a/docs/generic/MISP_CAKE_init.md
+++ b/docs/generic/MISP_CAKE_init.md
@@ -25,6 +25,9 @@ coreCAKE () {
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Session.autoRegenerate" 0
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Session.timeout" 600
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Session.cookieTimeout" 3600
+ 
+  # Set the default temp dir
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.tmpdir" "${PATH_TO_MISP}/app/tmp"
 
   # Change base url, either with this CLI command or in the UI
   [[ ! -z ${MISP_BASEURL} ]] && ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Baseurl $MISP_BASEURL
@@ -46,7 +49,7 @@ coreCAKE () {
   # Enable installer org and tune some configurables
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.host_org_id" 1
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.email" "info@admin.test"
-  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.disable_emailing" true
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.disable_emailing" true --force
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.contact" "info@admin.test"
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.disablerestalert" true
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.showCorrelationsOnIndex" true
@@ -57,7 +60,7 @@ coreCAKE () {
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Cortex_services_url" "http://127.0.0.1"
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Cortex_services_port" 9000
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Cortex_timeout" 120
-  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Cortex_authkey" ""
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Cortex_authkey" false
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Cortex_ssl_verify_peer" false
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Cortex_ssl_verify_host" false
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Cortex_ssl_allow_self_signed" true
@@ -116,7 +119,7 @@ coreCAKE () {
          Plugin.ElasticSearch_logging_enable
          Plugin.S3_enable)
   for PLUG in "${PLUGS[@]}"; do
-    ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting ${PLUG} false
+    ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting ${PLUG} false 2> /dev/null
   done
 
   # Plugin CustomAuth tuneable
@@ -132,7 +135,7 @@ coreCAKE () {
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.RPZ_minimum_ttl" "1h"
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.RPZ_ttl" "1w"
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.RPZ_ns" "localhost."
-  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.RPZ_ns_alt" ""
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.RPZ_ns_alt" false
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.RPZ_email" "root.localhost"
 
   # Kafka settings
diff --git a/docs/generic/globalVariables.md b/docs/generic/globalVariables.md
index bca8b71b7..a6f06f18a 100644
--- a/docs/generic/globalVariables.md
+++ b/docs/generic/globalVariables.md
@@ -59,7 +59,8 @@ MISPvars () {
   # MISP configuration variables
   PATH_TO_MISP="${PATH_TO_MISP:-/var/www/MISP}"
   PATH_TO_MISP_SCRIPTS="${PATH_TO_MISP}/app/files/scripts"
-
+  ## For future use
+  # TMPDIR="${TMPDIR:-$PATH_TO_MISP/app/tmp}"
 
   FQDN="${FQDN:-misp.local}"
 
diff --git a/docs/generic/misp-modules-debian.md b/docs/generic/misp-modules-debian.md
index 65dbd9a84..c5fedb220 100644
--- a/docs/generic/misp-modules-debian.md
+++ b/docs/generic/misp-modules-debian.md
@@ -38,6 +38,7 @@ mispmodules () {
   # If you build an egg, the user you build it as need write permissions in the CWD
   sudo chgrp $WWW_USER .
   sudo chmod og+w .
+  $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install pillow
   $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I -r REQUIREMENTS
   sudo chgrp staff .
   $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I .
diff --git a/docs/INSTALL.ubuntu2204.md b/docs/xINSTALL.ubuntu2204.md
similarity index 96%
rename from docs/INSTALL.ubuntu2204.md
rename to docs/xINSTALL.ubuntu2204.md
index 04f198088..8bece86c8 100644
--- a/docs/INSTALL.ubuntu2204.md
+++ b/docs/xINSTALL.ubuntu2204.md
@@ -73,7 +73,7 @@ Once the system is installed you can perform the following steps.
 installCoreDeps () {
   debug "Installing core dependencies"
   # Install the dependencies: (some might already be installed)
-  sudo apt-get install curl gcc git gpg-agent make python python3 openssl redis-server sudo vim zip unzip virtualenv libfuzzy-dev sqlite3 moreutils -qy
+  sudo apt-get install curl gcc git gpg-agent make python3 openssl redis-server sudo vim zip unzip virtualenv libfuzzy-dev sqlite3 moreutils -qy
 
   # Install MariaDB (a MySQL fork/alternative)
   sudo apt-get install mariadb-client mariadb-server -qy
@@ -88,19 +88,22 @@ installCoreDeps () {
 
 # <snippet-begin 0_installDepsPhp80.sh>
 # Install Php 8.0 dependencies
+# FIXME: Ugly hack to get 7.4 working until 8.0 (cake4) will be implemented.
+echo "deb http://ppa.launchpad.net/ondrej/php/ubuntu devel main" |sudo tee /etc/apt/sources.list.d/ondrej-ubuntu-php-devel.list
+sudo apt update
 installDepsPhp80 () {
   debug "Installing PHP 8.0 dependencies"
-  PHP_ETC_BASE=/etc/php/8.0
+  PHP_ETC_BASE=/etc/php/7.4
   PHP_INI=${PHP_ETC_BASE}/apache2/php.ini
   checkAptLock
   sudo apt install -qy \
-  libapache2-mod-php8.0 \
-  php8.0 php8.0-cli \
-  php8.0-dev \
-  php-json php8.0-xml php8.0-mysql php8.0-opcache php8.0-readline php8.0-mbstring php8.0-zip \
-  php8.0-redis php-gnupg \
-  php8.0-intl php8.0-bcmath \
-  php8.0-gd
+  libapache2-mod-php7.4 \
+  php7.4 php7.4-cli \
+  php7.4-dev \
+  php-json php7.4-xml php7.4-mysql php7.4-opcache php7.4-readline php7.4-mbstring php7.4-zip \
+  php7.4-redis php-gnupg \
+  php7.4-intl php7.4-bcmath \
+  php7.4-gd
 
   for key in upload_max_filesize post_max_size max_execution_time max_input_time memory_limit
   do
diff --git a/mkdocs.yml b/mkdocs.yml
index 7b4ffd98b..7cb3686a2 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -77,6 +77,7 @@ nav:
       - 'Debian 10': 'xINSTALL.debian10.md'
       - 'Tsurugi Linux': 'xINSTALL.tsurugi.md'
       - 'OpenBSD 7.0': 'xINSTALL.OpenBSD.md'
+      - 'Ubuntu 22.04': 'xINSTALL.ubuntu2204.md'
     - Config Guides:
       - 'Elastic Search Logging': 'CONFIG.elasticsearch-logging.md'
       - 'Amazon S3 attachments': 'CONFIG.s3-attachments.md'

From b6969156f04768901db0d53de5906d5484f59f19 Mon Sep 17 00:00:00 2001
From: Steve Clement <steve@localhost.lu>
Date: Thu, 4 Nov 2021 15:44:31 +0100
Subject: [PATCH 07/45] chg: [install] Update to latest installer

---
 INSTALL/INSTALL.sh        | 19 ++++++++++---------
 INSTALL/INSTALL.sh.sfv    |  6 +++---
 INSTALL/INSTALL.sh.sha1   |  2 +-
 INSTALL/INSTALL.sh.sha256 |  2 +-
 INSTALL/INSTALL.sh.sha384 |  2 +-
 INSTALL/INSTALL.sh.sha512 |  2 +-
 6 files changed, 17 insertions(+), 16 deletions(-)

diff --git a/INSTALL/INSTALL.sh b/INSTALL/INSTALL.sh
index e34c926c5..933675b57 100755
--- a/INSTALL/INSTALL.sh
+++ b/INSTALL/INSTALL.sh
@@ -117,7 +117,8 @@ MISPvars () {
   # MISP configuration variables
   PATH_TO_MISP="${PATH_TO_MISP:-/var/www/MISP}"
   PATH_TO_MISP_SCRIPTS="${PATH_TO_MISP}/app/files/scripts"
-
+  ## For future use
+  # TMPDIR="${TMPDIR:-$PATH_TO_MISP/app/tmp}"
 
   FQDN="${FQDN:-misp.local}"
 
@@ -1541,6 +1542,9 @@ coreCAKE () {
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Session.autoRegenerate" 0
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Session.timeout" 600
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Session.cookieTimeout" 3600
+ 
+  # Set the default temp dir
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.tmpdir" "${PATH_TO_MISP}/app/tmp"
 
   # Change base url, either with this CLI command or in the UI
   [[ ! -z ${MISP_BASEURL} ]] && ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Baseurl $MISP_BASEURL
@@ -1562,7 +1566,7 @@ coreCAKE () {
   # Enable installer org and tune some configurables
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.host_org_id" 1
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.email" "info@admin.test"
-  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.disable_emailing" true
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.disable_emailing" true --force
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.contact" "info@admin.test"
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.disablerestalert" true
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.showCorrelationsOnIndex" true
@@ -1573,7 +1577,7 @@ coreCAKE () {
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Cortex_services_url" "http://127.0.0.1"
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Cortex_services_port" 9000
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Cortex_timeout" 120
-  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Cortex_authkey" ""
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Cortex_authkey" false
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Cortex_ssl_verify_peer" false
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Cortex_ssl_verify_host" false
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Cortex_ssl_allow_self_signed" true
@@ -1632,7 +1636,7 @@ coreCAKE () {
          Plugin.ElasticSearch_logging_enable
          Plugin.S3_enable)
   for PLUG in "${PLUGS[@]}"; do
-    ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting ${PLUG} false
+    ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting ${PLUG} false 2> /dev/null
   done
 
   # Plugin CustomAuth tuneable
@@ -1648,7 +1652,7 @@ coreCAKE () {
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.RPZ_minimum_ttl" "1h"
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.RPZ_ttl" "1w"
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.RPZ_ns" "localhost."
-  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.RPZ_ns_alt" ""
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.RPZ_ns_alt" false
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.RPZ_email" "root.localhost"
 
   # Kafka settings
@@ -1899,6 +1903,7 @@ mispmodules () {
   # If you build an egg, the user you build it as need write permissions in the CWD
   sudo chgrp $WWW_USER .
   sudo chmod og+w .
+  $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install pillow
   $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I -r REQUIREMENTS
   sudo chgrp staff .
   $SUDO_WWW ${PATH_TO_MISP}/venv/bin/pip install -I .
@@ -3045,10 +3050,6 @@ installSupported () {
   echo "Proceeding with the installation of MISP core"
   space
 
-  # Set Base URL - functionLocation('generic/supportFunctions.md')
-  [[ -n $CORE ]]   || [[ -n $ALL ]] && setBaseURL
-  progress 4
-
   # Check if sudo is installed and etckeeper - functionLocation('generic/sudo_etckeeper.md')
   [[ -n $CORE ]]   || [[ -n $ALL ]] && checkSudoKeeper
   [[ ! -z ${MISP_USER} ]] && [[ ! -f /etc/sudoers.d/misp ]] && echo "%${MISP_USER} ALL=(ALL:ALL) NOPASSWD:ALL" |sudo tee /etc/sudoers.d/misp
diff --git a/INSTALL/INSTALL.sh.sfv b/INSTALL/INSTALL.sh.sfv
index f3899ddd5..d0ec619bb 100644
--- a/INSTALL/INSTALL.sh.sfv
+++ b/INSTALL/INSTALL.sh.sfv
@@ -1,5 +1,5 @@
-; Generated by RHash v1.3.9 on 2021-10-18 at 10:56.53
+; Generated by RHash v1.4.2 on 2021-11-04 at 15:44.11
 ; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/
 ;
-;       160201  10:56.53 2021-10-18 INSTALL.sh
-INSTALL.sh 8F59974F7AE69DFBF7B1C492E35F0B421AAC10C1 6F9E9C2C24880D2E69E04AB6AE490F72D8B5CBE5BB98596F4FA50C1CFEAA632F CBCFBA692B57E027A9861C4D4FB1D4808511A23148516946802B0364D428638E60087AD6EA7E2F016B2F65CD216DE288 7221893A49C924974F7D28C094C6CB27FC8ACA6E07FECD7B8DE4D55D283C9D6A5FF63409F55EEC110BF6612E8578BD1373E39B83A7986A6369ACF32A6A92F538
+;       160342  15:44.11 2021-11-04 INSTALL.sh
+INSTALL.sh E10075FB44DD06A1C4248264085BDC8217B900CC 30E5EDCE721AF81B18744CA7B2062147BCF873FB5FE71798B8543EBA52F4FB4C 1E68603F4304D5B4EAA456A6B8A9A79C2CE86C48D595C9DCCD341A0D8959C52A7A9EEF0B3ABDB1C3534023350BC18B64 FAFAE6A7E6BD81C87AA1C90CD52721BF314BAD6BB41B33CF3E1E8070E5DDCA786761A6205AD104BF565DE68E4FF100EC7D55837D4F9CAD60A72825BCFFBE5D65
diff --git a/INSTALL/INSTALL.sh.sha1 b/INSTALL/INSTALL.sh.sha1
index aa6840830..5fb875a9b 100644
--- a/INSTALL/INSTALL.sh.sha1
+++ b/INSTALL/INSTALL.sh.sha1
@@ -1 +1 @@
-8f59974f7ae69dfbf7b1c492e35f0b421aac10c1  INSTALL.sh
+e10075fb44dd06a1c4248264085bdc8217b900cc  INSTALL.sh
diff --git a/INSTALL/INSTALL.sh.sha256 b/INSTALL/INSTALL.sh.sha256
index d44026cb3..bcadaefe7 100644
--- a/INSTALL/INSTALL.sh.sha256
+++ b/INSTALL/INSTALL.sh.sha256
@@ -1 +1 @@
-6f9e9c2c24880d2e69e04ab6ae490f72d8b5cbe5bb98596f4fa50c1cfeaa632f  INSTALL.sh
+30e5edce721af81b18744ca7b2062147bcf873fb5fe71798b8543eba52f4fb4c  INSTALL.sh
diff --git a/INSTALL/INSTALL.sh.sha384 b/INSTALL/INSTALL.sh.sha384
index 808f8420b..d2328815a 100644
--- a/INSTALL/INSTALL.sh.sha384
+++ b/INSTALL/INSTALL.sh.sha384
@@ -1 +1 @@
-cbcfba692b57e027a9861c4d4fb1d4808511a23148516946802b0364d428638e60087ad6ea7e2f016b2f65cd216de288  INSTALL.sh
+1e68603f4304d5b4eaa456a6b8a9a79c2ce86c48d595c9dccd341a0d8959c52a7a9eef0b3abdb1c3534023350bc18b64  INSTALL.sh
diff --git a/INSTALL/INSTALL.sh.sha512 b/INSTALL/INSTALL.sh.sha512
index 8d3437126..9cf5651e2 100644
--- a/INSTALL/INSTALL.sh.sha512
+++ b/INSTALL/INSTALL.sh.sha512
@@ -1 +1 @@
-7221893a49c924974f7d28c094c6cb27fc8aca6e07fecd7b8de4d55d283c9d6a5ff63409f55eec110bf6612e8578bd1373e39b83a7986a6369acf32a6a92f538  INSTALL.sh
+fafae6a7e6bd81c87aa1c90cd52721bf314bad6bb41b33cf3e1e8070e5ddca786761a6205ad104bf565de68e4ff100ec7d55837d4f9cad60a72825bcffbe5d65  INSTALL.sh

From 53be00d250d13b5614503431eec63d71371355b3 Mon Sep 17 00:00:00 2001
From: Steve Clement <steve@localhost.lu>
Date: Thu, 4 Nov 2021 18:31:49 +0100
Subject: [PATCH 08/45] chg: [doc] minor changes for 22.04 and ethX update

---
 docs/generic/ethX.md        | 6 +++++-
 docs/xINSTALL.ubuntu2204.md | 3 ++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/docs/generic/ethX.md b/docs/generic/ethX.md
index 3a0be0b8c..d660a9520 100644
--- a/docs/generic/ethX.md
+++ b/docs/generic/ethX.md
@@ -30,7 +30,7 @@ sudo update-grub  > /dev/null 2>&1
 ```
 
 !!! notice
-    On recent Ubuntu install Netplan is default and you need to change the Network name.
+    On recent Ubuntu install Netplan is default and you might need to change the Network name in its respective config file.
     ```
     sudo sed -i "s/enp0s3/eth0/" /etc/netplan/50-cloud-init.yaml
     ```
@@ -38,3 +38,7 @@ sudo update-grub  > /dev/null 2>&1
     ```
     sudo sed -i "s/enp0s3/eth0/" /etc/netplan/01-netcfg.yaml
     ```
+    OR on Ubuntu 22.04
+    ```
+    sudo sed -i "s/enp0s3/eth0/" /etc/netplan/00-installer-config.yaml
+    ```
diff --git a/docs/xINSTALL.ubuntu2204.md b/docs/xINSTALL.ubuntu2204.md
index 8bece86c8..bd0815cb2 100644
--- a/docs/xINSTALL.ubuntu2204.md
+++ b/docs/xINSTALL.ubuntu2204.md
@@ -90,6 +90,7 @@ installCoreDeps () {
 # Install Php 8.0 dependencies
 # FIXME: Ugly hack to get 7.4 working until 8.0 (cake4) will be implemented.
 echo "deb http://ppa.launchpad.net/ondrej/php/ubuntu devel main" |sudo tee /etc/apt/sources.list.d/ondrej-ubuntu-php-devel.list
+sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 4F4EA0AAE5267A6C
 sudo apt update
 installDepsPhp80 () {
   debug "Installing PHP 8.0 dependencies"
@@ -101,7 +102,7 @@ installDepsPhp80 () {
   php7.4 php7.4-cli \
   php7.4-dev \
   php-json php7.4-xml php7.4-mysql php7.4-opcache php7.4-readline php7.4-mbstring php7.4-zip \
-  php7.4-redis php-gnupg \
+  php-redis php-gnupg \
   php7.4-intl php7.4-bcmath \
   php7.4-gd
 

From f0a75f08d206fc89636ab694ba1b6d99063b2a95 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Fri, 5 Nov 2021 07:43:13 +0100
Subject: [PATCH 09/45] new: [MISP fetcher] added to create an offline update
 package

---
 tools/fetcher/README.md  | 14 ++++++++++++++
 tools/fetcher/fetcher.sh | 18 ++++++++++++++++++
 2 files changed, 32 insertions(+)
 create mode 100644 tools/fetcher/README.md
 create mode 100644 tools/fetcher/fetcher.sh

diff --git a/tools/fetcher/README.md b/tools/fetcher/README.md
new file mode 100644
index 000000000..18658bc0f
--- /dev/null
+++ b/tools/fetcher/README.md
@@ -0,0 +1,14 @@
+# MISP fetcher
+
+Simple shell script to generate a zip file containing MISP with all submodules and composer libraries.
+
+Simply run the script from its directory and use the zip's contents to update an airgapped MISP's codebase.
+
+You will need to have composer installed and accessible
+
+Assuming the standard MISP install path and www-data as your apache user, just run the following to update your MISP
+
+```
+unzip misp_flat.zip /var/www/MISP
+chown -R www-data:www-data /var/www/MISP
+```
diff --git a/tools/fetcher/fetcher.sh b/tools/fetcher/fetcher.sh
new file mode 100644
index 000000000..de3c5f9dd
--- /dev/null
+++ b/tools/fetcher/fetcher.sh
@@ -0,0 +1,18 @@
+# Stupid script to fetch MISP's install files including submodules and composer sourced libraries
+
+# This is currently a relative path, highly recommended to replace with an absolute path
+# For example, if you want the fetcher to work in /foo/bar/baz, use "/foo/bar/baz/MISPflat"
+MISP_FLAT_ROOT="MISPflat"
+
+git clone https://github.com/MISP/MISP.git $MISP_FLAT_ROOT
+cd $MISP_FLAT_ROOT
+git submodule update --init --recursive
+cd ..
+cd $MISP_FLAT_ROOT/app
+composer install --no-dev
+cd ../..
+cd $MISP_FLAT_ROOT
+zip -r ../misp_flat.zip .
+cd ..
+rm -rf $MISP_FLAT_ROOT
+

From c9597ba3e9f9efe2e1965cc9c178ac0a806091e6 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Thu, 4 Nov 2021 16:28:41 +0100
Subject: [PATCH 10/45] new: Store system settings in database

---
 app/Console/AppShell.php                    | 34 -----------------
 app/Console/Command/Task/ConfigLoadTask.php | 15 ++++++--
 app/Controller/AppController.php            | 14 +++++++
 app/Controller/AuditLogsController.php      |  1 +
 app/Controller/ServersController.php        | 20 +++++-----
 app/Model/AppModel.php                      |  9 ++++-
 app/Model/AuditLog.php                      |  2 +
 app/Model/Behavior/AuditLogBehavior.php     |  9 +++--
 app/Model/Server.php                        | 25 ++++++++-----
 app/Model/SystemSetting.php                 | 41 +++++++++++++++++++++
 app/View/Servers/server_settings.ctp        | 21 +++++------
 db_schema.json                              | 29 ++++++++++++++-
 12 files changed, 145 insertions(+), 75 deletions(-)
 delete mode 100644 app/Console/AppShell.php
 create mode 100644 app/Model/SystemSetting.php

diff --git a/app/Console/AppShell.php b/app/Console/AppShell.php
deleted file mode 100644
index 76a6f1fbc..000000000
--- a/app/Console/AppShell.php
+++ /dev/null
@@ -1,34 +0,0 @@
-<?php
-/**
- * AppShell file
- *
- * PHP 5
- *
- * CakePHP(tm) : Rapid Development Framework (http://cakephp.org)
- * Copyright 2005-2012, Cake Software Foundation, Inc. (http://cakefoundation.org)
- *
- * Licensed under The MIT License
- * Redistributions of files must retain the above copyright notice.
- *
- * @copyright     Copyright 2005-2012, Cake Software Foundation, Inc. (http://cakefoundation.org)
- * @link          http://cakephp.org CakePHP(tm) Project
- * @since         CakePHP(tm) v 2.0
- * @license       MIT License (http://www.opensource.org/licenses/mit-license.php)
- */
-
-App::uses('Shell', 'Console', 'AppModel', 'Model');
-
-/**
- * Application Shell
- *
- * Add your application-wide methods in the class below, your shells
- * will inherit them.
- *
- * @package       app.Console.Command
- */
-class AppShell extends Shell {
-	public function perform() {
-		$this->initialize();
-		$this->{array_shift($this->args)}();
-	}
-}
diff --git a/app/Console/Command/Task/ConfigLoadTask.php b/app/Console/Command/Task/ConfigLoadTask.php
index 78e506803..df156011d 100644
--- a/app/Console/Command/Task/ConfigLoadTask.php
+++ b/app/Console/Command/Task/ConfigLoadTask.php
@@ -1,7 +1,14 @@
 <?php
-    class ConfigLoadTask extends Shell {
-        public function execute() {
-            Configure::load('config');
+class ConfigLoadTask extends Shell
+{
+    public function execute()
+    {
+        Configure::load('config');
+
+        if (Configure::read('MISP.system_setting_db')) {
+            $this->loadModel('SystemSetting');
+            $settings = $this->SystemSetting->getSettings();
+            Configure::write($settings);
         }
     }
-?>
+}
diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php
index 6d6be801b..f9d878382 100755
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@@ -97,6 +97,10 @@ class AppController extends Controller
 
     public function beforeFilter()
     {
+        if (Configure::read('MISP.system_setting_db')) {
+            $this->_setupSystemSettings();
+        }
+
         $this->_setupBaseurl();
         $this->Auth->loginRedirect = $this->baseurl . '/users/routeafterlogin';
 
@@ -1404,4 +1408,14 @@ class AppController extends Controller
             return true;
         }
     }
+
+    /**
+     * Load system settings from database and set as setting in Configure
+     */
+    protected function _setupSystemSettings()
+    {
+        $this->loadModel('SystemSetting');
+        $settings = $this->SystemSetting->getSettings();
+        Configure::write($settings);
+    }
 }
diff --git a/app/Controller/AuditLogsController.php b/app/Controller/AuditLogsController.php
index 3b2fcf354..25c4006f0 100644
--- a/app/Controller/AuditLogsController.php
+++ b/app/Controller/AuditLogsController.php
@@ -37,6 +37,7 @@ class AuditLogsController extends AppController
         'Server',
         'ShadowAttribute',
         'SharingGroup',
+        'SystemSetting',
         'Tag',
         'TagCollection',
         'TagCollectionTag',
diff --git a/app/Controller/ServersController.php b/app/Controller/ServersController.php
index e0b0d0b3b..295df791a 100644
--- a/app/Controller/ServersController.php
+++ b/app/Controller/ServersController.php
@@ -1437,18 +1437,18 @@ class ServersController extends AppController
                 }
             }
             $this->autoRender = false;
-            $this->loadModel('Log');
-            if (!is_writeable(APP . 'Config/config.php')) {
+            if (!Configure::read('MISP.system_setting_db') && !is_writeable(APP . 'Config/config.php')) {
+                $this->loadModel('Log');
                 $this->Log->create();
                 $this->Log->save(array(
-                        'org' => $this->Auth->user('Organisation')['name'],
-                        'model' => 'Server',
-                        'model_id' => 0,
-                        'email' => $this->Auth->user('email'),
-                        'action' => 'serverSettingsEdit',
-                        'user_id' => $this->Auth->user('id'),
-                        'title' => 'Server setting issue',
-                        'change' => 'There was an issue witch changing ' . $setting['name'] . ' to ' . $this->request->data['Server']['value']  . '. The error message returned is: app/Config.config.php is not writeable to the apache user. No changes were made.',
+                    'org' => $this->Auth->user('Organisation')['name'],
+                    'model' => 'Server',
+                    'model_id' => 0,
+                    'email' => $this->Auth->user('email'),
+                    'action' => 'serverSettingsEdit',
+                    'user_id' => $this->Auth->user('id'),
+                    'title' => 'Server setting issue',
+                    'change' => 'There was an issue witch changing ' . $setting['name'] . ' to ' . $this->request->data['Server']['value']  . '. The error message returned is: app/Config.config.php is not writeable to the apache user. No changes were made.',
                 ));
                 if ($this->_isRest()) {
                     return $this->RestResponse->saveFailResponse('Servers', 'serverSettingsEdit', false, 'app/Config.config.php is not writeable to the apache user.', $this->response->type());
diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php
index f6ac7bef9..f4a8cfb2d 100644
--- a/app/Model/AppModel.php
+++ b/app/Model/AppModel.php
@@ -84,7 +84,7 @@ class AppModel extends Model
         57 => false, 58 => false, 59 => false, 60 => false, 61 => false, 62 => false,
         63 => true, 64 => false, 65 => false, 66 => false, 67 => false, 68 => false,
         69 => false, 70 => false, 71 => true, 72 => true, 73 => false, 74 => false,
-        75 => false,
+        75 => false, 76 => true,
     );
 
     public $advanced_updates_description = array(
@@ -1593,6 +1593,13 @@ class AppModel extends Model
                 $this->__dropIndex('object_references', 'relationship_type');
                 $this->__dropIndex('object_references', 'referenced_uuid');
                 break;
+            case 76:
+                $sqlArray[] = "CREATE TABLE IF NOT EXISTS `system_settings` (
+                      `setting` varchar(255) NOT NULL,
+                      `value` blob NOT NULL,
+                      PRIMARY KEY (`setting`)
+                    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;";
+                break;
             case 'fixNonEmptySharingGroupID':
                 $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
                 $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
diff --git a/app/Model/AuditLog.php b/app/Model/AuditLog.php
index 99118107a..efc09587f 100644
--- a/app/Model/AuditLog.php
+++ b/app/Model/AuditLog.php
@@ -131,6 +131,8 @@ class AuditLog extends AppModel
         if (in_array($auditLog['model'], ['Attribute', 'Object', 'ShadowAttribute'], true)) {
             $modelName = $auditLog['model'] === 'ShadowAttribute' ? 'Proposal' : $auditLog['model'];
             $title = __('%s from Event #%s', $modelName, $auditLog['event_id']);
+        } else if ($auditLog['model'] === 'SystemSetting') {
+            $title = $auditLog['model'];
         } else {
             $title = "{$auditLog['model']} #{$auditLog['model_id']}";
         }
diff --git a/app/Model/Behavior/AuditLogBehavior.php b/app/Model/Behavior/AuditLogBehavior.php
index d085d50d4..7a6226bfa 100644
--- a/app/Model/Behavior/AuditLogBehavior.php
+++ b/app/Model/Behavior/AuditLogBehavior.php
@@ -39,6 +39,7 @@ class AuditLogBehavior extends ModelBehavior
         'TagCollection' => 'name',
         'Taxonomy' => 'namespace',
         'Organisation' => 'name',
+        'SystemSetting' => 'setting',
         'AdminSetting' => 'setting',
         'UserSetting' => 'setting',
         'Galaxy' => 'name',
@@ -117,7 +118,7 @@ class AuditLogBehavior extends ModelBehavior
             return;
         }
 
-        $id = $model->id ?: null;
+        $id = $model->id ?: 0;
         $data = $model->data[$model->alias];
 
         if ($created) {
@@ -177,14 +178,14 @@ class AuditLogBehavior extends ModelBehavior
             }
             $id = $modelName === 'AttributeTag' ? $data['attribute_id'] : $data['event_id'];
             $modelName = $modelName === 'AttributeTag' ? 'Attribute' : 'Event';
-        }
-
-        if ($modelName === 'Event') {
+        } else if ($modelName === 'Event') {
             if (isset($changedFields['published'][1]) && $changedFields['published'][1]) {
                 $action = AuditLog::ACTION_PUBLISH;
             } else if (isset($changedFields['sighting_timestamp'][1]) && $changedFields['sighting_timestamp'][1]) {
                 $action = AuditLog::ACTION_PUBLISH_SIGHTINGS;
             }
+        } else if ($modelName === 'SystemSetting') {
+            $id = 0;
         }
 
         $this->auditLog()->insert(['AuditLog' => [
diff --git a/app/Model/Server.php b/app/Model/Server.php
index b7243bf5d..4c6b749df 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -2123,15 +2123,15 @@ class Server extends AppModel
             if ($beforeResult !== true) {
                 $this->Log = ClassRegistry::init('Log');
                 $this->Log->create();
-                $result = $this->Log->save(array(
-                        'org' => $user['Organisation']['name'],
-                        'model' => 'Server',
-                        'model_id' => 0,
-                        'email' => $user['email'],
-                        'action' => 'serverSettingsEdit',
-                        'user_id' => $user['id'],
-                        'title' => 'Server setting issue',
-                        'change' => 'There was an issue witch changing ' . $setting['name'] . ' to ' . $value  . '. The error message returned is: ' . $beforeResult . 'No changes were made.',
+                $this->Log->save(array(
+                    'org' => $user['Organisation']['name'],
+                    'model' => 'Server',
+                    'model_id' => 0,
+                    'email' => $user['email'],
+                    'action' => 'serverSettingsEdit',
+                    'user_id' => $user['id'],
+                    'title' => 'Server setting issue',
+                    'change' => 'There was an issue witch changing ' . $setting['name'] . ' to ' . $value  . '. The error message returned is: ' . $beforeResult . 'No changes were made.',
                 ));
                 return $beforeResult;
             }
@@ -2184,6 +2184,13 @@ class Server extends AppModel
      */
     public function serverSettingsSaveValue($setting, $value)
     {
+        if (Configure::read('MISP.system_setting_db')) {
+            /** @var SystemSetting $systemSetting */
+            $systemSetting = ClassRegistry::init('SystemSetting');
+            $systemSetting->setSetting($setting, $value);
+            return true;
+        }
+
         $configFilePath = APP . 'Config' . DS . 'config.php';
         if (!is_writable($configFilePath)) {
             return false; // config file is not writeable
diff --git a/app/Model/SystemSetting.php b/app/Model/SystemSetting.php
new file mode 100644
index 000000000..a9abd79b4
--- /dev/null
+++ b/app/Model/SystemSetting.php
@@ -0,0 +1,41 @@
+<?php
+App::uses('AppModel', 'Model');
+App::uses('JsonTool', 'Tools');
+
+class SystemSetting extends AppModel
+{
+    public $actsAs = [
+        'SysLogLogable.SysLogLogable' => [
+            'userModel' => 'User',
+            'userKey' => 'user_id',
+            'change' => 'full'
+        ],
+        'AuditLog'
+    ];
+
+    public $primaryKey = 'setting';
+
+    public function getSettings()
+    {
+        $settings = $this->find('list', [
+            'fields' => ['SystemSetting.setting', 'SystemSetting.value'],
+        ]);
+        return array_map(['JsonTool', 'decode'], $settings);
+    }
+
+    /**
+     * @param string $setting
+     * @param mixed $value
+     * @throws Exception
+     */
+    public function setSetting($setting, $value)
+    {
+        $valid = $this->save(['SystemSetting' => [
+            'setting' => $setting,
+            'value' => JsonTool::encode($value),
+        ]]);
+        if (!$valid) {
+            throw new Exception("Could not save system setting `$setting` because of validation errors: " . JsonTool::encode($this->validationErrors));
+        }
+    }
+}
diff --git a/app/View/Servers/server_settings.ctp b/app/View/Servers/server_settings.ctp
index 2a3196c44..709ae1e0d 100644
--- a/app/View/Servers/server_settings.ctp
+++ b/app/View/Servers/server_settings.ctp
@@ -1,17 +1,17 @@
 <div class="server index">
-    <?php if ($writeableFiles[APP . 'Config/config.php'] != 0): ?>
-    <div class="bold" style="background-color:red;width:100%;color:white;"><span style="padding-left:10px;"><?php echo __('Warning: app/Config/config.php is not writeable. This means that any setting changes made here will NOT be saved.');?></span></div>
+    <?php if ($writeableFiles[APP . 'Config/config.php'] != 0 && !Configure::read('MISP.system_setting_db')): ?>
+    <div class="alert alert-error"><?= __('Warning: app/Config/config.php is not writeable. This means that any setting changes made here will NOT be saved.') ?></div>
     <?php endif; ?>
     <h2><?php echo __('Server Settings & Maintenance');?></h2>
     <?php
         echo $this->element('healthElements/tabs', array('active_tab' => $tab));
-        if (in_array($tab, array('MISP', 'Security', 'Encryption', 'Proxy', 'Plugin'))) {
+        if (in_array($tab, ['MISP', 'Security', 'Encryption', 'Proxy', 'Plugin'], true)) {
             echo $this->element('healthElements/settings_tab');
-        } else if ($tab == 'diagnostics') {
+        } else if ($tab === 'diagnostics') {
             echo $this->element('healthElements/diagnostics');
-        } else if ($tab == 'workers') {
+        } else if ($tab === 'workers') {
             echo $this->element('healthElements/workers');
-        } else if ($tab == 'files') {
+        } else if ($tab === 'files') {
             echo $this->element('healthElements/files');
         } else {
             echo $this->element('healthElements/overview');
@@ -20,14 +20,11 @@
     <div style="font-style: italic;"><?php echo __('To edit a setting, simply double click it.');?></div>
 </div>
 <script type="text/javascript">
-    $(document).ready(function() {
-        $('#liveFilterField').focus();
-        $('#liveFilterField').keyup(function() {
+    $(function() {
+        $('#liveFilterField').focus().keyup(function() {
             liveFilter();
         });
     });
 
 </script>
-<?php
-    echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'admin', 'menuItem' => 'serverSettings'));
-?>
+<?= $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'admin', 'menuItem' => 'serverSettings'));
diff --git a/db_schema.json b/db_schema.json
index 0da8d020b..5a3b3cde5 100644
--- a/db_schema.json
+++ b/db_schema.json
@@ -6144,6 +6144,30 @@
                 "extra": ""
             }
         ],
+        "system_settings": [
+            {
+                "column_name": "setting",
+                "is_nullable": "NO",
+                "data_type": "varchar",
+                "character_maximum_length": "255",
+                "numeric_precision": null,
+                "collation_name": "utf8mb4_unicode_ci",
+                "column_type": "varchar(255)",
+                "column_default": null,
+                "extra": ""
+            },
+            {
+                "column_name": "value",
+                "is_nullable": "NO",
+                "data_type": "blob",
+                "character_maximum_length": "65535",
+                "numeric_precision": null,
+                "collation_name": null,
+                "column_type": "blob",
+                "column_default": null,
+                "extra": ""
+            }
+        ],
         "tags": [
             {
                 "column_name": "id",
@@ -8096,6 +8120,9 @@
             "source": false,
             "type": false
         },
+        "system_settings": {
+            "setting": true
+        },
         "tags": {
             "id": true,
             "name": true,
@@ -8180,5 +8207,5 @@
             "id": true
         }
     },
-    "db_version": "75"
+    "db_version": "76"
 }

From 82ed12e4cb8cca9aa724cef0e44bec5c2c3ca5bf Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Thu, 4 Nov 2021 17:57:19 +0100
Subject: [PATCH 11/45] fix: [config] Remove not used
 Attributes_Values_Filter_In_Event

---
 app/Config/config.default.php | 1 -
 1 file changed, 1 deletion(-)

diff --git a/app/Config/config.default.php b/app/Config/config.default.php
index b2d3f010a..6f26a116e 100644
--- a/app/Config/config.default.php
+++ b/app/Config/config.default.php
@@ -50,7 +50,6 @@ $config = array(
         'unpublishedprivate'             => false,
         'disable_emailing'               => false,
         'manage_workers'                 => true,
-        'Attributes_Values_Filter_In_Event' => 'id, uuid, value, comment, type, category, Tag.name',
         'python_bin'                     => null,
         'external_baseurl'               => '',
         'forceHTTPSforPreLoginRequestedURL' => false,

From d67ac118a3adef937b171486c28dfa70fcfcee39 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Thu, 4 Nov 2021 18:07:59 +0100
Subject: [PATCH 12/45] chg: [internal] Code style

---
 app/Model/Server.php | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/app/Model/Server.php b/app/Model/Server.php
index 4c6b749df..ef5625997 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -4992,7 +4992,13 @@ class Server extends AppModel
                     'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
-                    'options' => array('0' => __('Your organisation only'), '1' => __('This community only'), '2' => __('Connected communities'), '3' => __('All communities'), 'event' => __('Inherit from event')),
+                    'options' => array(
+                        '0' => __('Your organisation only'),
+                        '1' => __('This community only'),
+                        '2' => __('Connected communities'),
+                        '3' => __('All communities'),
+                        'event' => __('Inherit from event')
+                    ),
                 ),
                 'default_event_threat_level' => array(
                     'level' => 1,
@@ -5606,7 +5612,7 @@ class Server extends AppModel
                     'null' => true,
                 ],
                 'warning_for_all' => [
-                    'level' => 1,
+                    'level' => self::SETTING_RECOMMENDED,
                     'description' => __('Enable warning list triggers regardless of the IDS flag value'),
                     'value' => false,
                     'errorMessage' => '',

From 5408bb284f4603108e21da55517c81b187b03d0c Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Fri, 5 Nov 2021 09:17:06 +0100
Subject: [PATCH 13/45] new: [setting] Add new MISP.system_setting_db setting

---
 app/Model/Server.php | 324 +++----------------------------------------
 1 file changed, 21 insertions(+), 303 deletions(-)

diff --git a/app/Model/Server.php b/app/Model/Server.php
index ef5625997..ef58dda4e 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -1397,7 +1397,7 @@ class Server extends AppModel
     {
         $dirs = glob(APP . 'Locale/*', GLOB_ONLYDIR);
         $languages = array('eng' => 'eng');
-        foreach ($dirs as $k => $dir) {
+        foreach ($dirs as $dir) {
             $dir = str_replace(APP . 'Locale' . DS, '', $dir);
             $languages[$dir] = $dir;
         }
@@ -4515,7 +4515,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('The base url of the application (in the format https://www.mymispinstance.com) as visible externally/by other MISPs. MISP will encode this URL in sharing groups when including itself. If this value is not set, the baseurl is used as a fallback.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testURL',
                     'type' => 'string',
                 ),
@@ -4523,7 +4522,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('Unless set to true, the instance will only be accessible by site admins.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testLive',
                     'type' => 'boolean',
                 ),
@@ -4531,7 +4529,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('Select the language MISP should use. The default is english.'),
                     'value' => 'eng',
-                    'errorMessage' => '',
                     'test' => 'testLanguage',
                     'type' => 'string',
                     'optionsSource' => function () {
@@ -4543,7 +4540,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('This values controls the internal fetcher\'s memory envelope when it comes to attributes. The number provided is the amount of attributes that can be loaded for each MB of PHP memory available in one shot. Consider lowering this number if your instance has a lot of attribute tags / attribute galaxies attached.'),
                     'value' => 80,
-                    'errorMessage' => '',
                     'test' => 'testForNumeric',
                     'type' => 'numeric',
                     'null' => true
@@ -4552,7 +4548,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('This value controls the divisor for attribute weighting when it comes to loading full events. Meaning that it will load coefficient / divisor number of attributes per MB of memory available. Consider raising this number if you have a lot of correlations or highly contextualised events (large number of event level galaxies/tags).'),
                     'value' => 3,
-                    'errorMessage' => '',
                     'test' => 'testForNumeric',
                     'type' => 'numeric',
                     'null' => true
@@ -4561,7 +4556,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('Enable some performance heavy correlations (currently CIDR correlation)'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -4570,7 +4564,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Enable this setting to directly save the config.php file without first creating a temporary file and moving it to avoid concurency issues. Generally not recommended, but useful when for example other tools modify/maintain the config.php file.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -4579,7 +4572,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('It is highly recommended to install all the python dependencies in a virtualenv. The recommended location is: %s/venv', ROOT),
                     'value' => false,
-                    'errorMessage' => '',
                     'null' => false,
                     'test' => 'testForBinExec',
                     'beforeHook' => 'beforeHookBinExec',
@@ -4590,7 +4582,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('MISP will default to the bundled mozilla certificate bundle shipped with the framework, which is rather stale. If you wish to use an alternate bundle, just set this setting using the path to the bundle to use. This setting can only be modified via the CLI.'),
                     'value' => APP . 'Lib/cakephp/lib/Cake/Config/cacert.pem',
-                    'errorMessage' => '',
                     'null' => true,
                     'test' => 'testForCABundle',
                     'type' => 'string',
@@ -4600,7 +4591,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('In some cases, a heavily used MISP instance can generate unwanted blackhole errors due to a high number of requests hitting the server. Disable the auto logout functionality to ease the burden on the system.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -4609,7 +4599,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Set the ssdeep score at which to consider two ssdeep hashes as correlating [1-100]'),
                     'value' => 40,
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'numeric'
                 ),
@@ -4617,7 +4606,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Sets the maximum number of correlations that can be fetched with a single event. For extreme edge cases this can prevent memory issues. The default value is 5k.'),
                     'value' => 5000,
-                    'errorMessage' => '',
                     'test' => 'testForNumeric',
                     'type' => 'numeric',
                     'null' => true
@@ -4634,7 +4622,6 @@ class Server extends AppModel
                     'level' => 3,
                     'description' => __('This setting is deprecated and can be safely removed.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -4642,7 +4629,6 @@ class Server extends AppModel
                     'level' => 3,
                     'description' => __('This setting is deprecated and can be safely removed.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -4651,7 +4637,6 @@ class Server extends AppModel
                     'description' => __('Cached exports can take up a considerable amount of space and can be disabled instance wide using this setting. Disabling the cached exports is not recommended as it\'s a valuable feature, however, if your server is having free space issues it might make sense to take this step.'),
                     'value' => false,
                     'null' => true,
-                    'errorMessage' => '',
                     'test' => 'testDisableCache',
                     'type' => 'boolean',
                     'afterHook' => 'disableCacheAfterHook',
@@ -4661,7 +4646,6 @@ class Server extends AppModel
                     'description' => __('Disable displaying / modifications to the threat level altogether on the instance (deprecated field).'),
                     'value' => false,
                     'null' => true,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -4669,7 +4653,6 @@ class Server extends AppModel
                     'level' => 3,
                     'description' => __('This setting is deprecated and can be safely removed.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -4677,7 +4660,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Footer text prepending the "Powered by MISP" text.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -4685,7 +4667,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Footer text following the "Powered by MISP" text.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -4693,7 +4674,6 @@ class Server extends AppModel
                     'level' => 3,
                     'description' => __('This setting is deprecated and can be safely removed.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -4701,7 +4681,6 @@ class Server extends AppModel
                     'level' => 3,
                     'description' => __('This setting is deprecated and can be safely removed.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -4709,7 +4688,6 @@ class Server extends AppModel
                     'level' => 3,
                     'description' => __('This setting is deprecated and can be safely removed.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -4717,31 +4695,27 @@ class Server extends AppModel
                     'level' => 3,
                     'description' => __('This setting is deprecated and can be safely removed.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
                 'footer_logo' => array(
-                    'level' => 2 ,
+                    'level' => 2,
                     'description' => __('If set, this setting allows you to display a logo on the right side of the footer. Upload it as a custom image in the file management tool.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForCustomImage',
                     'type' => 'string',
                 ),
                 'home_logo' => array(
-                    'level' => 2 ,
+                    'level' => 2,
                     'description' => __('If set, this setting allows you to display a logo as the home icon. Upload it as a custom image in the file management tool.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForCustomImage',
                     'type' => 'string',
                 ),
                 'main_logo' => array(
-                    'level' => 2 ,
+                    'level' => 2,
                     'description' => __('If set, the image specified here will replace the main MISP logo on the login screen. Upload it as a custom image in the file management tool.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForCustomImage',
                     'type' => 'string',
                 ),
@@ -4749,7 +4723,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('The organisation tag of the hosting organisation. This is used in the e-mail subjects.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -4757,7 +4730,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('The hosting organisation of this instance. If this is not selected then replication instances cannot be added.'),
                     'value' => '0',
-                    'errorMessage' => '',
                     'test' => 'testLocalOrgStrict',
                     'type' => 'numeric',
                     'optionsSource' => function () {
@@ -4776,7 +4748,6 @@ class Server extends AppModel
                     'level' => 3,
                     'description' => __('This setting is deprecated and can be safely removed.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -4784,7 +4755,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('Setting this setting to \'false\' will hide all organisation names / logos.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                 ),
@@ -4792,7 +4762,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Put the event threat level in the notification E-mail subject.'),
                     'value' => true,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                 ),
@@ -4800,7 +4769,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('This is the TLP string for e-mails when email_subject_tag is not found.'),
                     'value' => 'tlp:amber',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -4808,7 +4776,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('If this tag is set on an event it\'s value will be sent in the E-mail subject. If the tag is not set the email_subject_TLP_string will be used.'),
                     'value' => 'tlp',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -4816,7 +4783,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Include in name of the email_subject_tag in the subject. When false only the tag value is used.'),
                     'value' => true,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                 ),
@@ -4824,7 +4790,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Notification e-mail sender name.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ],
@@ -4832,7 +4797,6 @@ class Server extends AppModel
                     'level' => 3,
                     'description' => __('This setting is deprecated and can be safely removed.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -4840,7 +4804,6 @@ class Server extends AppModel
                     'level' => 3,
                     'description' => __('This setting is deprecated and can be safely removed.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -4848,15 +4811,13 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Enables the use of MISP\'s background processing.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testBoolTrue',
                     'type' => 'boolean',
                 ),
                 'attachments_dir' => array(
                     'level' => 2,
                     'description' => __('Directory where attachments are stored. MISP will NOT migrate the existing data if you change this setting. The only safe way to change this setting is in config.php, when MISP is not running, and after having moved/copied the existing data to the new location. This directory must already exist and be writable and readable by the MISP application.'),
-                    'value' =>  APP . '/files', # GUI display purpose only.
-                    'errorMessage' => '',
+                    'value' => APP . '/files', # GUI display purpose only.
                     'null' => false,
                     'test' => 'testForWritableDir',
                     'type' => 'string',
@@ -4866,7 +4827,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Allow the XML caches to include the encoded attachments.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                 ),
@@ -4874,7 +4834,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Always download attachments when loaded by a user in a browser'),
                     'value' => true,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                 ),
@@ -4882,7 +4841,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('The Unix user MISP (php) is running as'),
                     'value' => 'www-data',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -4890,7 +4848,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('The e-mail address that MISP should use for all notifications'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -4898,7 +4855,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('You can disable all e-mailing using this setting. When enabled, no outgoing e-mails will be sent by MISP.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'null' => true,
                     'test' => 'testDisableEmail',
                     'type' => 'boolean',
@@ -4907,7 +4863,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('This setting is deprecated. Please use `MISP.event_alert_metadata_only` instead.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'null' => true,
                     'test' => 'testBool',
                     'type' => 'boolean',
@@ -4916,7 +4871,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('The e-mail address that MISP should include as a contact address for the instance\'s support team.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -4924,7 +4878,6 @@ class Server extends AppModel
                     'level' => 3,
                     'description' => __('This setting is deprecated and can be safely removed.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -4932,7 +4885,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Turn Vulnerability type attributes into links linking to the provided CVE lookup'),
                     'value' => 'https://cve.circl.lu/cve/',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -4940,7 +4892,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Turn Weakness type attributes into links linking to the provided CWE lookup'),
                     'value' => 'https://cve.circl.lu/cwe/',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -4948,7 +4899,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('This setting controls whether notification e-mails will be sent when an event is created via the REST interface. It might be a good idea to disable this setting when first setting up a link to another instance to avoid spamming your users during the initial pull. Quick recap: True = Emails are NOT sent, False = Emails are sent on events published via sync / REST.'),
                     'value' => true,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                 ),
@@ -4956,7 +4906,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Enabling this flag will allow the event description to be transmitted in the alert e-mail\'s subject. Be aware that this is not encrypted by GnuPG, so only enable it if you accept that part of the event description will be sent out in clear-text.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -4964,7 +4913,6 @@ class Server extends AppModel
                     'level' => self::SETTING_OPTIONAL,
                     'description' => __('If enabled, any requested URL before login will have their HTTP part replaced by HTTPS. This can be usefull if MISP is running behind a reverse proxy responsible for SSL and communicating unencrypted with MISP.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -4972,7 +4920,6 @@ class Server extends AppModel
                     'level' => self::SETTING_OPTIONAL,
                     'description' => __('Send just event metadata (attributes and objects will be omitted) for event alert.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ],
@@ -4980,7 +4927,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('The default distribution setting for events (0-3).'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'options' => array('0' => __('Your organisation only'), '1' => __('This community only'), '2' => __('Connected communities'), '3' => __('All communities')),
@@ -4989,7 +4935,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('The default distribution setting for attributes, set it to \'event\' if you would like the attributes to default to the event distribution level. (0-3 or "event")'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'options' => array(
@@ -5004,7 +4949,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('The default threat level setting when creating events.'),
                     'value' => 4,
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'options' => array('1' => 'High', '2' => 'Medium', '3' => 'Low', '4' => 'undefined'),
@@ -5013,7 +4957,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('The tag collection to be applied to all events created manually.'),
                     'value' => 0,
-                    'errorMessage' => '',
                     'test' => 'testTagCollections',
                     'type' => 'numeric',
                     'optionsSource' => function () {
@@ -5024,7 +4967,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('The default setting for publish alerts when creating users.'),
                     'value' => true,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -5033,7 +4975,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Enable the tagging feature of MISP. This is highly recommended.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                 ),
@@ -5041,7 +4982,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Show the full tag names on the event index.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'options' => array(0 => 'Minimal tags', 1 => 'Full tags', 2 => 'Shortened tags'),
@@ -5050,7 +4990,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Used on the login page, before the MISP logo'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -5058,7 +4997,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Used on the login page, after the MISP logo'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -5066,7 +5004,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Used on the login page, to the left of the MISP logo, upload it as a custom image in the file management tool.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForCustomImage',
                     'type' => 'string',
                 ),
@@ -5074,7 +5011,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Used on the login page, to the right of the MISP logo, upload it as a custom image in the file management tool.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForCustomImage',
                     'type' => 'string',
                 ),
@@ -5082,7 +5018,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Used in the page title, after the name of the page'),
                     'value' => 'MISP',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -5090,7 +5025,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Allows users to take ownership of an event uploaded via the "Add MISP XML" button. This allows spoofing the creator of a manually imported event, also breaking possibly breaking the original intended releasability. Synchronising with an instance that has a different creator for the same event can lead to unwanted consequences.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                 ),
@@ -5098,7 +5032,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Choose whether the terms and conditions should be displayed inline (false) or offered as a download (true)'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -5106,7 +5039,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The filename of the terms and conditions file. Make sure that the file is located in your MISP/app/files/terms directory'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForTermsFile',
                     'type' => 'string'
                 ),
@@ -5114,7 +5046,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('True enables the alternate org fields for the event index (source org and member org) instead of the traditional way of showing only an org field. This allows users to see if an event was uploaded by a member organisation on their MISP instance, or if it originated on an interconnected instance.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -5122,7 +5053,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('True will deny access to unpublished events to users outside the organization of the submitter except site admins.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -5131,7 +5061,6 @@ class Server extends AppModel
                     'bigField' => true,
                     'description' => __('The message sent to the user after account creation (has to be sent manually from the administration interface). Use \\n for line-breaks. The following variables will be automatically replaced in the text: $password = a new temporary password that MISP generates, $username = the user\'s e-mail address, $misp = the url of this instance, $org = the organisation that the instance belongs to, as set in MISP.org, $contact = the e-mail address used to contact the support team, as set in MISP.contact. For example, "the password for $username is $password" would appear to a user with the e-mail address user@misp.org as "the password for user@misp.org is hNamJae81".'),
                     'value' => 'Dear new MISP user,\n\nWe would hereby like to welcome you to the $org MISP community.\n\n Use the credentials below to log into MISP at $misp, where you will be prompted to manually change your password to something of your own choice.\n\nUsername: $username\nPassword: $password\n\nIf you have any questions, don\'t hesitate to contact us at: $contact.\n\nBest regards,\nYour $org MISP support team',
-                    'errorMessage' => '',
                     'test' => 'testPasswordResetText',
                     'type' => 'string'
                 ),
@@ -5140,7 +5069,6 @@ class Server extends AppModel
                     'bigField' => true,
                     'description' => __('The message sent to the users when a password reset is triggered. Use \\n for line-breaks. The following variables will be automatically replaced in the text: $password = a new temporary password that MISP generates, $username = the user\'s e-mail address, $misp = the url of this instance, $contact = the e-mail address used to contact the support team, as set in MISP.contact. For example, "the password for $username is $password" would appear to a user with the e-mail address user@misp.org as "the password for user@misp.org is hNamJae81".'),
                     'value' => 'Dear MISP user,\n\nA password reset has been triggered for your account. Use the below provided temporary password to log into MISP at $misp, where you will be prompted to manually change your password to something of your own choice.\n\nUsername: $username\nYour temporary password: $password\n\nIf you have any questions, don\'t hesitate to contact us at: $contact.\n\nBest regards,\nYour $org MISP support team',
-                    'errorMessage' => '',
                     'test' => 'testPasswordResetText',
                     'type' => 'string'
                 ),
@@ -5162,7 +5090,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('If enabled, all log entries will include the IP address of the user.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'beforeHook' => 'ipLogBeforeHook'
@@ -5171,7 +5098,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('If log_client_ip is enabled, you can customize which header field contains the client\'s IP address. This is generally used when you have a reverse proxy infront of your MISP instance.'),
                     'value' => 'REMOTE_ADDR',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'null' => true,
@@ -5180,7 +5106,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('If enabled, MISP will log all successful authentications using API keys. The requested URLs are also logged.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                 ),
@@ -5197,7 +5122,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('If this functionality is enabled all page requests will be logged. Keep in mind this is extremely verbose and will become a burden to your database.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBoolFalse',
                     'type' => 'boolean',
                     'null' => true
@@ -5206,7 +5130,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('You can decide to skip the logging of the paranoid logs to the database.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testParanoidSkipDb',
                     'type' => 'boolean',
                     'null' => true
@@ -5215,7 +5138,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('If paranoid logging is enabled, include the POST body in the entries.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -5224,7 +5146,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('Log user IPs on each request. 30 day retention for lookups by IP to get the last authenticated user ID for the given IP, whilst on the reverse, indefinitely stores all associated IPs for a user ID.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -5233,7 +5154,6 @@ class Server extends AppModel
                     'level' => self::SETTING_RECOMMENDED,
                     'description' => __('Log user IP and key usage on each API request. All logs for given keys are deleted after one year when this key is not used.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -5242,7 +5162,6 @@ class Server extends AppModel
                     'level' => self::SETTING_RECOMMENDED,
                     'description' => __('Enable new audit log system.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -5251,7 +5170,6 @@ class Server extends AppModel
                     'level' => self::SETTING_OPTIONAL,
                     'description' => __('Compress log changes by brotli algorithm. This will reduce log database size.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -5260,7 +5178,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('This feature allows users to create org only events and ask another organisation to take ownership of the event. This allows organisations to remain anonymous by asking a partner to publish an event for them.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -5269,7 +5186,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('When enabled, the number of correlations visible to the currently logged in user will be visible on the event index UI. This comes at a performance cost but can be very useful to see correlating events at a glance.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -5278,7 +5194,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('When enabled, the number of proposals for the events are shown on the index.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -5287,7 +5202,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('When enabled, the aggregate number of attribute sightings within the event becomes visible to the currently logged in user on the event index UI.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -5296,7 +5210,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('When enabled, the aggregate number of discussion posts for the event becomes visible to the currently logged in user on the event index UI.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -5305,7 +5218,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('When enabled, the aggregate number of event reports for the event becomes visible to the currently logged in user on the event index UI.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -5314,7 +5226,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('When enabled only Org and Site admins can edit a user\'s profile.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => false,
@@ -5323,7 +5234,6 @@ class Server extends AppModel
                     'level' => self::SETTING_RECOMMENDED,
                     'description' => __('When enabled only Site admins can change user email. This should be enabled if you manage user logins by external system.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => false,
@@ -5332,7 +5242,6 @@ class Server extends AppModel
                     'level' => self::SETTING_RECOMMENDED,
                     'description' => __('When enabled only Site admins can change user password. This should be enabled if you manage user passwords by external system.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => false,
@@ -5341,7 +5250,6 @@ class Server extends AppModel
                     'level' => self::SETTING_RECOMMENDED,
                     'description' => __('When enabled, Org Admins could not add new users. This should be enabled if you manage users by external system.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => false,
@@ -5350,7 +5258,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Enable this setting to start blocking alert e-mails for events with a certain tag. Define the tag in MISP.block_event_alert_tag.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => false,
@@ -5359,7 +5266,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('If the MISP.block_event_alert setting is set, alert e-mails for events tagged with the tag defined by this setting will be blocked.'),
                     'value' => 'no-alerts="true"',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'null' => false,
@@ -5368,7 +5274,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Enable this setting to start blocking alert e-mails for events that have already been published since a specified amount of time. This threshold is defined by MISP.event_alert_republish_ban_threshold'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => false,
@@ -5377,7 +5282,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('If the MISP.event_alert_republish_ban setting is set, this setting will control how long no alerting by email will be done. Expected format: integer, in minutes'),
                     'value' => 5,
-                    'errorMessage' => '',
                     'test' => 'testForNumeric',
                     'type' => 'numeric',
                     'null' => false,
@@ -5386,7 +5290,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('If the MISP.event_alert_republish_ban setting is set, this setting will control if a ban time should be reset if emails are tried to be sent during the ban.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => false,
@@ -5395,7 +5298,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Enable this setting to start blocking users to send too many e-mails notification since a specified amount of time. This threshold is defined by MISP.user_email_notification_ban_threshold'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => false,
@@ -5404,7 +5306,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('If the MISP.user_email_notification_ban setting is set, this setting will control how long no notification by email will be done. Expected format: integer, in minutes'),
                     'value' => 120,
-                    'errorMessage' => '',
                     'test' => 'testForNumeric',
                     'type' => 'numeric',
                     'null' => false,
@@ -5413,7 +5314,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('If the MISP.user_email_notification_ban setting is set, this setting will control how many notification by email can be send for the timeframe defined in MISP.user_email_notification_ban_time_threshold. Expected format: integer'),
                     'value' => 10,
-                    'errorMessage' => '',
                     'test' => 'testForNumeric',
                     'type' => 'numeric',
                     'null' => false,
@@ -5422,7 +5322,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Set a value to limit the number of email alerts that events can generate per creator organisation (for example, if an organisation pushes out 2000 events in one shot, only alert on the first 20).'),
                     'value' => 0,
-                    'errorMessage' => '',
                     'test' => 'testForNumeric',
                     'type' => 'numeric',
                     'null' => true,
@@ -5431,7 +5330,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Enable this setting to start blocking alert e-mails for old events. The exact timing of what constitutes an old event is defined by MISP.block_old_event_alert_age.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => false,
@@ -5440,7 +5338,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('If the MISP.block_old_event_alert setting is set, this setting will control how old an event can be for it to be alerted on. The "timestamp" field of the event is used. Expected format: integer, in days'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testForNumeric',
                     'type' => 'numeric',
                     'null' => false,
@@ -5449,7 +5346,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('If the MISP.block_old_event_alert setting is set, this setting will control the threshold for the event.date field, indicating how old an event can be for it to be alerted on. The "date" field of the event is used. Expected format: integer, in days'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testForNumeric',
                     'type' => 'numeric',
                     'null' => false,
@@ -5458,7 +5354,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Please indicate the temp directory you wish to use for certain functionalities in MISP. By default this is set to /tmp and will be used among others to store certain temporary files extracted from imports during the import process.'),
                     'value' => '/tmp',
-                    'errorMessage' => '',
                     'test' => 'testForPath',
                     'type' => 'string',
                     'null' => true,
@@ -5468,7 +5363,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('If you would like to customise the css, simply drop a css file in the /var/www/MISP/app/webroot/css directory and enter the name here.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForStyleFile',
                     'type' => 'string',
                     'null' => true,
@@ -5477,7 +5371,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('Enable this setting to allow blocking attributes from to_ids sensitive exports if a proposal has been made to it to remove the IDS flag or to remove the attribute altogether. This is a powerful tool to deal with false-positives efficiently.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => false,
@@ -5486,7 +5379,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Enable this settings if new tags synced / added via incoming events from any source should not be selectable by users by default.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => false
@@ -5495,7 +5387,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('*WARNING* This setting will completely disable the correlation on this instance and remove any existing saved correlations. Enabling this will trigger a full recorrelation of all data which is an extremely long and costly procedure. Only enable this if you know what you\'re doing.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBoolFalse',
                     'type' => 'boolean',
                     'null' => true,
@@ -5505,7 +5396,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('*WARNING* This setting will give event creators the possibility to disable the correlation of individual events / attributes that they have created.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBoolFalse',
                     'type' => 'boolean',
                     'null' => true
@@ -5514,7 +5404,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('The host running the redis server to be used for generic MISP tasks such as caching. This is not to be confused by the redis server used by the background processing.'),
                     'value' => '127.0.0.1',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -5522,7 +5411,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('The port used by the redis server to be used for generic MISP tasks such as caching. This is not to be confused by the redis server used by the background processing.'),
                     'value' => 6379,
-                    'errorMessage' => '',
                     'test' => 'testForNumeric',
                     'type' => 'numeric'
                 ),
@@ -5530,7 +5418,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('The database on the redis server to be used for generic MISP tasks. If you run more than one MISP instance, please make sure to use a different database on each instance.'),
                     'value' => 13,
-                    'errorMessage' => '',
                     'test' => 'testForNumeric',
                     'type' => 'numeric'
                 ),
@@ -5538,7 +5425,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('The password on the redis server (if any) to be used for generic MISP tasks.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => null,
                     'type' => 'string',
                     'redacted' => true
@@ -5547,7 +5433,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Specify which fields to filter on when you search on the event view. Default values are : "id, uuid, value, comment, type, category, Tag.name"'),
                     'value' => 'id, uuid, value, comment, type, category, Tag.name',
-                    'errorMessage' => '',
                     'test' => null,
                     'type' => 'string',
                 ),
@@ -5555,7 +5440,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Set this to false if you would like to disable MISP managing its own worker processes (for example, if you are managing the workers with a systemd unit).'),
                     'value' => true,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -5563,7 +5447,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Only enable this if you have some tools using MISP with extreme high concurency. General performance will be lower as normal as certain transactional queries are avoided in favour of shorter table locks.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -5580,7 +5463,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('This is a performance tweak to change the behaviour of restSearch to use attribute filters solely for blocking. This means that a lookup on the event scope with for example the type field set will be ignored unless it\'s used to strip unwanted attributes from the results. If left disabled, passing [ip-src, ip-dst] for example will return any event with at least one ip-src or ip-dst attribute. This is generally not considered to be too useful and is a heavy burden on the database.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -5589,7 +5471,6 @@ class Server extends AppModel
                     'level' => self::SETTING_OPTIONAL,
                     'description' => __('Name of enrichment module that will be used for attachment malware scanning. This module must return av-signature or sb-signature object.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'type' => 'string',
                     'null' => true,
                 ],
@@ -5597,7 +5478,6 @@ class Server extends AppModel
                     'level' => self::SETTING_OPTIONAL,
                     'description' => __('Send to attachment scan module just file hash. This can be useful if module sends attachment to remote service and you don\'t want to leak real data.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true,
@@ -5606,20 +5486,27 @@ class Server extends AppModel
                     'level' => self::SETTING_OPTIONAL,
                     'description' => __('How long to wait for scan results in seconds.'),
                     'value' => 30,
-                    'errorMessage' => '',
                     'test' => 'testForPositiveInteger',
                     'type' => 'numeric',
                     'null' => true,
                 ],
                 'warning_for_all' => [
                     'level' => self::SETTING_RECOMMENDED,
-                    'description' => __('Enable warning list triggers regardless of the IDS flag value'),
+                    'description' => __('Enable warning list triggers regardless of the IDS flag value.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
                 ],
+                'system_setting_db' => [
+                    'level' => self::SETTING_RECOMMENDED,
+                    'description' => __('Enable storing setting in database.'),
+                    'value' => false,
+                    'test' => 'testBool',
+                    'type' => 'boolean',
+                    'null' => true,
+                    'cli_only' => true,
+                ]
             ),
             'GnuPG' => array(
                 'branch' => 1,
@@ -5627,7 +5514,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The location of the GnuPG executable. If you would like to use a different GnuPG executable than /usr/bin/gpg, you can set it here. If the default is fine, just keep the setting suggested by MISP.'),
                     'value' => '/usr/bin/gpg',
-                    'errorMessage' => '',
                     'test' => 'testForGPGBinary',
                     'type' => 'string',
                     'cli_only' => 1
@@ -5636,7 +5522,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('Allow (false) unencrypted e-mails to be sent to users that don\'t have a GnuPG key.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                 ),
@@ -5644,7 +5529,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Allow (false) the body of unencrypted e-mails to contain details about the event.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                 ),
@@ -5652,7 +5536,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enable the signing of GnuPG emails. By default, GnuPG emails are signed'),
                     'value' => true,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                 ),
@@ -5660,7 +5543,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('The e-mail address that the instance\'s GnuPG key is tied to.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -5668,7 +5550,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('The password (if it is set) of the GnuPG key of the instance.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'redacted' => true
@@ -5677,7 +5558,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('The location of the GnuPG homedir.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -5685,7 +5565,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('When enabled, the subject in signed and encrypted e-mails will not be sent in unencrypted form.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                 )
@@ -5696,7 +5575,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enable S/MIME encryption. The encryption posture of the GnuPG.onlyencrypted and GnuPG.bodyonlyencrypted settings are inherited if S/MIME is enabled.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                 ),
@@ -5704,7 +5582,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The e-mail address that the instance\'s S/MIME key is tied to.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -5712,7 +5589,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The location of the public half of the signing certificate.'),
                     'value' => '/var/www/MISP/.smime/email@address.com.pem',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -5720,7 +5596,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The location of the private half of the signing certificate.'),
                     'value' => '/var/www/MISP/.smime/email@address.com.key',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -5728,7 +5603,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The password (if it is set) of the S/MIME key of the instance.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'redacted' => true
@@ -5740,7 +5614,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The hostname of an HTTP proxy for outgoing sync requests. Leave empty to not use a proxy.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -5748,7 +5621,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The TCP port for the HTTP proxy.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForNumeric',
                     'type' => 'numeric',
                 ),
@@ -5756,7 +5628,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The authentication method for the HTTP proxy. Currently supported are Basic or Digest. Leave empty for no proxy authentication.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -5764,7 +5635,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The authentication username for the HTTP proxy.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -5772,7 +5642,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The authentication password for the HTTP proxy.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -5792,7 +5661,6 @@ class Server extends AppModel
                     'level' => self::SETTING_CRITICAL,
                     'description' => __('Enforce CSP. Content Security Policy (CSP) is an added layer of security that helps to detect and mitigate certain types of attacks, including Cross Site Scripting (XSS) and data injection attacks. When disabled, violations will be just logged.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                 ],
@@ -5800,17 +5668,15 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('The salt used for the hashed passwords. You cannot reset this from the GUI, only manually from the settings.php file. Keep in mind, this will invalidate all passwords in the database.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testSalt',
                     'type' => 'string',
                     'editable' => false,
                     'redacted' => true
                 ),
-                'log_each_individual_auth_fail' =>[
+                'log_each_individual_auth_fail' => [
                     'level' => 1,
                     'description' => __('By default API authentication failures that happen within the same hour for the same key are omitted and a single log entry is generated. This allows administrators to more easily keep track of attackers that try to brute force API authentication, by reducing the noise generated by expired API keys. On the other hand, this makes little sense for internal MISP instances where detecting the misconfiguration of tools becomes more interesting, so if you fall into the latter category, enable this feature.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                 ],
@@ -5818,7 +5684,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('Advanced authkeys will allow each user to create and manage a set of authkeys for themselves, each with individual expirations and comments. API keys are stored in a hashed state and can no longer be recovered from MISP. Users will be prompted to note down their key when creating a new authkey. You can generate a new set of API keys for all users on demand in the diagnostics page, or by triggering %s.', sprintf('<a href="%s/servers/serverSettings/diagnostics#advanced_authkey_update">%s</a>', $this->baseurl, __('the advanced upgrade'))),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                 ),
@@ -5826,7 +5691,6 @@ class Server extends AppModel
                     'level' => self::SETTING_OPTIONAL,
                     'description' => __('Maximal key lifetime in days. Use can limit that validity even more. Just newly created keys will be affected. When not set, key validity is not limited.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'type' => 'numeric',
                     'test' => 'testForNumeric',
                     'null' => true,
@@ -5835,7 +5699,6 @@ class Server extends AppModel
                     'level' => self::SETTING_OPTIONAL,
                     'description' => __('When enabled, session is kept between API requests.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true,
@@ -5844,7 +5707,6 @@ class Server extends AppModel
                     'level' => self::SETTING_OPTIONAL,
                     'description' => __('This optionally can be enabled if an external auth provider is used. When set to true, it will disable the default form authentication.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                 ],
@@ -5852,7 +5714,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('Enable this setting if you wish for users to be able to query any arbitrary URL via the rest client. Keep in mind that queries are executed by the MISP server, so internal IPs in your MISP\'s network may be reachable.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -5861,7 +5722,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('If left empty, the baseurl of your MISP is used. However, in some instances (such as port-forwarded VM installations) this will not work. You can override the baseurl with a url through which your MISP can reach itself (typically https://127.0.0.1 would work).'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => null,
                     'type' => 'string'
                 ),
@@ -5869,7 +5729,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('Enable this setting to pass all audit log entries directly to syslog. Keep in mind, this is verbose and will include user, organisation, event data.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -5878,7 +5737,6 @@ class Server extends AppModel
                     'level' => self::SETTING_OPTIONAL,
                     'description' => __('Write syslog messages also to standard error output.'),
                     'value' => true,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -5887,7 +5745,6 @@ class Server extends AppModel
                     'level' => self::SETTING_OPTIONAL,
                     'description' => __('Syslog message identifier.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'null' => true
@@ -5896,7 +5753,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('If enabled, any authkey will be replaced by asterisks in Audit log.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -5905,7 +5761,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('If enabled, HTTP headers that block browser cache will be send. Static files (like images or JavaScripts) will still be cached, but not generated pages.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true,
@@ -5914,7 +5769,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('If enabled, any POST, PUT or AJAX request will be allow just when Sec-Fetch-Site header is not defined or contains "same-origin".'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true,
@@ -5923,35 +5777,31 @@ class Server extends AppModel
                     'level' => self::SETTING_OPTIONAL,
                     'description' => __('If enabled, MISP server will consider all requests as secure. This is usually useful when you run MISP behind reverse proxy that terminates HTTPS.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true,
                 ],
                 'email_otp_enabled' => array(
-                    'level'=> 2,
+                    'level' => 2,
                     'description' => __('Enable two step authentication with a OTP sent by email. Requires e-mailing to be enabled. Warning: You cannot use it in combination with external authentication plugins.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'beforeHook' => 'otpBeforeHook',
                     'type' => 'boolean',
                     'null' => true
                 ),
-                'email_otp_length' => array (
+                'email_otp_length' => array(
                     'level' => 2,
                     'description' => __('Define the length of the OTP code sent by email'),
                     'value' => '6',
-                    'errorMessage' => '',
                     'type' => 'numeric',
                     'test' => 'testForNumeric',
                     'null' => true,
                 ),
-                'email_otp_validity' => array (
+                'email_otp_validity' => array(
                     'level' => 2,
                     'description' => __('Define the validity (in minutes) of the OTP code sent by email'),
                     'value' => '5',
-                    'errorMessage' => '',
                     'type' => 'numeric',
                     'test' => 'testForNumeric',
                     'null' => true,
@@ -5961,7 +5811,6 @@ class Server extends AppModel
                     'bigField' => true,
                     'description' => __('The message sent to the user when a new OTP is requested. Use \\n for line-breaks. The following variables will be automatically replaced in the text: $otp = the new OTP generated by MISP, $username = the user\'s e-mail address, $org the Organisation managing the instance, $misp = the url of this instance, $contact = the e-mail address used to contact the support team (as set in MISP.contact), $ip the IP used to complete the first step of the login and $validity the validity time in minutes.'),
                     'value' => 'Dear MISP user,\n\nYou have attempted to login to MISP ($misp) from $ip with username $username.\n\n Use the following OTP to log into MISP: $otp\n This code is valid for the next $validity minutes.\n\nIf you have any questions, don\'t hesitate to contact us at: $contact.\n\nBest regards,\nYour $org MISP support team',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'null' => true,
@@ -5971,7 +5820,6 @@ class Server extends AppModel
                     'bigField' => true,
                     'description' => __('A comma separated list of emails for which the OTP is disabled. Note that if you remove someone from this list, the OTP will only be asked at next login.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'null' => true,
@@ -5980,7 +5828,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Enabling this setting will allow users to have access to the pre-auth registration form. This will create an inbox entry for administrators to review.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -5990,7 +5837,6 @@ class Server extends AppModel
                     'bigField' => true,
                     'description' => __('The message sent shown to anyone trying to self-register.'),
                     'value' => 'If you would like to send us a registration request, please fill out the form below. Make sure you fill out as much information as possible in order to ease the task of the administrators.',
-                    'errorMessage' => '',
                     'test' => false,
                     'type' => 'string'
                 ),
@@ -5998,7 +5844,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Password length requirement. If it is not set or it is set to 0, then the default value is assumed (12).'),
                     'value' => '12',
-                    'errorMessage' => '',
                     'test' => 'testPasswordLength',
                     'type' => 'numeric',
                 ),
@@ -6006,7 +5851,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Password complexity requirement. Leave it empty for the default setting (3 out of 4, with either a digit or a special char) or enter your own regex. Keep in mind that the length is checked in another key. Default (simple 3 out of 4 or minimum 16 characters): /^((?=.*\d)|(?=.*\W+))(?![\n])(?=.*[A-Z])(?=.*[a-z]).*$|.{16,}/'),
                     'value' => '/^((?=.*\d)|(?=.*\W+))(?![\n])(?=.*[A-Z])(?=.*[a-z]).*$|.{16,}/',
-                    'errorMessage' => '',
                     'test' => 'testPasswordRegex',
                     'type' => 'string',
                 ),
@@ -6014,7 +5858,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Enabling this setting will require users to submit their current password on any edits to their profile (including a triggered password change). For administrators, the confirmation will be required when changing the profile of any user. Could potentially mitigate an attacker trying to change a compromised user\'s password in order to establish persistance, however, enabling this feature will be highly annoying to users.'),
                     'value' => true,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -6023,7 +5866,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Enabling this setting will sanitise the contents of an attribute on a soft delete'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -6032,7 +5874,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Enabling this setting will block the organisation index from being visible to anyone besides site administrators on the current instance. Keep in mind that users can still see organisations that produce data via events, proposals, event history log entries, etc.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -6041,7 +5882,6 @@ class Server extends AppModel
                     'level' => self::SETTING_RECOMMENDED,
                     'description' => __('Enabling this setting will block the organisation list from being visible in sharing group besides user with sharing group permission.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -6050,7 +5890,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('Disabling this setting will allow the creation/modification of local feeds (as opposed to network feeds). Enabling this setting will restrict feed sources to be network based only. When disabled, keep in mind that a malicious site administrator could get access to any arbitrary file on the system that the apache user has access to. Make sure that proper safe-guards are in place. This setting can only be modified via the CLI.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true,
@@ -6069,7 +5908,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Allow cross-origin requests to this instance, matching origins given in Security.cors_origins. Set to false to totally disable'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -6078,7 +5916,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Set the origins from which MISP will allow cross-origin requests. Useful for external integration. Comma seperate if you need more than one.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'null' => true
@@ -6087,7 +5924,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Enable this setting to create verbose logs of synced event data for debugging reasons. Logs are saved in your MISP directory\'s app/files/scripts/tmp/ directory.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBoolFalse',
                     'type' => 'boolean',
                     'null' => true
@@ -6096,7 +5932,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Enables the functionality to monitor users - thereby enabling all logging functionalities for a single user. This functionality is intrusive and potentially heavy on the system - use it with care.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -6105,7 +5940,6 @@ class Server extends AppModel
                     'level' => self::SETTING_OPTIONAL,
                     'description' => __('When enabled, logged in username will be included in X-Username HTTP response header. This is useful for request logging on webserver/proxy side.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -6117,7 +5951,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('The number of tries a user can try to login and fail before the bruteforce protection kicks in.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForNumeric',
                     'type' => 'string',
                 ),
@@ -6125,7 +5958,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('The duration (in seconds) of how long the user will be locked out when the allowed number of login attempts are exhausted.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForNumeric',
                     'type' => 'string',
                 ),
@@ -6136,7 +5968,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('Set to true to automatically regenerate sessions after x number of requests. This might lead to the user getting de-authenticated and is frustrating in general, so only enable it if you really need to regenerate sessions. (Not recommended)'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBoolFalse',
                     'type' => 'boolean',
                 ),
@@ -6144,7 +5975,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('Set to true to check for the user agent string in each request. This can lead to occasional logouts (not recommended).'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBoolFalse',
                     'type' => 'boolean',
                 ),
@@ -6152,7 +5982,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('The session type used by MISP. The default setting is php, which will use the session settings configured in php.ini for the session data (supported options: php, database). The recommended option is php and setting your PHP up to use redis sessions via your php.ini. Just add \'session.save_handler = redis\' and "session.save_path = \'tcp://localhost:6379\'" (replace the latter with your redis connection) to '),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForSessionDefaults',
                     'type' => 'string',
                     'options' => array('php' => 'php', 'database' => 'database', 'cake' => 'cake', 'cache' => 'cache'),
@@ -6161,7 +5990,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('The timeout duration of sessions (in MINUTES). 0 does not mean infinite for the PHP session handler, instead sessions will invalidate immediately.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForNumeric',
                     'type' => 'string'
                 ),
@@ -6169,7 +5997,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('The expiration of the cookie (in MINUTES). The session timeout gets refreshed frequently, however the cookies do not. Generally it is recommended to have a much higher cookie_timeout than timeout.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForCookieTimeout',
                     'type' => 'numeric'
                 )
@@ -6180,16 +6007,14 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The default policy action for the values added to the RPZ.'),
                     'value' => 1,
-                    'errorMessage' => '',
                     'test' => 'testForRPZBehaviour',
                     'type' => 'numeric',
-                    'options' => array(0 => 'DROP', 1 => 'NXDOMAIN', 2 => 'NODATA', 3 => 'Local-Data', 4 => 'PASSTHRU', 5 => 'TCP-only' ),
+                    'options' => array(0 => 'DROP', 1 => 'NXDOMAIN', 2 => 'NODATA', 3 => 'Local-Data', 4 => 'PASSTHRU', 5 => 'TCP-only'),
                 ),
                 'RPZ_walled_garden' => array(
                     'level' => 2,
                     'description' => __('The default walled garden used by the RPZ export if the Local-Data policy setting is picked for the export.'),
                     'value' => '127.0.0.1',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -6197,7 +6022,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The serial in the SOA portion of the zone file. (numeric, best practice is yyyymmddrr where rr is the two digit sub-revision of the file. $date will automatically get converted to the current yyyymmdd, so $date00 is a valid setting). Setting it to $time will give you an unixtime-based serial (good then you need more than 99 revisions per day).'),
                     'value' => '$date00',
-                    'errorMessage' => '',
                     'test' => 'testForRPZSerial',
                     'type' => 'string',
                 ),
@@ -6205,7 +6029,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The refresh specified in the SOA portion of the zone file. (in seconds, or shorthand duration such as 15m)'),
                     'value' => '2h',
-                    'errorMessage' => '',
                     'test' => 'testForRPZDuration',
                     'type' => 'string',
                 ),
@@ -6213,7 +6036,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The retry specified in the SOA portion of the zone file. (in seconds, or shorthand duration such as 15m)'),
                     'value' => '30m',
-                    'errorMessage' => '',
                     'test' => 'testForRPZDuration',
                     'type' => 'string',
                 ),
@@ -6221,7 +6043,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The expiry specified in the SOA portion of the zone file. (in seconds, or shorthand duration such as 15m)'),
                     'value' => '30d',
-                    'errorMessage' => '',
                     'test' => 'testForRPZDuration',
                     'type' => 'string',
                 ),
@@ -6229,7 +6050,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The minimum TTL specified in the SOA portion of the zone file. (in seconds, or shorthand duration such as 15m)'),
                     'value' => '1h',
-                    'errorMessage' => '',
                     'test' => 'testForRPZDuration',
                     'type' => 'string',
                 ),
@@ -6237,7 +6057,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The TTL of the zone file. (in seconds, or shorthand duration such as 15m)'),
                     'value' => '1w',
-                    'errorMessage' => '',
                     'test' => 'testForRPZDuration',
                     'type' => 'string',
                 ),
@@ -6245,7 +6064,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Nameserver'),
                     'value' => 'localhost.',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -6253,7 +6071,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Alternate nameserver'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -6261,7 +6078,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The e-mail address specified in the SOA portion of the zone file.'),
                     'value' => 'root.localhost',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -6269,7 +6085,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables the Kafka pub feature of MISP. Make sure that you install the requirements for the plugin to work. Refer to the installation instructions for more information.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                 ),
@@ -6277,7 +6092,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('A comma separated list of Kafka bootstrap brokers'),
                     'value' => 'kafka:9092',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -6285,7 +6099,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('A path to an ini file with configuration options to be passed to rdkafka. Section headers in the ini file will be ignored.'),
                     'value' => '/etc/rdkafka.ini',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ),
@@ -6293,7 +6106,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enable this setting to include the base64 encoded payloads of malware-samples/attachments in the output.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6301,7 +6113,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables the publishing of any event creations/edits/deletions.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6309,7 +6120,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Topic for publishing event creations/edits/deletions.'),
                     'value' => 'misp_event',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -6317,7 +6127,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('If enabled it will publish to Kafka the event at the time that the event gets published in MISP. Event actions (creation or edit) will not be published to Kafka.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6325,7 +6134,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Topic for publishing event information on publish.'),
                     'value' => 'misp_event_publish',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -6333,7 +6141,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables the publishing of any object creations/edits/deletions.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6341,7 +6148,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Topic for publishing object creations/edits/deletions.'),
                     'value' => 'misp_object',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -6349,7 +6155,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables the publishing of any object reference creations/deletions.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6357,7 +6162,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Topic for publishing object reference creations/deletions.'),
                     'value' => 'misp_object_reference',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -6365,7 +6169,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables the publishing of any attribute creations/edits/soft deletions.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6373,7 +6176,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Topic for publishing attribute creations/edits/soft deletions.'),
                     'value' => 'misp_attribute',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -6381,7 +6183,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables the publishing of any proposal creations/edits/deletions.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6389,7 +6190,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Topic for publishing proposal creations/edits/deletions.'),
                     'value' => 'misp_shadow_attribute',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -6397,7 +6197,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables the publishing of any tag creations/edits/deletions as well as tags being attached to / detached from various MISP elements.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6405,7 +6204,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Topic for publishing tag creations/edits/deletions as well as tags being attached to / detached from various MISP elements.'),
                     'value' => 'misp_tag',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -6413,7 +6211,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables the publishing of new sightings.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6421,7 +6218,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Topic for publishing sightings.'),
                     'value' => 'misp_sighting',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -6429,7 +6225,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables the publishing of new/modified users.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6437,7 +6232,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Topic for publishing new/modified users.'),
                     'value' => 'misp_user',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -6445,7 +6239,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables the publishing of new/modified organisations.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6453,7 +6246,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Topic for publishing new/modified organisations.'),
                     'value' => 'misp_organisation',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -6461,7 +6253,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables the publishing of log entries. Keep in mind, this can get pretty verbose depending on your logging settings.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6469,7 +6260,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Topic for publishing log entries.'),
                     'value' => 'misp_audit',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -6477,7 +6267,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables the pub/sub feature of MISP. Make sure that you install the requirements for the plugin to work. Refer to the installation instructions for more information.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'afterHook' => 'zmqAfterHook',
@@ -6486,7 +6275,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The host that the pub/sub feature will use.'),
                     'value' => '127.0.0.1',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'afterHook' => 'zmqAfterHook',
@@ -6495,7 +6283,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The port that the pub/sub feature will use.'),
                     'value' => 50000,
-                    'errorMessage' => '',
                     'test' => 'testForZMQPortNumber',
                     'type' => 'numeric',
                     'afterHook' => 'zmqAfterHook',
@@ -6504,7 +6291,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The username that client need to use to connect to ZeroMQ.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'afterHook' => 'zmqAfterHook',
@@ -6513,7 +6299,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The password that client need to use to connect to ZeroMQ.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'afterHook' => 'zmqAfterHook',
@@ -6522,7 +6307,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Location of the Redis db used by MISP and the Python PUB script to queue data to be published.'),
                     'value' => 'localhost',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'afterHook' => 'zmqAfterHook',
@@ -6531,7 +6315,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The port that Redis is listening on.'),
                     'value' => 6379,
-                    'errorMessage' => '',
                     'test' => 'testForPortNumber',
                     'type' => 'numeric',
                     'afterHook' => 'zmqAfterHook',
@@ -6540,7 +6323,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The password, if set for Redis.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'type' => 'string',
                     'afterHook' => 'zmqAfterHook',
                 ),
@@ -6548,7 +6330,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The database to be used for queuing messages for the pub/sub functionality.'),
                     'value' => 1,
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'afterHook' => 'zmqAfterHook',
@@ -6557,7 +6338,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The namespace to be used for queuing messages for the pub/sub functionality.'),
                     'value' => 'mispq',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'afterHook' => 'zmqAfterHook',
@@ -6566,7 +6346,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enable this setting to include the base64 encoded payloads of malware-samples/attachments in the output.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6574,7 +6353,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables the publishing of any event creations/edits/deletions.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6582,7 +6360,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables the publishing of any object creations/edits/deletions.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6590,7 +6367,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables the publishing of any object reference creations/deletions.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6598,7 +6374,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables the publishing of any attribute creations/edits/soft deletions.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6606,7 +6381,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables the publishing of any tag creations/edits/deletions as well as tags being attached to / detached from various MISP elements.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6614,7 +6388,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables the publishing of new sightings to the ZMQ pubsub feed.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6622,7 +6395,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables the publishing of new/modified users to the ZMQ pubsub feed.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6630,7 +6402,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables the publishing of new/modified organisations to the ZMQ pubsub feed.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6638,15 +6409,13 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables the publishing of log entries to the ZMQ pubsub feed. Keep in mind, this can get pretty verbose depending on your logging settings.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
-                'ZeroMQ_warninglist_notifications_enable' =>  array(
+                'ZeroMQ_warninglist_notifications_enable' => array(
                     'level' => 2,
                     'description' => __('Enables or disables the publishing of new/modified warninglist to the ZMQ pubsub feed.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6654,7 +6423,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enabled logging to an ElasticSearch instance'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6662,7 +6430,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The URL(s) at which to access ElasticSearch - comma separate if you want to have more than one.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -6670,7 +6437,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The index in which to place logs'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -6678,7 +6444,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enables or disables uploading of malware samples to S3 rather than to disk (WARNING: Get permission from amazon first!)'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6686,7 +6451,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Bucket name to upload to'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -6694,7 +6458,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Region in which your S3 bucket resides'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -6702,7 +6465,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('AWS key to use when uploading samples (WARNING: It\' highly recommended that you use EC2 IAM roles if at all possible)'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -6710,7 +6472,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('AWS secret key to use when uploading samples'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -6718,7 +6479,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('This setting defines who will have access to seeing the reported sightings. The default setting is the event owner organisation alone (in addition to everyone seeing their own contribution) with the other options being Sighting reporters (meaning the event owner and any organisation that provided sighting data about the event) and Everyone (meaning anyone that has access to seeing the event / attribute).'),
                     'value' => 0,
-                    'errorMessage' => '',
                     'test' => 'testForSightingVisibility',
                     'type' => 'numeric',
                     'options' => array(
@@ -6732,7 +6492,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Enabling the anonymisation of sightings will simply aggregate all sightings instead of showing the organisations that have reported a sighting. Users will be able to tell the number of sightings their organisation has submitted and the number of sightings for other organisations'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                 ),
@@ -6740,7 +6499,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('When pushing sightings to another server, report all sightings from this instance as this organisation. This effectively hides all sightings from this instance behind a single organisation to the outside world. Sightings pulled from this instance follow the Sightings_policy above.'),
                     'value' => '0',
-                    'errorMessage' => '',
                     'test' => 'testLocalOrg',
                     'type' => 'numeric',
                     'optionsSource' => function () {
@@ -6751,7 +6509,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Set the range in which sightings will be taken into account when generating graphs. For example a sighting with a sighted_date of 7 years ago might not be relevant anymore. Setting given in number of days, default is 365 days'),
                     'value' => 365,
-                    'errorMessage' => '',
                     'test' => 'testForNumeric',
                     'type' => 'numeric'
                 ),
@@ -6759,7 +6516,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Enable SightingDB integration.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6767,7 +6523,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Enable this functionality if you would like to handle the authentication via an external tool and authenticate with MISP using a custom header.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true,
@@ -6777,7 +6532,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Set the header that MISP should look for here. If left empty it will default to the Authorization header.'),
                     'value' => 'Authorization',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'null' => true
@@ -6786,7 +6540,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Use a header namespace for the auth header - default setting is enabled'),
                     'value' => true,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -6795,7 +6548,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The default header namespace for the auth header - default setting is HTTP_'),
                     'value' => 'HTTP_',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'null' => true
@@ -6804,7 +6556,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('If this setting is enabled then the only way to authenticate will be using the custom header. Alternatively, you can run in mixed mode that will log users in via the header if found, otherwise users will be redirected to the normal login page.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -6813,7 +6564,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('If you are using an external tool to authenticate with MISP and would like to only allow the tool\'s url as a valid point of entry then set this field. '),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'null' => true
@@ -6822,7 +6572,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('The name of the authentication method, this is cosmetic only and will be shown on the user creation page and logs.'),
                     'value' => 'External authentication',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'null' => true
@@ -6831,7 +6580,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Disable the logout button for users authenticate with the external auth mechanism.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6839,7 +6587,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('Enable/disable the enrichment services'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6847,7 +6594,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Set a timeout for the enrichment services'),
                     'value' => 10,
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'numeric'
                 ),
@@ -6855,7 +6601,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('Enable/disable the import services'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6863,7 +6608,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Set a timeout for the import services'),
                     'value' => 10,
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'numeric'
                 ),
@@ -6871,7 +6615,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('The url used to access the import services. By default, it is accessible at http://127.0.0.1:6666'),
                     'value' => 'http://127.0.0.1',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -6879,7 +6622,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('The port used to access the import services. By default, it is accessible at 127.0.0.1:6666'),
                     'value' => '6666',
-                    'errorMessage' => '',
                     'test' => 'testForPortNumber',
                     'type' => 'numeric'
                 ),
@@ -6887,7 +6629,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('The url used to access the export services. By default, it is accessible at http://127.0.0.1:6666'),
                     'value' => 'http://127.0.0.1',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -6895,7 +6636,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('The port used to access the export services. By default, it is accessible at 127.0.0.1:6666'),
                     'value' => '6666',
-                    'errorMessage' => '',
                     'test' => 'testForPortNumber',
                     'type' => 'numeric'
                 ),
@@ -6903,7 +6643,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('Enable/disable the export services'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6911,7 +6650,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Set a timeout for the export services'),
                     'value' => 10,
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'numeric'
                 ),
@@ -6919,7 +6657,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('Enable/disable the hover over information retrieved from the enrichment modules'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6927,7 +6664,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('When enabled, users have to click on the magnifier icon to show the enrichment'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6935,7 +6671,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Set a timeout for the hover services'),
                     'value' => 5,
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'numeric'
                 ),
@@ -6943,7 +6678,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('The url used to access the enrichment services. By default, it is accessible at http://127.0.0.1:6666'),
                     'value' => 'http://127.0.0.1',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -6951,7 +6685,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('The port used to access the enrichment services. By default, it is accessible at 127.0.0.1:6666'),
                     'value' => 6666,
-                    'errorMessage' => '',
                     'test' => 'testForPortNumber',
                     'type' => 'numeric'
                 ),
@@ -6959,7 +6692,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('The url used to access Cortex. By default, it is accessible at http://cortex-url'),
                     'value' => 'http://127.0.0.1',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string'
                 ),
@@ -6967,7 +6699,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('The port used to access Cortex. By default, this is port 9000'),
                     'value' => 9000,
-                    'errorMessage' => '',
                     'test' => 'testForPortNumber',
                     'type' => 'numeric'
                 ),
@@ -6975,7 +6706,6 @@ class Server extends AppModel
                     'level' => 0,
                     'description' => __('Enable/disable the Cortex services'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
@@ -6983,7 +6713,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Set an authentication key to be passed to Cortex'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'null' => true
@@ -6992,7 +6721,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Set a timeout for the Cortex services'),
                     'value' => 120,
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'numeric'
                 ),
@@ -7000,7 +6728,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Set to false to disable SSL verification. This is not recommended.'),
                     'value' => true,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -7009,7 +6736,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Set to false if you wish to ignore hostname match errors when validating certificates.'),
                     'value' => true,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -7018,7 +6744,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Set to true to enable self-signed certificates to be accepted. This requires Cortex_ssl_verify_peer to be enabled.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -7027,7 +6752,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Set to the absolute path of the Certificate Authority file that you wish to use for verifying SSL certificates.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'null' => true
@@ -7036,7 +6760,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Provide your custom authentication users with an external URL to the authentication system to reset their passwords.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'null' => true
@@ -7045,7 +6768,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('Provide a custom logout URL for your users that will log them out using the authentication system you use.'),
                     'value' => '',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'null' => true
@@ -7054,7 +6776,6 @@ class Server extends AppModel
                     'level' => 1,
                     'description' => __('Enable lookups for additional relations via CyCat.'),
                     'value' => false,
-                    'errorMessage' => '',
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
@@ -7063,7 +6784,6 @@ class Server extends AppModel
                     'level' => 2,
                     'description' => __('URL to use for CyCat lookups, if enabled.'),
                     'value' => 'https://api.cycat.org',
-                    'errorMessage' => '',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'null' => true
@@ -7073,7 +6793,6 @@ class Server extends AppModel
                 'level' => 0,
                 'description' => __('The debug level of the instance, always use 0 for production instances.'),
                 'value' => '',
-                'errorMessage' => '',
                 'test' => 'testDebug',
                 'type' => 'numeric',
                 'options' => array(0 => 'Debug off', 1 => 'Debug on', 2 => 'Debug + SQL dump'),
@@ -7082,7 +6801,6 @@ class Server extends AppModel
                 'level' => 0,
                 'description' => __('The debug level of the instance for site admins. This feature allows site admins to run debug mode on a live instance without exposing it to other users. The most verbose option of debug and site_admin_debug is used for site admins.'),
                 'value' => '',
-                'errorMessage' => '',
                 'test' => 'testDebugAdmin',
                 'type' => 'boolean',
                 'null' => true

From abb8cecb7ed25f5e1a302b23ff408738ab687fd8 Mon Sep 17 00:00:00 2001
From: Thijs Kinkhorst <thijs@kinkhorst.com>
Date: Fri, 5 Nov 2021 15:29:54 +0100
Subject: [PATCH 14/45] Fix docblock formatting and add newer settings to
 README documentation

---
 .../Component/Auth/ApacheShibbAuthenticate.php           | 9 +++++----
 app/Plugin/ShibbAuth/README.md                           | 3 +++
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/app/Plugin/ShibbAuth/Controller/Component/Auth/ApacheShibbAuthenticate.php b/app/Plugin/ShibbAuth/Controller/Component/Auth/ApacheShibbAuthenticate.php
index 34ea870e2..a77fd389f 100644
--- a/app/Plugin/ShibbAuth/Controller/Component/Auth/ApacheShibbAuthenticate.php
+++ b/app/Plugin/ShibbAuth/Controller/Component/Auth/ApacheShibbAuthenticate.php
@@ -39,10 +39,11 @@ class ApacheShibbAuthenticate extends BaseAuthenticate
      *          'group_one' => 1,
      *       ),
      *      'DefaultOrg' => 'MY_ORG',
-     *      'BlockOrgModifications' => false         // set to true if you wish for the user's organisation never to be updated during login. Especially useful if you manually change organisations in MISP
-     *      'DefaultRole' => false                   // set to a specific value if you wish to hard-set users created via ApacheShibbAuth
-     *      'BlockRoleModifications' => false        // set to true if you wish for the roles never to be updated during login. Especially *                                               // useful if you manually change roles in MISP
-     *      'BlockOrgModifications' => false        // set to true if you wish for the organizations never to be updated during login. Especially *                                        // useful if you manually change orgs in MISP
+     *      'DefaultRole' => false,                  // set to a specific value if you wish to hard-set users created via ApacheShibbAuth
+     *      'BlockRoleModifications' => false,       // set to true if you wish for the roles never to be updated during login. Especially
+     *                                               // useful if you manually change roles in MISP
+     *      'BlockOrgModifications' => false,        // set to true if you wish for the organizations never to be updated during login. Especially
+     *                                               // useful if you manually change orgs in MISP
      * ),
      * @param CakeRequest $request The request that contains login information.
      * @param CakeResponse $response Unused response object.
diff --git a/app/Plugin/ShibbAuth/README.md b/app/Plugin/ShibbAuth/README.md
index 2ea9eb8b9..abfb80efe 100644
--- a/app/Plugin/ShibbAuth/README.md
+++ b/app/Plugin/ShibbAuth/README.md
@@ -145,6 +145,9 @@ in the list given by apache. By default, you can leave it at ';'.
 	       'possible_group_attribute_value_1' => 1,
           ),
          'DefaultOrg' => 'MISP_DEFAULT_ORG',
+         'DefaultRole' => false,        // set to a specific value if you wish to hard-set users created via ApacheShibbAuth
+         'BlockRoleModifications' => false,        // set to true if you wish for the roles never to be updated during login. Especially useful if you manually change roles in MISP
+         'BlockOrgModifications' => false,         // set to true if you wish for the organizations never to be updated during login. Especially useful if you manually change orgs in MISP
     ),
 ```
 

From c7cbdadccb2c4ad9632e2756a57cc49fd4f479e3 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Fri, 5 Nov 2021 10:49:16 +0100
Subject: [PATCH 15/45] new: [setting] Allow to encrypt setting

---
 app/Console/Command/AdminShell.php |  5 +-
 app/Console/Command/AppShell.php   |  6 +--
 app/Model/Server.php               | 17 ++++---
 app/Model/SystemSetting.php        | 75 ++++++++++++++++++++++++++++--
 4 files changed, 87 insertions(+), 16 deletions(-)

diff --git a/app/Console/Command/AdminShell.php b/app/Console/Command/AdminShell.php
index e5c3ea3b0..c2866284c 100644
--- a/app/Console/Command/AdminShell.php
+++ b/app/Console/Command/AdminShell.php
@@ -375,14 +375,13 @@ class AdminShell extends AppShell
 
     public function getSetting()
     {
-        $this->ConfigLoad->execute();
         $param = empty($this->args[0]) ? 'all' : $this->args[0];
         $settings = $this->Server->serverSettingsRead();
         $result = $settings;
-        if ($param != 'all') {
+        if ($param !== 'all') {
             $result = 'No valid setting found for ' . $param;
             foreach ($settings as $setting) {
-                if ($setting['setting'] == $param) {
+                if ($setting['setting'] === $param) {
                     $result = $setting;
                     break;
                 }
diff --git a/app/Console/Command/AppShell.php b/app/Console/Command/AppShell.php
index b691646fa..1687122cd 100644
--- a/app/Console/Command/AppShell.php
+++ b/app/Console/Command/AppShell.php
@@ -30,10 +30,10 @@ class AppShell extends Shell
 {
     public $tasks = array('ConfigLoad');
     
-    public function perform()
+    public function initialize()
     {
-        $this->initialize();
-        $this->{array_shift($this->args)}();
+        parent::initialize();
+        $this->ConfigLoad->execute();
     }
 
     protected function _welcome()
diff --git a/app/Model/Server.php b/app/Model/Server.php
index ef58dda4e..e5bab99d5 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -3,6 +3,7 @@ App::uses('AppModel', 'Model');
 App::uses('GpgTool', 'Tools');
 App::uses('ServerSyncTool', 'Tools');
 App::uses('FileAccessTool', 'Tools');
+App::uses('SystemSetting', 'Model');
 
 /**
  * @property-read array $serverSettings
@@ -1372,6 +1373,9 @@ class Server extends AppModel
     private function __evaluateLeaf($leafValue, $leafKey, $setting)
     {
         if (isset($setting)) {
+            if ($setting instanceof EncryptedValue) {
+                $setting = $setting->decrypt();
+            }
             if (!empty($leafValue['test'])) {
                 $result = $this->{$leafValue['test']}($setting, empty($leafValue['errorMessage']) ? false : $leafValue['errorMessage']);
                 if ($result !== true) {
@@ -1433,7 +1437,7 @@ class Server extends AppModel
             'fields' => array('Organisation.id', 'Organisation.name')
         ));
 
-        if(!$strict){
+        if (!$strict) {
             return array_replace(array(0 => __('No organisation selected.')), $localOrgs);
         }
 
@@ -1619,7 +1623,6 @@ class Server extends AppModel
         return true;
     }
 
-
     public function getHost()
     {
         if (function_exists('apache_request_headers')) {
@@ -2069,10 +2072,10 @@ class Server extends AppModel
     private function __serverSettingNormaliseValue($data, $value, $setting)
     {
         if (!empty($data['type'])) {
-            if ($data['type'] == 'boolean') {
-                $value = $value ? true : false;
-            } elseif ($data['type'] == 'numeric') {
-                $value = intval($value);
+            if ($data['type'] === 'boolean') {
+                $value = (bool)$value;
+            } elseif ($data['type'] === 'numeric') {
+                $value = (int)$value;
             }
         }
         return $value;
@@ -3263,7 +3266,7 @@ class Server extends AppModel
                 try {
                     $output['version'] = $gpg->getVersion();
                 } catch (Exception $e) {
-                    // ingore
+                    // ignore
                 }
 
                 try {
diff --git a/app/Model/SystemSetting.php b/app/Model/SystemSetting.php
index a9abd79b4..8fd4d6a65 100644
--- a/app/Model/SystemSetting.php
+++ b/app/Model/SystemSetting.php
@@ -2,6 +2,50 @@
 App::uses('AppModel', 'Model');
 App::uses('JsonTool', 'Tools');
 
+/**
+ * Class for ondemand encryption of JSON serialized value
+ */
+class EncryptedValue implements JsonSerializable
+{
+    const ENCRYPTED_MAGIC = "\x1F\x1D";
+
+    /** @var string */
+    private $value;
+
+    public function __construct($value)
+    {
+        $this->value = $value;
+    }
+
+    /**
+     * @return mixed
+     * @throws JsonException
+     * @throws Exception
+     */
+    public function decrypt()
+    {
+        $key = Configure::read('Security.encryption_key');
+        if (empty($key)) {
+            throw new Exception("Value is encrypted, but encryption key is not sed.");
+        }
+        $decrypt = Security::decrypt($this->value, $key);
+        if ($decrypt === false) {
+            throw new Exception("Could not decrypt.");
+        }
+        return JsonTool::decode($decrypt);
+    }
+
+    public function __toString()
+    {
+        return $this->decrypt();
+    }
+
+    public function jsonSerialize()
+    {
+        return $this->decrypt();
+    }
+}
+
 class SystemSetting extends AppModel
 {
     public $actsAs = [
@@ -15,27 +59,52 @@ class SystemSetting extends AppModel
 
     public $primaryKey = 'setting';
 
+    /**
+     * @return array
+     */
     public function getSettings()
     {
         $settings = $this->find('list', [
             'fields' => ['SystemSetting.setting', 'SystemSetting.value'],
         ]);
-        return array_map(['JsonTool', 'decode'], $settings);
+        return array_map([$this, 'decode'], $settings);
     }
 
     /**
-     * @param string $setting
+     * @param string $setting Setting name
      * @param mixed $value
      * @throws Exception
      */
     public function setSetting($setting, $value)
     {
+        $value = JsonTool::encode($value);
+
+        // If encryption is enabled and setting name contains `password` or `apikey` string, encrypt value to protect it
+        $key = Configure::read('Security.encryption_key');
+        if ($key && (strpos($setting, 'password') !== false || strpos($setting, 'apikey') !== false)) {
+            $value = EncryptedValue::ENCRYPTED_MAGIC . Security::encrypt($value, $key);
+        }
+
         $valid = $this->save(['SystemSetting' => [
             'setting' => $setting,
-            'value' => JsonTool::encode($value),
+            'value' => $value,
         ]]);
         if (!$valid) {
             throw new Exception("Could not save system setting `$setting` because of validation errors: " . JsonTool::encode($this->validationErrors));
         }
     }
+
+    /**
+     * @param string $value
+     * @return EncryptedValue|mixed
+     * @throws JsonException
+     */
+    private function decode($value)
+    {
+        if (substr($value, 0, 2) === EncryptedValue::ENCRYPTED_MAGIC) {
+            return new EncryptedValue(substr($value, 2));
+        } else {
+            return JsonTool::decode($value);
+        }
+    }
 }

From e0a6565706eabc7078bdab97db4795e8693e000c Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Fri, 5 Nov 2021 17:01:11 +0100
Subject: [PATCH 16/45] chg: [misp-objects] updated to the latest version

---
 app/files/misp-objects | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/misp-objects b/app/files/misp-objects
index ae6a527bc..0f0093ba4 160000
--- a/app/files/misp-objects
+++ b/app/files/misp-objects
@@ -1 +1 @@
-Subproject commit ae6a527bcb11767abafcc46939c269d297c5de19
+Subproject commit 0f0093ba4b59e469c0e75bd15415ccddc679e4c7

From c5f008acd20e58b67f72f3412a26e6b61717b000 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Fri, 5 Nov 2021 16:50:19 +0100
Subject: [PATCH 17/45] new: [internal] BetterSecurity tool

---
 app/Lib/Tools/BetterSecurity.php | 66 ++++++++++++++++++++++++++++++++
 app/Model/SystemSetting.php      | 12 ++----
 2 files changed, 69 insertions(+), 9 deletions(-)
 create mode 100644 app/Lib/Tools/BetterSecurity.php

diff --git a/app/Lib/Tools/BetterSecurity.php b/app/Lib/Tools/BetterSecurity.php
new file mode 100644
index 000000000..0e98f108c
--- /dev/null
+++ b/app/Lib/Tools/BetterSecurity.php
@@ -0,0 +1,66 @@
+<?php
+
+class BetterSecurity
+{
+    const METHOD = 'AES-256-GCM';
+    const TAG_SIZE = 16;
+
+    /**
+     * @param string $plain
+     * @param string $key
+     * @return string
+     * @throws Exception
+     */
+    public static function encrypt($plain, $key)
+    {
+        if (strlen($key) < 32) {
+            throw new Exception('Invalid key, key must be at least 256 bits (32 bytes) long.');
+        }
+
+        // Generate the encryption key.
+        $key = hash('sha256', $key, true);
+
+        $ivlen = openssl_cipher_iv_length(self::METHOD);
+        $iv = openssl_random_pseudo_bytes($ivlen);
+        if ($iv === false) {
+            throw new Exception('Could not generate random bytes.');
+        }
+        $ciphertext = openssl_encrypt($plain, self::METHOD, $key, OPENSSL_RAW_DATA, $iv, $tag);
+        if ($ciphertext === false) {
+            throw new Exception('Could not encrypt.');
+        }
+        return $iv . $tag . $ciphertext;
+    }
+
+    /**
+     * @param string $cipher
+     * @param string $key
+     * @return string
+     * @throws Exception
+     */
+    public static function decrypt($cipher, $key)
+    {
+        if (strlen($key) < 32) {
+            throw new Exception('Invalid key, key must be at least 256 bits (32 bytes) long.');
+        }
+        if (empty($cipher)) {
+            throw new Exception('The data to decrypt cannot be empty.');
+        }
+
+        // Generate the encryption key.
+        $key = hash('sha256', $key, true);
+
+        $ivSize = openssl_cipher_iv_length(self::METHOD);
+
+        // Split out hmac for comparison
+        $iv = substr($cipher, 0, $ivSize);
+        $tag = substr($cipher, $ivSize, self::TAG_SIZE);
+        $cipher = substr($cipher, $ivSize + self::TAG_SIZE);
+
+        $decrypted = openssl_decrypt($cipher, self::METHOD, $key, true, $iv, $tag);
+        if ($decrypted === false) {
+            throw new Exception('Could not decrypt. Maybe invalid encryption key?');
+        }
+        return $decrypted;
+    }
+}
diff --git a/app/Model/SystemSetting.php b/app/Model/SystemSetting.php
index 8fd4d6a65..ff7283b79 100644
--- a/app/Model/SystemSetting.php
+++ b/app/Model/SystemSetting.php
@@ -1,6 +1,7 @@
 <?php
 App::uses('AppModel', 'Model');
 App::uses('JsonTool', 'Tools');
+App::uses('BetterSecurity', 'Tools');
 
 /**
  * Class for ondemand encryption of JSON serialized value
@@ -24,14 +25,7 @@ class EncryptedValue implements JsonSerializable
      */
     public function decrypt()
     {
-        $key = Configure::read('Security.encryption_key');
-        if (empty($key)) {
-            throw new Exception("Value is encrypted, but encryption key is not sed.");
-        }
-        $decrypt = Security::decrypt($this->value, $key);
-        if ($decrypt === false) {
-            throw new Exception("Could not decrypt.");
-        }
+        $decrypt = BetterSecurity::decrypt($this->value, Configure::read('Security.encryption_key'));
         return JsonTool::decode($decrypt);
     }
 
@@ -82,7 +76,7 @@ class SystemSetting extends AppModel
         // If encryption is enabled and setting name contains `password` or `apikey` string, encrypt value to protect it
         $key = Configure::read('Security.encryption_key');
         if ($key && (strpos($setting, 'password') !== false || strpos($setting, 'apikey') !== false)) {
-            $value = EncryptedValue::ENCRYPTED_MAGIC . Security::encrypt($value, $key);
+            $value = EncryptedValue::ENCRYPTED_MAGIC . BetterSecurity::encrypt($value, $key);
         }
 
         $valid = $this->save(['SystemSetting' => [

From 44737e99a77201a744acfad96c0612f545d36886 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Fri, 5 Nov 2021 21:30:17 +0100
Subject: [PATCH 18/45] chg: [internal] Hide sensitive setting in AuditLog

---
 app/Model/Behavior/AuditLogBehavior.php |  2 +-
 app/Model/Server.php                    |  6 +++++-
 app/Model/SystemSetting.php             | 11 ++++++++++-
 3 files changed, 16 insertions(+), 3 deletions(-)

diff --git a/app/Model/Behavior/AuditLogBehavior.php b/app/Model/Behavior/AuditLogBehavior.php
index 7a6226bfa..ad9751318 100644
--- a/app/Model/Behavior/AuditLogBehavior.php
+++ b/app/Model/Behavior/AuditLogBehavior.php
@@ -351,7 +351,7 @@ class AuditLogBehavior extends ModelBehavior
                 continue;
             }
 
-            if ($key === 'password' || $key === 'authkey') {
+            if ($key === 'password' || $key === 'authkey' || ($key === 'value' && $model->name === 'SystemSetting' && SystemSetting::isSensitive($model->data[$model->alias]['setting']))) {
                 $value = '*****';
                 if ($old !== null) {
                     $old = $value;
diff --git a/app/Model/Server.php b/app/Model/Server.php
index e5bab99d5..c1e262848 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -2161,7 +2161,11 @@ class Server extends AppModel
         $oldValue = Configure::read($setting['name']);
         $settingSaveResult = $this->serverSettingsSaveValue($setting['name'], $value);
         if ($settingSaveResult) {
-            $change = array($setting['name'] => array($oldValue, $value));
+            if (SystemSetting::isSensitive($setting['name'])) {
+                $change = array($setting['name'] => array('*****', '*****'));
+            } else {
+                $change = array($setting['name'] => array($oldValue, $value));
+            }
             $this->loadLog()->createLogEntry($user, 'serverSettingsEdit', 'Server', 0, 'Server setting changed', $change);
 
             // execute after hook
diff --git a/app/Model/SystemSetting.php b/app/Model/SystemSetting.php
index ff7283b79..96d753d33 100644
--- a/app/Model/SystemSetting.php
+++ b/app/Model/SystemSetting.php
@@ -75,7 +75,7 @@ class SystemSetting extends AppModel
 
         // If encryption is enabled and setting name contains `password` or `apikey` string, encrypt value to protect it
         $key = Configure::read('Security.encryption_key');
-        if ($key && (strpos($setting, 'password') !== false || strpos($setting, 'apikey') !== false)) {
+        if ($key && self::isSensitive($setting)) {
             $value = EncryptedValue::ENCRYPTED_MAGIC . BetterSecurity::encrypt($value, $key);
         }
 
@@ -101,4 +101,13 @@ class SystemSetting extends AppModel
             return JsonTool::decode($value);
         }
     }
+
+    /**
+     * @param $setting
+     * @return bool
+     */
+    public static function isSensitive($setting)
+    {
+        return strpos($setting, 'password') !== false || strpos($setting, 'apikey') !== false;
+    }
 }

From f1fb4768799cf5518362ae012d6d5e91d5285aa7 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Fri, 5 Nov 2021 21:32:23 +0100
Subject: [PATCH 19/45] new: [internal] Fix when authkey is invalid

---
 app/Model/Server.php | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/app/Model/Server.php b/app/Model/Server.php
index c1e262848..b381d1eb7 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -1374,7 +1374,12 @@ class Server extends AppModel
     {
         if (isset($setting)) {
             if ($setting instanceof EncryptedValue) {
-                $setting = $setting->decrypt();
+                try {
+                    $setting = $setting->decrypt();
+                } catch (Exception $e) {
+                    $leafValue['errorMessage'] = 'Could not decrypt.';
+                    return $leafValue;
+                }
             }
             if (!empty($leafValue['test'])) {
                 $result = $this->{$leafValue['test']}($setting, empty($leafValue['errorMessage']) ? false : $leafValue['errorMessage']);

From fb448b0f76706d3056db8afb0b930ffbc7360232 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Fri, 5 Nov 2021 21:44:19 +0100
Subject: [PATCH 20/45] chg: [auditlog] Smarter title

---
 app/Model/AuditLog.php | 14 ++++++++------
 1 file changed, 8 insertions(+), 6 deletions(-)

diff --git a/app/Model/AuditLog.php b/app/Model/AuditLog.php
index efc09587f..a1415c20f 100644
--- a/app/Model/AuditLog.php
+++ b/app/Model/AuditLog.php
@@ -131,15 +131,17 @@ class AuditLog extends AppModel
         if (in_array($auditLog['model'], ['Attribute', 'Object', 'ShadowAttribute'], true)) {
             $modelName = $auditLog['model'] === 'ShadowAttribute' ? 'Proposal' : $auditLog['model'];
             $title = __('%s from Event #%s', $modelName, $auditLog['event_id']);
-        } else if ($auditLog['model'] === 'SystemSetting') {
-            $title = $auditLog['model'];
-        } else {
-            $title = "{$auditLog['model']} #{$auditLog['model_id']}";
         }
+
         if (isset($auditLog['model_title']) && $auditLog['model_title']) {
-            $title .= ": {$auditLog['model_title']}";
+            if (isset($title)) {
+                $title .= ": {$auditLog['model_title']}";
+                return $title;
+            } else {
+                return $auditLog['model_title'];
+            }
         }
-        return $title;
+        return '';
     }
 
     /**

From 9bd1bd50468ca6cafdae5cb68fb561274e41cde5 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Fri, 5 Nov 2021 21:51:31 +0100
Subject: [PATCH 21/45] new: [internal] encryption_key config

---
 app/Model/Server.php                          | 52 ++++++++++++++++---
 app/Model/SystemSetting.php                   | 29 +++++++++++
 .../Elements/healthElements/settings_row.ctp  |  6 ++-
 3 files changed, 77 insertions(+), 10 deletions(-)

diff --git a/app/Model/Server.php b/app/Model/Server.php
index b381d1eb7..07f1215b4 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -1382,7 +1382,11 @@ class Server extends AppModel
                 }
             }
             if (!empty($leafValue['test'])) {
-                $result = $this->{$leafValue['test']}($setting, empty($leafValue['errorMessage']) ? false : $leafValue['errorMessage']);
+                if ($leafValue['test'] instanceof Closure) {
+                    $result = $leafValue['test']($setting);
+                } else {
+                    $result = $this->{$leafValue['test']}($setting, empty($leafValue['errorMessage']) ? false : $leafValue['errorMessage']);
+                }
                 if ($result !== true) {
                     $leafValue['error'] = 1;
                     if ($result !== false) {
@@ -2150,8 +2154,12 @@ class Server extends AppModel
         } else if ($setting['type'] === 'numeric') {
             $value = (int)($value);
         }
-        if (!empty($setting['test'])) {
-            $testResult = $this->{$setting['test']}($value);
+        if (isset($setting['test'])) {
+            if ($setting['test'] instanceof Closure) {
+                $testResult = $setting['test']($value);
+            } else {
+                $testResult = $this->{$setting['test']}($value);
+            }
         } else {
             $testResult = true;  # No test defined for this setting: cannot fail
         }
@@ -2164,7 +2172,8 @@ class Server extends AppModel
             return $errorMessage;
         }
         $oldValue = Configure::read($setting['name']);
-        $settingSaveResult = $this->serverSettingsSaveValue($setting['name'], $value);
+        $fileOnly = isset($setting['file_only']) && $setting['file_only'];
+        $settingSaveResult = $this->serverSettingsSaveValue($setting['name'], $value, $fileOnly);
         if ($settingSaveResult) {
             if (SystemSetting::isSensitive($setting['name'])) {
                 $change = array($setting['name'] => array('*****', '*****'));
@@ -2175,7 +2184,11 @@ class Server extends AppModel
 
             // execute after hook
             if (isset($setting['afterHook'])) {
-                $afterResult = call_user_func_array(array($this, $setting['afterHook']), array($setting['name'], $value));
+                if ($setting['afterHook'] instanceof Closure) {
+                    $afterResult = $setting['afterHook']($setting['name'], $value, $oldValue);
+                } else {
+                    $afterResult = call_user_func_array(array($this, $setting['afterHook']), array($setting['name'], $value, $oldValue));
+                }
                 if ($afterResult !== true) {
                     $change = 'There was an issue after setting a new setting. The error message returned is: ' . $afterResult;
                     $this->loadLog()->createLogEntry($user, 'serverSettingsEdit', 'Server', 0, 'Server setting issue', $change);
@@ -2191,12 +2204,13 @@ class Server extends AppModel
     /**
      * @param string $setting
      * @param mixed $value
+     * @param bool $fileOnly If true, always store value in config file even when `MISP.system_setting_db` is enabled
      * @return bool
      * @throws Exception
      */
-    public function serverSettingsSaveValue($setting, $value)
+    public function serverSettingsSaveValue($setting, $value, $fileOnly = false)
     {
-        if (Configure::read('MISP.system_setting_db')) {
+        if (!$fileOnly && Configure::read('MISP.system_setting_db')) {
             /** @var SystemSetting $systemSetting */
             $systemSetting = ClassRegistry::init('SystemSetting');
             $systemSetting->setSetting($setting, $value);
@@ -5955,7 +5969,29 @@ class Server extends AppModel
                     'test' => 'testBool',
                     'type' => 'boolean',
                     'null' => true
-                ]
+                ],
+                'encryption_key' => [
+                    'level' => self::SETTING_OPTIONAL,
+                    'description' => __('Encryption key used to store sensitive data (like authkeys) in database encrypted. If empty, data are stored unecrypted. Required PHP 7.1 or newer.'),
+                    'value' => '',
+                    'test' => function ($value) {
+                        if (strlen($value) < 32) {
+                            return __('Encryption key must be at least 32 chars long.');
+                        }
+                        return true;
+                    },
+                    'afterHook' => function ($setting, $new, $old) {
+                        /** @var SystemSetting $systemSetting */
+                        $systemSetting = ClassRegistry::init('SystemSetting');
+                        $systemSetting->reencrypt($old, $new);
+                        return true;
+                    },
+                    'type' => 'string',
+                    'null' => true,
+                    'cli_only' => true,
+                    'redacted' => true,
+                    'file_only' => true,
+                ],
             ),
             'SecureAuth' => array(
                 'branch' => 1,
diff --git a/app/Model/SystemSetting.php b/app/Model/SystemSetting.php
index 96d753d33..bfda18954 100644
--- a/app/Model/SystemSetting.php
+++ b/app/Model/SystemSetting.php
@@ -88,6 +88,35 @@ class SystemSetting extends AppModel
         }
     }
 
+    /**
+     * @param string|null $old Old (or current) encryption key.
+     * @param string|null $new New encryption key. If empty, encrypted values will be decrypted.
+     * @throws JsonException
+     */
+    public function reencrypt($old, $new)
+    {
+        $settings = $this->find('list', [
+            'fields' => ['SystemSetting.setting', 'SystemSetting.value'],
+        ]);
+        $toSave = [];
+        foreach ($settings as $setting => $value) {
+            if (!self::isSensitive($setting)) {
+                continue;
+            }
+            if (substr($value, 0, 2) === EncryptedValue::ENCRYPTED_MAGIC) {
+                $value = BetterSecurity::decrypt(substr($value, 2), $old);
+            }
+            if (!empty($new)) {
+                $value = EncryptedValue::ENCRYPTED_MAGIC . BetterSecurity::encrypt($value, $new);
+            }
+            $toSave[] = ['SystemSetting' => [
+                'setting' => $setting,
+                'value' => $value,
+            ]];
+        }
+        return $this->saveMany($toSave);
+    }
+
     /**
      * @param string $value
      * @return EncryptedValue|mixed
diff --git a/app/View/Elements/healthElements/settings_row.ctp b/app/View/Elements/healthElements/settings_row.ctp
index d46c03478..f236e7492 100644
--- a/app/View/Elements/healthElements/settings_row.ctp
+++ b/app/View/Elements/healthElements/settings_row.ctp
@@ -7,7 +7,9 @@
             2 => 'success',
             3 => 'info'
         );
-        if ($setting['type'] == 'boolean') $setting['value'] = ($setting['value'] === true ? 'true' : 'false');
+        if ($setting['type'] === 'boolean') {
+            $setting['value'] = $setting['value'] === true ? 'true' : 'false';
+        }
         if (isset($setting['options'])) {
             $setting['value'] = empty($setting['options'][$setting['value']]) ? null : $setting['options'][$setting['value']];
         }
@@ -100,4 +102,4 @@
             $columns
         );
     }
-?>
+

From f35053d2883f5e61c6daa30257feeeebfe6f2df8 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Fri, 5 Nov 2021 22:26:12 +0100
Subject: [PATCH 22/45] new: [internal] Save to config file just what was in
 file

---
 app/Model/Server.php        | 46 +++++++++++++++----------------------
 app/Model/SystemSetting.php |  1 +
 2 files changed, 19 insertions(+), 28 deletions(-)

diff --git a/app/Model/Server.php b/app/Model/Server.php
index 07f1215b4..36435f14c 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -1440,11 +1440,15 @@ class Server extends AppModel
 
     private function loadLocalOrganisations($strict = false)
     {
-        $localOrgs = $this->Organisation->find('list', array(
-            'conditions' => array('local' => 1),
-            'recursive' => -1,
-            'fields' => array('Organisation.id', 'Organisation.name')
-        ));
+        static $localOrgs;
+
+        if ($localOrgs === null) {
+            $localOrgs = $this->Organisation->find('list', array(
+                'conditions' => array('local' => 1),
+                'recursive' => -1,
+                'fields' => array('Organisation.id', 'Organisation.name')
+            ));
+        }
 
         if (!$strict) {
             return array_replace(array(0 => __('No organisation selected.')), $localOrgs);
@@ -1518,15 +1522,10 @@ class Server extends AppModel
 
     public function testLocalOrgStrict($value)
     {
-        $this->Organisation = ClassRegistry::init('Organisation');
         if ($value == 0) {
             return 'No organisation selected';
         }
-        $local_orgs = $this->Organisation->find('list', array(
-            'conditions' => array('local' => 1),
-            'recursive' => -1,
-            'fields' => array('Organisation.id', 'Organisation.name')
-        ));
+        $local_orgs = $this->loadLocalOrganisations(true);
         if (in_array($value, array_keys($local_orgs))) {
             return true;
         }
@@ -2250,22 +2249,11 @@ class Server extends AppModel
                 }
             }
         }
-        Configure::write($setting, $value);
-        $arrayFix = array(
-            'Security.auth',
-            'ApacheSecureAuth.ldapFilter'
-        );
-        foreach ($arrayFix as $settingFix) {
-            if (Configure::read($settingFix) && is_array(Configure::read($settingFix)) && !empty(Configure::read($settingFix))) {
-                $arrayElements = array();
-                foreach (Configure::read($settingFix) as $array) {
-                    if (!in_array($array, $arrayElements)) {
-                        $arrayElements[] = $array;
-                    }
-                }
-                Configure::write($settingFix, $arrayElements);
-            }
-        }
+
+        /** @var array $config */
+        require $configFilePath;
+        $config = Hash::insert($config, $setting, $value);
+
         $settingsToSave = array(
             'debug', 'MISP', 'GnuPG', 'SMIME', 'Proxy', 'SecureAuth',
             'Security', 'Session.defaults', 'Session.timeout', 'Session.cookieTimeout',
@@ -2274,7 +2262,9 @@ class Server extends AppModel
         );
         $settingsArray = array();
         foreach ($settingsToSave as $setting) {
-            $settingsArray[$setting] = Configure::read($setting);
+            if (Hash::check($config, $setting)) {
+                $settingsArray[$setting] = Hash::get($config, $setting);
+            }
         }
         $settingsString = var_export($settingsArray, true);
         $settingsString = '<?php' . "\n" . '$config = ' . $settingsString . ';';
diff --git a/app/Model/SystemSetting.php b/app/Model/SystemSetting.php
index bfda18954..8ae25a293 100644
--- a/app/Model/SystemSetting.php
+++ b/app/Model/SystemSetting.php
@@ -132,6 +132,7 @@ class SystemSetting extends AppModel
     }
 
     /**
+     * Sensitive setting are passwords or api keys.
      * @param $setting
      * @return bool
      */

From 51feb8034f8dddd1e30569470467bf3315256fca Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Fri, 5 Nov 2021 22:49:11 +0100
Subject: [PATCH 23/45] new: [CLI] Allow to set setting value to `null`

---
 app/Console/Command/AdminShell.php | 20 +++++++++++++------
 app/Model/Server.php               | 31 ++++++++++++++++++------------
 2 files changed, 33 insertions(+), 18 deletions(-)

diff --git a/app/Console/Command/AdminShell.php b/app/Console/Command/AdminShell.php
index c2866284c..d520c5de0 100644
--- a/app/Console/Command/AdminShell.php
+++ b/app/Console/Command/AdminShell.php
@@ -22,12 +22,18 @@ class AdminShell extends AppShell
                     'value' => ['help' => __('Setting value'), 'required' => true],
                 ],
                 'options' => [
-                    'force' => array(
+                    'force' => [
                         'short' => 'f',
                         'help' => 'Force the command.',
                         'default' => false,
                         'boolean' => true
-                    )
+                    ],
+                    'null' => [
+                        'short' => 'n',
+                        'help' => 'Set the value to null.',
+                        'default' => false,
+                        'boolean' => true
+                    ],
                 ]
             ],
         ]);
@@ -392,15 +398,17 @@ class AdminShell extends AppShell
 
     public function setSetting()
     {
-        $setting_name = !isset($this->args[0]) ? null : $this->args[0];
-        $value = !isset($this->args[1]) ? null : $this->args[1];
+        list($setting_name, $value) = $this->args;
         if ($value === 'false') {
             $value = 0;
         } elseif ($value === 'true') {
             $value = 1;
         }
+        if ($this->params['null']) {
+            $value = null;
+        }
         $cli_user = array('id' => 0, 'email' => 'SYSTEM', 'Organisation' => array('name' => 'SYSTEM'));
-        if (empty($setting_name) || $value === null) {
+        if (empty($setting_name) || ($value === null && !$this->params['null'])) {
             die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Set setting'] . PHP_EOL);
         }
         $setting = $this->Server->getSettingData($setting_name);
@@ -410,7 +418,7 @@ class AdminShell extends AppShell
         }
         $result = $this->Server->serverSettingsEditValue($cli_user, $setting, $value, $this->params['force']);
         if ($result === true) {
-            echo 'Setting "' . $setting_name . '" changed to ' . $value . PHP_EOL;
+            $this->out(__('Setting "%s" changed to %s', $setting_name, is_string($value) ? '"' . $value . '"' : (string)$value));
         } else {
             $message = __("The setting change was rejected. MISP considers the requested setting value as invalid and would lead to the following error:\n\n\"%s\"\n\nIf you still want to force this change, please supply the --force argument.\n", $result);
             $this->error(__('Setting change rejected.'), $message);
diff --git a/app/Model/Server.php b/app/Model/Server.php
index 36435f14c..8a47b3d88 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -2147,21 +2147,28 @@ class Server extends AppModel
                 return $beforeResult;
             }
         }
-        $value = trim($value);
-        if ($setting['type'] === 'boolean') {
-            $value = (bool)$value;
-        } else if ($setting['type'] === 'numeric') {
-            $value = (int)($value);
-        }
-        if (isset($setting['test'])) {
-            if ($setting['test'] instanceof Closure) {
-                $testResult = $setting['test']($value);
-            } else {
-                $testResult = $this->{$setting['test']}($value);
+        if ($value !== null) {
+            $value = trim($value);
+            if ($setting['type'] === 'boolean') {
+                $value = (bool)$value;
+            } else if ($setting['type'] === 'numeric') {
+                $value = (int)($value);
             }
+            if (isset($setting['test'])) {
+                if ($setting['test'] instanceof Closure) {
+                    $testResult = $setting['test']($value);
+                } else {
+                    $testResult = $this->{$setting['test']}($value);
+                }
+            } else {
+                $testResult = true;  # No test defined for this setting: cannot fail
+            }
+        } else if (isset($setting['null']) && $setting['null']) {
+            $testResult = true;
         } else {
-            $testResult = true;  # No test defined for this setting: cannot fail
+            $testResult = __('Value could not be null.');
         }
+
         if (!$forceSave && $testResult !== true) {
             if ($testResult === false) {
                 $errorMessage = $setting['errorMessage'];

From 3e6e906ca6bf424e34bc4f0f1d0999bcdea78533 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 6 Nov 2021 00:53:10 +0100
Subject: [PATCH 24/45] chg: [optimise] Reduce number of SQL queries for login
 page

---
 app/Controller/UsersController.php | 33 +++++++++++++++---------------
 1 file changed, 17 insertions(+), 16 deletions(-)

diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php
index 52b8c0539..30f1be3e3 100644
--- a/app/Controller/UsersController.php
+++ b/app/Controller/UsersController.php
@@ -1210,8 +1210,7 @@ class UsersController extends AppController
                 }
             }
             // populate the DB with the first role (site admin) if it's empty
-            $this->loadModel('Role');
-            if ($this->Role->find('count') == 0) {
+            if (!$this->User->Role->hasAny()) {
                 $siteAdmin = array('Role' => array(
                     'id' => 1,
                     'name' => 'Site Admin',
@@ -1230,14 +1229,14 @@ class UsersController extends AppController
                     'perm_template' => 1,
                     'perm_tagger' => 1,
                 ));
-                $this->Role->save($siteAdmin);
+                $this->User->Role->save($siteAdmin);
                 // PostgreSQL: update value of auto incremented serial primary key after setting the column by force
-                if ($dataSource == 'Database/Postgres') {
+                if ($dataSource === 'Database/Postgres') {
                     $sql = "SELECT setval('roles_id_seq', (SELECT MAX(id) FROM roles));";
-                    $this->Role->query($sql);
+                    $this->User->Role->query($sql);
                 }
             }
-            if ($this->User->Organisation->find('count', array('conditions' => array('Organisation.local' => true))) == 0) {
+            if (!$this->User->Organisation->hasAny(array('Organisation.local' => true))) {
                 $this->User->runUpdates();
                 $date = date('Y-m-d H:i:s');
                 $org = array('Organisation' => array(
@@ -1253,23 +1252,25 @@ class UsersController extends AppController
                 ));
                 $this->User->Organisation->save($org);
                 // PostgreSQL: update value of auto incremented serial primary key after setting the column by force
-                if ($dataSource == 'Database/Postgres') {
+                if ($dataSource === 'Database/Postgres') {
                     $sql = "SELECT setval('organisations_id_seq', (SELECT MAX(id) FROM organisations));";
                     $this->User->Organisation->query($sql);
                 }
                 $org_id = $this->User->Organisation->id;
-            } else {
-                $hostOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.name' => Configure::read('MISP.org'), 'Organisation.local' => true), 'recursive' => -1));
-                if (!empty($hostOrg)) {
-                    $org_id = $hostOrg['Organisation']['id'];
-                } else {
-                    $firstOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.local' => true), 'order' => 'Organisation.id ASC'));
-                    $org_id = $firstOrg['Organisation']['id'];
-                }
             }
 
             // populate the DB with the first user if it's empty
-            if ($this->User->find('count') == 0) {
+            if (!$this->User->hasAny()) {
+                if (!isset($org_id)) {
+                    $hostOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.name' => Configure::read('MISP.org'), 'Organisation.local' => true), 'recursive' => -1));
+                    if (!empty($hostOrg)) {
+                        $org_id = $hostOrg['Organisation']['id'];
+                    } else {
+                        $firstOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.local' => true), 'order' => 'Organisation.id ASC'));
+                        $org_id = $firstOrg['Organisation']['id'];
+                    }
+                }
+
                 $this->User->runUpdates();
                 $this->User->createInitialUser($org_id);
             }

From 8f5a357e0844b8a4ef6259798bad19afa44995a6 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 6 Nov 2021 01:08:20 +0100
Subject: [PATCH 25/45] chg: [systemsetting] Better checking if setting is
 sensitive

---
 app/Model/SystemSetting.php | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/app/Model/SystemSetting.php b/app/Model/SystemSetting.php
index 8ae25a293..d9d50e90f 100644
--- a/app/Model/SystemSetting.php
+++ b/app/Model/SystemSetting.php
@@ -133,11 +133,17 @@ class SystemSetting extends AppModel
 
     /**
      * Sensitive setting are passwords or api keys.
-     * @param $setting
+     * @param string $setting Setting name
      * @return bool
      */
     public static function isSensitive($setting)
     {
-        return strpos($setting, 'password') !== false || strpos($setting, 'apikey') !== false;
+        if ($setting === 'Security.encryption_key' || $setting === 'Security.salt') {
+            return true;
+        }
+        if (substr($setting, 0, 7) === 'Plugin.' && (strpos($setting, 'apikey') !== false || strpos($setting, 'secret') !== false)) {
+            return true;
+        }
+        return strpos($setting, 'password') !== false;
     }
 }

From 45591786c2c30197ff530e3dcce7e43d58fc423d Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 6 Nov 2021 09:41:28 +0100
Subject: [PATCH 26/45] chg: [internal] Simplify Server model code

---
 app/Model/Server.php | 35 ++++++++++++++++-------------------
 1 file changed, 16 insertions(+), 19 deletions(-)

diff --git a/app/Model/Server.php b/app/Model/Server.php
index 8a47b3d88..fc31082e2 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -3,6 +3,7 @@ App::uses('AppModel', 'Model');
 App::uses('GpgTool', 'Tools');
 App::uses('ServerSyncTool', 'Tools');
 App::uses('FileAccessTool', 'Tools');
+App::uses('JsonTool', 'Tools');
 App::uses('SystemSetting', 'Model');
 
 /**
@@ -162,14 +163,6 @@ class Server extends AppModel
         )
     );
 
-    private $__settingTabMergeRules = array(
-            'GnuPG' => 'Encryption',
-            'SMIME' => 'Encryption',
-            'misc' => 'Security',
-            'Security' => 'Security',
-            'Session' => 'Security'
-    );
-
     public $validEventIndexFilters = array('searchall', 'searchpublished', 'searchorg', 'searchtag', 'searcheventid', 'searchdate', 'searcheventinfo', 'searchthreatlevel', 'searchdistribution', 'searchanalysis', 'searchattribute');
 
     public function beforeSave($options = array())
@@ -1343,12 +1336,20 @@ class Server extends AppModel
 
     public function serverSettingsRead($unsorted = false)
     {
+        $settingTabMergeRules = array(
+            'GnuPG' => 'Encryption',
+            'SMIME' => 'Encryption',
+            'misc' => 'Security',
+            'Security' => 'Security',
+            'Session' => 'Security'
+        );
+
         $serverSettings = $this->getCurrentServerSettings();
         $currentSettings = Configure::read();
         $finalSettingsUnsorted = $this->__serverSettingsRead($serverSettings, $currentSettings);
         foreach ($finalSettingsUnsorted as $key => $temp) {
-            if (isset($this->__settingTabMergeRules[$temp['tab']])) {
-                $finalSettingsUnsorted[$key]['tab'] = $this->__settingTabMergeRules[$temp['tab']];
+            if (isset($settingTabMergeRules[$temp['tab']])) {
+                $finalSettingsUnsorted[$key]['tab'] = $settingTabMergeRules[$temp['tab']];
             }
         }
         if ($unsorted) {
@@ -2831,13 +2832,10 @@ class Server extends AppModel
 
     public function getExpectedDBSchema()
     {
-        App::uses('Folder', 'Utility');
-        $file = new File(ROOT . DS . 'db_schema.json', true);
-        $dbExpectedSchema = json_decode($file->read(), true);
-        $file->close();
-        if (!is_null($dbExpectedSchema)) {
-            return $dbExpectedSchema;
-        } else {
+        try {
+            $content = FileAccessTool::readFromFile(ROOT . DS . 'db_schema.json');
+            return JsonTool::decode($content);
+        } catch (Exception $e) {
             return false;
         }
     }
@@ -2890,8 +2888,7 @@ class Server extends AppModel
                 }
                 $dbActualIndexes[$table] = $this->getDatabaseIndexes($this->getDataSource()->config['database'], $table);
             }
-        }
-        else if ($dataSource == 'Database/Postgres') {
+        } else if ($dataSource == 'Database/Postgres') {
             return array('Database/Postgres' => array('description' => __('Can\'t check database schema for Postgres database type')));
         }
         return ['schema' => $dbActualSchema, 'column' => $tableColumnNames, 'indexes' => $dbActualIndexes];

From 2c1c1e14efd194e46abc3e6a1a386498ea829cfc Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 6 Nov 2021 10:11:57 +0100
Subject: [PATCH 27/45] chg: [internal] Simplify code for pulling events

---
 app/Model/Server.php | 42 +++++++++++++++++++++++++-----------------
 1 file changed, 25 insertions(+), 17 deletions(-)

diff --git a/app/Model/Server.php b/app/Model/Server.php
index b7243bf5d..6d478b537 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -233,7 +233,12 @@ class Server extends AppModel
         return false;
     }
 
-    private function __updatePulledEventBeforeInsert(&$event, $server, $user)
+    /**
+     * @param array $event
+     * @param array $server
+     * @param array $user
+     */
+    private function __updatePulledEventBeforeInsert(array &$event, array $server, array $user)
     {
         // we have an Event array
         // The event came from a pull, so it should be locked.
@@ -261,9 +266,9 @@ class Server extends AppModel
                     }
                 }
             }
-            if (isset($event['Event']['Attribute']) && !empty($event['Event']['Attribute'])) {
-                foreach ($event['Event']['Attribute'] as $key => $a) {
-                    switch ($a['distribution']) {
+            if (isset($event['Event']['Attribute'])) {
+                foreach ($event['Event']['Attribute'] as $key => $attribute) {
+                    switch ($attribute['distribution']) {
                         case '1':
                             $event['Event']['Attribute'][$key]['distribution'] = '0';
                             break;
@@ -272,8 +277,8 @@ class Server extends AppModel
                             break;
                     }
                     // We remove local tags obtained via pull
-                    if (isset($a['Tag'])) {
-                        foreach ($a['Tag'] as $k => $v) {
+                    if (isset($attribute['Tag'])) {
+                        foreach ($attribute['Tag'] as $k => $v) {
                             if ($v['local']) {
                                 unset($event['Event']['Attribute'][$key]['Tag'][$k]);
                             }
@@ -281,9 +286,9 @@ class Server extends AppModel
                     }
                 }
             }
-            if (isset($event['Event']['Object']) && !empty($event['Event']['Object'])) {
-                foreach ($event['Event']['Object'] as $i => $o) {
-                    switch ($o['distribution']) {
+            if (isset($event['Event']['Object'])) {
+                foreach ($event['Event']['Object'] as $i => $object) {
+                    switch ($object['distribution']) {
                         case '1':
                             $event['Event']['Object'][$i]['distribution'] = '0';
                             break;
@@ -291,8 +296,8 @@ class Server extends AppModel
                             $event['Event']['Object'][$i]['distribution'] = '1';
                             break;
                     }
-                    if (isset($event['Event']['Object'][$i]['Attribute']) && !empty($event['Event']['Object'][$i]['Attribute'])) {
-                        foreach ($event['Event']['Object'][$i]['Attribute'] as $j => $a) {
+                    if (isset($object['Attribute'])) {
+                        foreach ($object['Attribute'] as $j => $a) {
                             switch ($a['distribution']) {
                                 case '1':
                                     $event['Event']['Object'][$i]['Attribute'][$j]['distribution'] = '0';
@@ -313,7 +318,7 @@ class Server extends AppModel
                     }
                 }
             }
-            if (isset($event['Event']['EventReport']) && !empty($event['Event']['EventReport'])) {
+            if (isset($event['Event']['EventReport'])) {
                 foreach ($event['Event']['EventReport'] as $key => $r) {
                     switch ($r['distribution']) {
                         case '1':
@@ -329,10 +334,13 @@ class Server extends AppModel
 
         // Distribution, set reporter of the event, being the admin that initiated the pull
         $event['Event']['user_id'] = $user['id'];
-        return $event;
     }
 
-    private function __checkIfEventSaveAble($event)
+    /**
+     * @param array $event
+     * @return bool True if event is not empty
+     */
+    private function __checkIfEventSaveAble(array $event)
     {
         if (!empty($event['Event']['Attribute'])) {
             foreach ($event['Event']['Attribute'] as $attribute) {
@@ -379,7 +387,7 @@ class Server extends AppModel
             $result = $eventModel->_add($event, true, $user, $server['Server']['org_id'], $passAlong, true, $jobId);
             if ($result) {
                 $successes[] = $eventId;
-                if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) {
+                if ($this->pubToZmq('event')) {
                     $pubSubTool = $this->getPubSubTool();
                     $pubSubTool->event_save(array('Event' => $eventId, 'Server' => $server['Server']['id']), 'add_from_connected_server');
                 }
@@ -393,7 +401,7 @@ class Server extends AppModel
                 $result = $eventModel->_edit($event, $user, $existingEvent['Event']['id'], $jobId, $passAlong, $force);
                 if ($result === true) {
                     $successes[] = $eventId;
-                    if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) {
+                    if ($this->pubToZmq('event')) {
                         $pubSubTool = $this->getPubSubTool();
                         $pubSubTool->event_save(array('Event' => $eventId, 'Server' => $server['Server']['id']), 'edit_from_connected_server');
                     }
@@ -431,7 +439,7 @@ class Server extends AppModel
             if ($this->__checkIfEventIsBlockedBeforePull($event)) {
                 return false;
             }
-            $event = $this->__updatePulledEventBeforeInsert($event, $serverSync->server(), $user);
+            $this->__updatePulledEventBeforeInsert($event, $serverSync->server(), $user);
             if (!$this->__checkIfEventSaveAble($event)) {
                 $fails[$eventId] = __('Empty event detected.');
             } else {

From 859ed7c373c8eb31c54d22f792b70a42f025a0de Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 6 Nov 2021 10:44:29 +0100
Subject: [PATCH 28/45] chg: [internal] Remove SysLogLogable from SystemSetting

---
 app/Model/SystemSetting.php | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/app/Model/SystemSetting.php b/app/Model/SystemSetting.php
index d9d50e90f..80ed192d2 100644
--- a/app/Model/SystemSetting.php
+++ b/app/Model/SystemSetting.php
@@ -43,11 +43,6 @@ class EncryptedValue implements JsonSerializable
 class SystemSetting extends AppModel
 {
     public $actsAs = [
-        'SysLogLogable.SysLogLogable' => [
-            'userModel' => 'User',
-            'userKey' => 'user_id',
-            'change' => 'full'
-        ],
         'AuditLog'
     ];
 

From 5905ce3a889c538ced1b40969ff52b91f526831d Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 6 Nov 2021 11:44:13 +0100
Subject: [PATCH 29/45] chg: [internal] Optimise loading attributes when doing
 search

---
 app/Controller/AttributesController.php | 12 ++++++------
 app/Model/Attribute.php                 | 20 ++++++++++++--------
 2 files changed, 18 insertions(+), 14 deletions(-)

diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php
index 742cce259..10f4ecf22 100644
--- a/app/Controller/AttributesController.php
+++ b/app/Controller/AttributesController.php
@@ -67,19 +67,18 @@ class AttributesController extends AppController
             'Event' => array(
                 'fields' =>  array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.info', 'Event.user_id', 'Event.date'),
             ),
-            'AttributeTag' => array('Tag'),
+            'AttributeTag',
             'Object' => array(
                 'fields' => array('Object.id', 'Object.distribution', 'Object.sharing_group_id')
             ),
             'SharingGroup' => ['fields' => ['SharingGroup.name']],
         );
-        $this->Attribute->contain(array('AttributeTag' => array('Tag')));
         $this->set('isSearch', 0);
         $attributes = $this->paginate();
+        $this->Attribute->attachTagsToAttributes($attributes, ['includeAllTags' => true]);
+
         if ($this->_isRest()) {
-            foreach ($attributes as $k => $attribute) {
-                $attributes[$k] = $attribute['Attribute'];
-            }
+            $attributes = array_column($attributes, 'Attribute');
             return $this->RestResponse->viewData($attributes, $this->response->type());
         }
 
@@ -1589,13 +1588,14 @@ class AttributesController extends AppController
                 'Event' => array(
                     'fields' =>  array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.info', 'Event.user_id', 'Event.date'),
                 ),
-                'AttributeTag' => array('Tag'),
+                'AttributeTag',
                 'Object' => array(
                     'fields' => array('Object.id', 'Object.distribution', 'Object.sharing_group_id')
                 ),
                 'SharingGroup' => ['fields' => ['SharingGroup.name']],
             );
             $attributes = $this->paginate();
+            $this->Attribute->attachTagsToAttributes($attributes, ['includeAllTags' => true]);
 
             $orgTable = $this->Attribute->Event->Orgc->find('all', [
                 'fields' => ['Orgc.id', 'Orgc.name', 'Orgc.uuid'],
diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php
index 77c9f3d61..1de64f4f7 100644
--- a/app/Model/Attribute.php
+++ b/app/Model/Attribute.php
@@ -2031,7 +2031,7 @@ class Attribute extends AppModel
                 'Event' => array(
                     'fields' => array('id', 'info', 'org_id', 'orgc_id', 'uuid'),
                 ),
-                'AttributeTag', // tags are fetched separately, @see Attribute::__attachTagsToAttributes
+                'AttributeTag', // tags are fetched separately, @see Attribute::attachTagsToAttributes
                 'Object' => array(
                     'fields' => array('id', 'distribution', 'sharing_group_id')
                 )
@@ -2232,7 +2232,7 @@ class Attribute extends AppModel
                 unset($eventsById, $result); // unset result is important, because it is reference
             }
 
-            $this->__attachTagsToAttributes($results, $options);
+            $this->attachTagsToAttributes($results, $options);
 
             foreach ($results as $k => $result) {
                 if (!empty($options['includeSightings'])) {
@@ -2361,7 +2361,14 @@ class Attribute extends AppModel
         return $eventsById;
     }
 
-    private function __attachTagsToAttributes(array &$attributes, array $options)
+    /**
+     * Options:
+     *  - includeAllTags - if true, include also exportable tags
+     *
+     * @param array $attributes
+     * @param array $options
+     */
+    public function attachTagsToAttributes(array &$attributes, array $options)
     {
         $tagIdsToFetch = [];
         foreach ($attributes as $attribute) {
@@ -2382,15 +2389,12 @@ class Attribute extends AppModel
             $conditions['Tag.exportable'] = 1;
         }
 
-        $tagsToModify = $this->AttributeTag->Tag->find('all', [
+        $tags = $this->AttributeTag->Tag->find('all', [
             'conditions' => $conditions,
             'fields' => ['id', 'name', 'colour', 'numerical_value'],
             'recursive' => -1,
         ]);
-        $tags = [];
-        foreach ($tagsToModify as $tag) {
-            $tags[$tag['Tag']['id']] = $tag['Tag'];
-        }
+        $tags = array_column(array_column($tags, 'Tag'), null, 'id');
 
         foreach ($attributes as $k => $attribute) {
             $tagCulled = false;

From 0263bff6b8a15e98ddccc5c7da553d685425e575 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 6 Nov 2021 11:56:40 +0100
Subject: [PATCH 30/45] fix: [internal] Fetching clusters

---
 app/Model/GalaxyCluster.php         | 28 +++++++++++++++++++++-------
 app/Model/GalaxyClusterRelation.php |  2 +-
 2 files changed, 22 insertions(+), 8 deletions(-)

diff --git a/app/Model/GalaxyCluster.php b/app/Model/GalaxyCluster.php
index 132705880..864e5fa2b 100644
--- a/app/Model/GalaxyCluster.php
+++ b/app/Model/GalaxyCluster.php
@@ -1085,18 +1085,23 @@ class GalaxyCluster extends AppModel
         if (isset($options['list']) && $options['list']) {
             return $this->find('list', $params);
         }
+
         if (isset($options['first']) && $options['first']) {
             $clusters = $this->find('first', $params);
         } else if (isset($options['count']) && $options['count']) {
-            $clusterCount = $this->find('count', $params);
-            return $clusterCount;
+            return $this->find('count', $params);
         } else {
             $clusters = $this->find('all', $params);
         }
+
         if (empty($clusters)) {
             return $clusters;
         }
 
+        if (isset($options['first']) && $options['first']) {
+            $clusters = [$clusters];
+        }
+
         if ($full) {
             $clusterIds = array_column(array_column($clusters, 'GalaxyCluster'), 'id');
             $targetingClusterRelations = $this->TargetingClusterRelation->fetchRelations($user, array(
@@ -1112,11 +1117,15 @@ class GalaxyCluster extends AppModel
             $tagsToFetch = Hash::extract($clusters, "{n}.GalaxyClusterRelation.{n}.GalaxyClusterRelationTag.{n}.tag_id");
             $tagsToFetch = array_merge($tagsToFetch, Hash::extract($targetingClusterRelations, "GalaxyClusterRelationTag.{n}.tag_id"));
 
-            $tags = $this->GalaxyClusterRelation->GalaxyClusterRelationTag->Tag->find('all', [
-                'conditions' => ['id' => array_unique($tagsToFetch)],
-                'recursive' => -1,
-            ]);
-            $tags = array_column(array_column($tags, 'Tag'), null, 'id');
+            if (!empty($tagsToFetch)) {
+                $tags = $this->GalaxyClusterRelation->GalaxyClusterRelationTag->Tag->find('all', [
+                    'conditions' => ['id' => array_unique($tagsToFetch)],
+                    'recursive' => -1,
+                ]);
+                $tags = array_column(array_column($tags, 'Tag'), null, 'id');
+            } else {
+                $tags = [];
+            }
 
             foreach ($targetingClusterRelations as $k => $targetingClusterRelation) {
                 if (!empty($targetingClusterRelation['GalaxyClusterRelationTag'])) {
@@ -1162,6 +1171,11 @@ class GalaxyCluster extends AppModel
             }
             $clusters[$i] = $this->arrangeData($clusters[$i]);
         }
+
+        if (isset($options['first']) && $options['first']) {
+            return $clusters[0];
+        }
+
         return $clusters;
     }
 
diff --git a/app/Model/GalaxyClusterRelation.php b/app/Model/GalaxyClusterRelation.php
index bfb656f38..00b69153b 100644
--- a/app/Model/GalaxyClusterRelation.php
+++ b/app/Model/GalaxyClusterRelation.php
@@ -79,9 +79,9 @@ class GalaxyClusterRelation extends AppModel
 
     public function buildConditions($user, $clusterConditions = true)
     {
-        $this->Event = ClassRegistry::init('Event');
         $conditions = [];
         if (!$user['Role']['perm_site_admin']) {
+            $this->Event = ClassRegistry::init('Event');
             $alias = $this->alias;
             $sgids = $this->Event->cacheSgids($user, true);
             $gcOwnerIds = $this->SourceCluster->cacheGalaxyClusterOwnerIDs($user);

From 9da544666804f3d8814b480ee4da10cb6bc8e376 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 6 Nov 2021 12:38:08 +0100
Subject: [PATCH 31/45] chg: [internal] Assign galaxies in one query

---
 app/Controller/AttributesController.php | 50 ++++++++++++++++++++-----
 1 file changed, 40 insertions(+), 10 deletions(-)

diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php
index 10f4ecf22..e2b75adfa 100644
--- a/app/Controller/AttributesController.php
+++ b/app/Controller/AttributesController.php
@@ -1649,7 +1649,7 @@ class AttributesController extends AppController
         }
     }
 
-    private function __searchUI($attributes)
+    private function __searchUI(array $attributes)
     {
         if (empty($attributes)) {
             return [[], []];
@@ -1661,9 +1661,9 @@ class AttributesController extends AppController
         $this->loadModel('AttachmentScan');
         $user = $this->Auth->user();
         $attributeIds = [];
-        foreach ($attributes as $k => $attribute) {
-            $attributeId = $attribute['Attribute']['id'];
-            $attributeIds[] = $attributeId;
+        $galaxyTags = [];
+        foreach ($attributes as &$attribute) {
+            $attributeIds[] = $attribute['Attribute']['id'];
             if ($this->Attribute->isImage($attribute['Attribute'])) {
                 if (extension_loaded('gd')) {
                     // if extension is loaded, the data is not passed to the view because it is asynchronously fetched
@@ -1671,21 +1671,30 @@ class AttributesController extends AppController
                 } else {
                     $attribute['Attribute']['image'] = $this->Attribute->base64EncodeAttachment($attribute['Attribute']);
                 }
-                $attributes[$k] = $attribute;
             }
             if ($attribute['Attribute']['type'] === 'attachment' && $this->AttachmentScan->isEnabled()) {
                 $infected = $this->AttachmentScan->isInfected(AttachmentScan::TYPE_ATTRIBUTE, $attribute['Attribute']['id']);
-                $attributes[$k]['Attribute']['infected'] = $infected;
+                $attribute['Attribute']['infected'] = $infected;
             }
 
             if ($attribute['Attribute']['distribution'] == 4) {
-                $attributes[$k]['Attribute']['SharingGroup'] = $attribute['SharingGroup'];
+                $attribute['Attribute']['SharingGroup'] = $attribute['SharingGroup'];
             }
 
-            $attributes[$k]['Attribute']['AttributeTag'] = $attributes[$k]['AttributeTag'];
-            $attributes[$k]['Attribute'] = $this->Attribute->Event->massageTags($this->Auth->user(), $attributes[$k]['Attribute'], 'Attribute', $excludeGalaxy = false, $cullGalaxyTags = true);
-            unset($attributes[$k]['AttributeTag']);
+            $attribute['Attribute']['AttributeTag'] = $attribute['AttributeTag'];
+            foreach ($attribute['Attribute']['AttributeTag'] as $at) {
+                if (substr($at['Tag']['name'], 0, 12) === 'misp-galaxy:') {
+                    $galaxyTags[] = $at['Tag']['name'];
+                }
+            }
+            unset($attribute['AttributeTag']);
         }
+        unset($attribute);
+
+        // Fetch galaxy clusters in one query
+        $this->loadModel('GalaxyCluster');
+        $clusters = $this->GalaxyCluster->getClusters($galaxyTags, $user, true, false);
+        $clusters = array_column(array_column($clusters, 'GalaxyCluster'), null, 'tag_id');
 
         // Fetch correlations in one query
         $correlations = $this->Attribute->Event->getRelatedAttributes($user, $attributeIds, false, 'attribute');
@@ -1696,6 +1705,27 @@ class AttributesController extends AppController
         $attributesWithFeedCorrelations = $this->Feed->attachFeedCorrelations(array_column($attributes, 'Attribute'), $user, $fakeEventArray);
 
         foreach ($attributes as $k => $attribute) {
+            // Assign galaxies
+            $galaxies = [];
+            foreach ($attribute['Attribute']['AttributeTag'] as $k2 => $attributeTag) {
+                if (!isset($clusters[$attributeTag['Tag']['id']])) {
+                    continue;
+                }
+                $cluster = $clusters[$attributeTag['Tag']['id']];
+                $galaxyId = $cluster['Galaxy']['id'];
+                $cluster['local'] = isset($attributeTag['local']) ? $attributeTag['local'] : false;
+                if (isset($attribute['Attribute']['Galaxy'][$galaxyId])) {
+                    unset($cluster['Galaxy']);
+                    $galaxies[$galaxyId]['GalaxyCluster'][] = $cluster;
+                } else {
+                    $galaxies[$galaxyId] = $cluster['Galaxy'];
+                    unset($cluster['Galaxy']);
+                    $galaxies[$galaxyId]['GalaxyCluster'] = [$cluster];
+                }
+                unset($attributes[$k]['Attribute']['AttributeTag'][$k2]); // remove galaxy tag
+            }
+            $attributes[$k]['Attribute']['Galaxy'] = array_values($galaxies);
+
             if (isset($attributesWithFeedCorrelations[$k]['Feed'])) {
                 $attributes[$k]['Attribute']['Feed'] = $attributesWithFeedCorrelations[$k]['Feed'];
             }

From 0de880f8ed87a89728d6e5fd695c89da6dd68a75 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 6 Nov 2021 12:57:55 +0100
Subject: [PATCH 32/45] chg: [internal] Faster attaching tags to events

---
 app/Model/Event.php | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)

diff --git a/app/Model/Event.php b/app/Model/Event.php
index cf18576b2..562c296ac 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -2362,10 +2362,7 @@ class Event extends AppModel
             return;
         }
 
-        $clustersByTagNames = [];
-        foreach ($clusters as $cluster) {
-            $clustersByTagNames[mb_strtolower($cluster['GalaxyCluster']['tag_name'])] = $cluster['GalaxyCluster'];
-        }
+        $clustersByTagIds = array_column(array_column($clusters, 'GalaxyCluster'), null, 'tag_id');
         unset($clusters);
 
         if (isset($event['EventTag'])) {
@@ -2373,9 +2370,9 @@ class Event extends AppModel
                 if (!$eventTag['Tag']['is_galaxy']) {
                     continue;
                 }
-                $tagName = mb_strtolower($eventTag['Tag']['name']);
-                if (isset($clustersByTagNames[$tagName])) {
-                    $cluster = $clustersByTagNames[$tagName];
+                $tagId = $eventTag['Tag']['id'];
+                if (isset($clustersByTagIds[$tagId])) {
+                    $cluster = $clustersByTagIds[$tagId];
                     $galaxyId = $cluster['Galaxy']['id'];
                     $cluster['local'] = isset($eventTag['local']) ? $eventTag['local'] : false;
                     if (isset($event['Galaxy'][$galaxyId])) {
@@ -2397,9 +2394,9 @@ class Event extends AppModel
                         if (!$attributeTag['Tag']['is_galaxy']) {
                             continue;
                         }
-                        $tagName = mb_strtolower($attributeTag['Tag']['name']);
-                        if (isset($clustersByTagNames[$tagName])) {
-                            $cluster = $clustersByTagNames[$tagName];
+                        $tagId = $attributeTag['Tag']['id'];
+                        if (isset($clustersByTagIds[$tagId])) {
+                            $cluster = $clustersByTagIds[$tagId];
                             $galaxyId = $cluster['Galaxy']['id'];
                             $cluster['local'] = isset($attributeTag['local']) ? $attributeTag['local'] : false;
                             if (isset($attribute['Galaxy'][$galaxyId])) {

From 788a888a8b339603bd89cff2855db1862b11aee3 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 6 Nov 2021 16:33:35 +0100
Subject: [PATCH 33/45] chg: [internal] Simplified attribute pagination

---
 app/Controller/AttributesController.php | 61 ++++++++-----------------
 1 file changed, 20 insertions(+), 41 deletions(-)

diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php
index e2b75adfa..3e52fe241 100644
--- a/app/Controller/AttributesController.php
+++ b/app/Controller/AttributesController.php
@@ -11,12 +11,23 @@ class AttributesController extends AppController
 {
     public $components = array('Security', 'RequestHandler');
 
-    public $paginate = array(
-            'limit' => 60,
-            'maxLimit' => 9999,
-            'conditions' => array('AND' => array('Attribute.deleted' => 0)),
-            'order' => 'Attribute.event_id DESC'
-    );
+    public $paginate = [
+        'limit' => 60,
+        'maxLimit' => 9999,
+        'conditions' => array('AND' => array('Attribute.deleted' => 0)),
+        'order' => 'Attribute.event_id DESC',
+        'recursive' => -1,
+        'contain' => array(
+            'Event' => array(
+                'fields' =>  array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.info', 'Event.user_id', 'Event.date'),
+            ),
+            'AttributeTag',
+            'Object' => array(
+                'fields' => array('Object.id', 'Object.distribution', 'Object.sharing_group_id')
+            ),
+            'SharingGroup' => ['fields' => ['SharingGroup.name']],
+        ),
+    ];
 
     public function beforeFilter()
     {
@@ -53,27 +64,11 @@ class AttributesController extends AppController
                 $this->params->addParams(array('pass' => array($id))); // FIXME find better way to change id variable if uuid is found. params->url and params->here is not modified accordingly now
             }
         }
-        // do not show private to other orgs
-        if (!$this->_isSiteAdmin()) {
-            $this->paginate = Set::merge($this->paginate, array('conditions' => $this->Attribute->buildConditions($this->Auth->user())));
-        }
     }
 
     public function index()
     {
-        $this->Attribute->recursive = -1;
-        $this->paginate['recursive'] = -1;
-        $this->paginate['contain'] = array(
-            'Event' => array(
-                'fields' =>  array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.info', 'Event.user_id', 'Event.date'),
-            ),
-            'AttributeTag',
-            'Object' => array(
-                'fields' => array('Object.id', 'Object.distribution', 'Object.sharing_group_id')
-            ),
-            'SharingGroup' => ['fields' => ['SharingGroup.name']],
-        );
-        $this->set('isSearch', 0);
+        $this->paginate['conditions']['AND'][] = $this->Attribute->buildConditions($this->Auth->user());
         $attributes = $this->paginate();
         $this->Attribute->attachTagsToAttributes($attributes, ['includeAllTags' => true]);
 
@@ -93,6 +88,7 @@ class AttributesController extends AppController
         }
 
         list($attributes, $sightingsData) = $this->__searchUI($attributes);
+        $this->set('isSearch', 0);
         $this->set('sightingsData', $sightingsData);
         $this->set('orgTable', array_column($orgTable, 'name', 'id'));
         $this->set('shortDist', $this->Attribute->shortDist);
@@ -1576,24 +1572,7 @@ class AttributesController extends AppController
             if (!isset($params['conditions']['Attribute.deleted'])) {
                 $params['conditions']['Attribute.deleted'] = 0;
             }
-            $this->paginate = $params;
-            if (empty($this->paginate['limit'])) {
-                $this->paginate['limit'] = 60;
-            }
-            if (empty($this->paginate['page'])) {
-                $this->paginate['page'] = 1;
-            }
-            $this->paginate['recursive'] = -1;
-            $this->paginate['contain'] = array(
-                'Event' => array(
-                    'fields' =>  array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.info', 'Event.user_id', 'Event.date'),
-                ),
-                'AttributeTag',
-                'Object' => array(
-                    'fields' => array('Object.id', 'Object.distribution', 'Object.sharing_group_id')
-                ),
-                'SharingGroup' => ['fields' => ['SharingGroup.name']],
-            );
+            $this->paginate['conditions'] = $params['conditions'];
             $attributes = $this->paginate();
             $this->Attribute->attachTagsToAttributes($attributes, ['includeAllTags' => true]);
 

From d3bc2a0ba93406e3dc97ae82b403671820691a14 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 6 Nov 2021 16:34:04 +0100
Subject: [PATCH 34/45] chg: [internal] Simplified editing field

---
 app/Controller/AttributesController.php | 24 ++++++++++++++----------
 app/Model/Behavior/AuditLogBehavior.php |  2 +-
 2 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php
index 3e52fe241..090aceaab 100644
--- a/app/Controller/AttributesController.php
+++ b/app/Controller/AttributesController.php
@@ -915,28 +915,25 @@ class AttributesController extends AppController
         if (empty($attribute)) {
             return new CakeResponse(array('body'=> json_encode(array('fail' => false, 'errors' => 'Invalid attribute')), 'status' => 200, 'type' => 'json'));
         }
-        $this->Attribute->data = $attribute;
-        $this->Attribute->id = $attribute['Attribute']['id'];
         if (!$this->__canModifyEvent($attribute)) {
             return new CakeResponse(array('body' => json_encode(array('fail' => false, 'errors' => 'You do not have permission to do that')), 'status' => 200, 'type' => 'json'));
         }
         if (!$this->_isRest()) {
             $this->Attribute->Event->insertLock($this->Auth->user(), $attribute['Attribute']['event_id']);
         }
-        $validFields = array('value', 'category', 'type', 'comment', 'to_ids', 'distribution', 'first_seen', 'last_seen');
-        $changed = false;
         if (empty($this->request->data['Attribute'])) {
             $this->request->data = array('Attribute' => $this->request->data);
             if (empty($this->request->data['Attribute'])) {
                 throw new MethodNotAllowedException(__('Invalid input.'));
             }
         }
+        $validFields = array('value', 'category', 'type', 'comment', 'to_ids', 'distribution', 'first_seen', 'last_seen');
+        $changed = false;
         foreach ($this->request->data['Attribute'] as $changedKey => $changedField) {
-            if (!in_array($changedKey, $validFields)) {
+            if (!in_array($changedKey, $validFields, true)) {
                 throw new MethodNotAllowedException(__('Invalid field.'));
             }
             if ($attribute['Attribute'][$changedKey] == $changedField) {
-                $this->autoRender = false;
                 return new CakeResponse(array('body'=> json_encode(array('errors'=> array('value' => 'nochange'))), 'status'=>200, 'type' => 'json'));
             }
             $attribute['Attribute'][$changedKey] = $changedField;
@@ -947,16 +944,23 @@ class AttributesController extends AppController
         }
         $date = new DateTime();
         $attribute['Attribute']['timestamp'] = $date->getTimestamp();
-        if ($this->Attribute->save($attribute)) {
-            $this->Attribute->Event->unpublishEvent($attribute['Attribute']['event_id']);
+
+        $fieldsToSave = ['timestamp'];
+        if ($changedKey === 'value') {
+            $fieldsToSave[] = 'value1';
+            $fieldsToSave[] = 'value2';
+        } else {
+            $fieldsToSave[] = $changedKey;
+        }
+
+        if ($this->Attribute->save($attribute, true, $fieldsToSave)) {
+            $this->Attribute->Event->unpublishEvent($attribute['Attribute']['event_id'], false, $date->getTimestamp());
 
             if ($attribute['Attribute']['object_id'] != 0) {
                 $this->Attribute->Object->updateTimestamp($attribute['Attribute']['object_id'], $date->getTimestamp());
             }
-            $this->autoRender = false;
             return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => 'Field updated.', 'check_publish' => true)), 'status'=>200, 'type' => 'json'));
         } else {
-            $this->autoRender = false;
             return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => $this->Attribute->validationErrors)), 'status'=>200, 'type' => 'json'));
         }
     }
diff --git a/app/Model/Behavior/AuditLogBehavior.php b/app/Model/Behavior/AuditLogBehavior.php
index ad9751318..128c6809a 100644
--- a/app/Model/Behavior/AuditLogBehavior.php
+++ b/app/Model/Behavior/AuditLogBehavior.php
@@ -128,7 +128,7 @@ class AuditLogBehavior extends ModelBehavior
             if (isset($data['deleted'])) {
                 if ($data['deleted']) {
                     $action = AuditLog::ACTION_SOFT_DELETE;
-                } else if ($this->old[$model->alias]['deleted']) {
+                } else if (isset($this->old[$model->alias]['deleted']) && $this->old[$model->alias]['deleted']) {
                     $action = AuditLog::ACTION_UNDELETE;
                 }
             }

From 18bcf683e08d08ab1b71b1f1ef97948b44cd0e55 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Wed, 19 May 2021 11:00:15 +0200
Subject: [PATCH 35/45] new: [UI] Define custom right menu link

---
 app/Model/Server.php              | 16 +++++++++++++++-
 app/View/Elements/global_menu.ctp | 12 +++++++++---
 2 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/app/Model/Server.php b/app/Model/Server.php
index 0b35b0eae..8f7100026 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -5530,7 +5530,21 @@ class Server extends AppModel
                     'type' => 'boolean',
                     'null' => true,
                     'cli_only' => true,
-                ]
+                ],
+                'menu_custom_right_link' => [
+                    'level' => self::SETTING_OPTIONAL,
+                    'description' => __('Custom right menu URL.'),
+                    'value' => null,
+                    'type' => 'string',
+                    'null' => true,
+                ],
+                'menu_custom_right_link_html' => [
+                    'level' => self::SETTING_OPTIONAL,
+                    'description' => __('Custom right menu text (it is possible to use HTML).'),
+                    'value' => null,
+                    'type' => 'string',
+                    'null' => true,
+                ],
             ),
             'GnuPG' => array(
                 'branch' => 1,
diff --git a/app/View/Elements/global_menu.ctp b/app/View/Elements/global_menu.ctp
index 9303626ad..2793797a6 100755
--- a/app/View/Elements/global_menu.ctp
+++ b/app/View/Elements/global_menu.ctp
@@ -495,8 +495,8 @@
                 'html' => sprintf(
                     '<span class="fas fa-star %s" id="setHomePage" title="%s" role="img" aria-label="%s" data-current-page="%s"></span>',
                     (!empty($homepage['path']) && $homepage['path'] === $this->here) ? 'orange' : '',
-		    __('Set the current page as your home page in MISP'),
-		    __('Set the current page as your home page in MISP'),
+		            __('Set the current page as your home page in MISP'),
+		            __('Set the current page as your home page in MISP'),
                     h($this->here)
                 )
             ),
@@ -505,6 +505,12 @@
                 'url' => empty($homepage['path']) ? $baseurl : $baseurl . h($homepage['path']),
                 'html' => '<span class="logoBlueStatic bold" id="smallLogo">MISP</span>'
             ),
+            [
+                'type' => 'root',
+                'url' => Configure::read('MISP.menu_custom_right_link'),
+                'html' => Configure::read('MISP.menu_custom_right_link_html'),
+                'requirement' => !empty(Configure::read('MISP.menu_custom_right_link')),
+            ],
             array(
                 'type' => 'root',
                 'url' => $baseurl . '/dashboards',
@@ -528,7 +534,7 @@
         );
     }
 ?>
-<div id="topBar" class="navbar navbar-inverse <?php echo $debugMode;?>">
+<div id="topBar" class="navbar navbar-inverse <?= $debugMode ?>">
   <div class="navbar-inner">
     <ul class="nav">
         <?php

From 70d1829ccb68613494a699f978238bd98062df7d Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 6 Nov 2021 16:49:56 +0100
Subject: [PATCH 36/45] chg: [internal] Unpublish event timestamp

---
 app/Model/Event.php | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/app/Model/Event.php b/app/Model/Event.php
index 562c296ac..cd43ba679 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -5817,7 +5817,14 @@ class Event extends AppModel
         }
     }
 
-    public function unpublishEvent($id, $proposalLock = false)
+    /**
+     * @param int $id
+     * @param bool $proposalLock
+     * @param int|null $timestamp If not provided, current time will be used
+     * @return array|bool|mixed|null
+     * @throws Exception
+     */
+    public function unpublishEvent($id, $proposalLock = false, $timestamp = null)
     {
         $event = $this->find('first', array(
             'recursive' => -1,
@@ -5829,7 +5836,7 @@ class Event extends AppModel
         }
         $fields = ['published', 'timestamp'];
         $event['Event']['published'] = 0;
-        $event['Event']['timestamp'] = time();
+        $event['Event']['timestamp'] = $timestamp ?: time();
         if ($proposalLock) {
             $event['Event']['proposal_email_lock'] = 0;
             $fields[] = 'proposal_email_lock';

From 2f88a5a60c64d5597a97bc2daf379ab1e4a669c8 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 6 Nov 2021 17:15:49 +0100
Subject: [PATCH 37/45] chg: [internal] Optimise loading event info in AuditLog

---
 app/Controller/AuditLogsController.php | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/app/Controller/AuditLogsController.php b/app/Controller/AuditLogsController.php
index 25c4006f0..74d1e93e8 100644
--- a/app/Controller/AuditLogsController.php
+++ b/app/Controller/AuditLogsController.php
@@ -485,10 +485,12 @@ class AuditLogsController extends AppController
 
         if (!empty($eventIds)) {
             $this->loadModel('Event');
-            $events = $this->Event->fetchSimpleEvents($this->Auth->user(), [
-                'conditions' => ['Event.id' => array_unique($eventIds)],
+            $conditions = $this->Event->createEventConditions($this->Auth->user());
+            $conditions['Event.id'] = array_unique($eventIds);
+            $events = $this->Event->find('list', [
+                'conditions' => $conditions,
+                'fields' => ['Event.id', 'Event.info'],
             ]);
-            $events = array_column(array_column($events, 'Event'), null, 'id');
         }
 
         $links = [
@@ -526,7 +528,7 @@ class AuditLogsController extends AppController
                 case 'Event':
                     if (isset($events[$modelId])) {
                         $url = '/events/view/' . $modelId;
-                        $eventInfo = $events[$modelId]['info'];
+                        $eventInfo = $events[$modelId];
                     }
                     break;
                 case 'ObjectReference':
@@ -536,7 +538,7 @@ class AuditLogsController extends AppController
                             $url .= '/deleted:2';
                         }
                         if (isset($events[$objects[$objectReferences[$modelId]]['event_id']])) {
-                            $eventInfo = $events[$objects[$objectReferences[$modelId]]['event_id']]['info'];
+                            $eventInfo = $events[$objects[$objectReferences[$modelId]]['event_id']];
                         }
                     }
                     break;
@@ -547,7 +549,7 @@ class AuditLogsController extends AppController
                             $url .= '/deleted:2';
                         }
                         if (isset($events[$objects[$modelId]['event_id']])) {
-                            $eventInfo = $events[$objects[$modelId]['event_id']]['info'];
+                            $eventInfo = $events[$objects[$modelId]['event_id']];
                         }
                     }
                     break;
@@ -558,7 +560,7 @@ class AuditLogsController extends AppController
                             $url .= '/deleted:2';
                         }
                         if (isset($events[$attributes[$modelId]['event_id']])) {
-                            $eventInfo = $events[$attributes[$modelId]['event_id']]['info'];
+                            $eventInfo = $events[$attributes[$modelId]['event_id']];
                         }
                     }
                     break;
@@ -566,7 +568,7 @@ class AuditLogsController extends AppController
                     if (isset($shadowAttributes[$modelId])) {
                         $url = '/events/view/' . $shadowAttributes[$modelId]['event_id'] . '/focus:' . $shadowAttributes[$modelId]['uuid'];
                         if (isset($events[$shadowAttributes[$modelId]['event_id']])) {
-                            $eventInfo = $events[$shadowAttributes[$modelId]['event_id']]['info'];
+                            $eventInfo = $events[$shadowAttributes[$modelId]['event_id']];
                         }
                     }
                     break;

From 4f90e79359726ec41b0eecb2f271bfdec0d56961 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 6 Nov 2021 17:33:58 +0100
Subject: [PATCH 38/45] chg: [internal] Do not try to fetch empty clusters

---
 app/Controller/AttributesController.php | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php
index 090aceaab..4af4ebb45 100644
--- a/app/Controller/AttributesController.php
+++ b/app/Controller/AttributesController.php
@@ -1675,9 +1675,13 @@ class AttributesController extends AppController
         unset($attribute);
 
         // Fetch galaxy clusters in one query
-        $this->loadModel('GalaxyCluster');
-        $clusters = $this->GalaxyCluster->getClusters($galaxyTags, $user, true, false);
-        $clusters = array_column(array_column($clusters, 'GalaxyCluster'), null, 'tag_id');
+        if (!empty($galaxyTags)) {
+            $this->loadModel('GalaxyCluster');
+            $clusters = $this->GalaxyCluster->getClusters($galaxyTags, $user, true, false);
+            $clusters = array_column(array_column($clusters, 'GalaxyCluster'), null, 'tag_id');
+        } else {
+            $clusters = [];
+        }
 
         // Fetch correlations in one query
         $correlations = $this->Attribute->Event->getRelatedAttributes($user, $attributeIds, false, 'attribute');

From c238e1f7c0ecd6255a05947d7c69aef2acbfabc8 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 6 Nov 2021 19:35:40 +0100
Subject: [PATCH 39/45] fix: [internal] Remove potentially problematic and non
 functional searchAlternate

---
 app/Controller/AttributesController.php   | 61 +----------------------
 app/Controller/Component/ACLComponent.php |  1 -
 app/View/Attributes/search.ctp            |  4 --
 3 files changed, 1 insertion(+), 65 deletions(-)

diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php
index 4af4ebb45..77c166603 100644
--- a/app/Controller/AttributesController.php
+++ b/app/Controller/AttributesController.php
@@ -70,13 +70,13 @@ class AttributesController extends AppController
     {
         $this->paginate['conditions']['AND'][] = $this->Attribute->buildConditions($this->Auth->user());
         $attributes = $this->paginate();
-        $this->Attribute->attachTagsToAttributes($attributes, ['includeAllTags' => true]);
 
         if ($this->_isRest()) {
             $attributes = array_column($attributes, 'Attribute');
             return $this->RestResponse->viewData($attributes, $this->response->type());
         }
 
+        $this->Attribute->attachTagsToAttributes($attributes, ['includeAllTags' => true]);
         $orgTable = $this->Attribute->Event->Orgc->find('all', [
             'fields' => ['Orgc.id', 'Orgc.name', 'Orgc.uuid'],
         ]);
@@ -1724,65 +1724,6 @@ class AttributesController extends AppController
         return array($attributes, $sightingsData);
     }
 
-    // If the checkbox for the alternate search is ticked, then this method is called to return the data to be represented
-    // This alternate view will show a list of events with matching search results and the percentage of those matched attributes being marked as to_ids
-    // events are sorted based on relevance (as in the percentage of matches being flagged as indicators for IDS)
-    public function searchAlternate($data)
-    {
-        $attributes = $this->Attribute->fetchAttributes(
-            $this->Auth->user(),
-            array(
-                'conditions' => array(
-                    'AND' => $data
-                ),
-                'contain' => array('Event' => array('Orgc' => array('fields' => array('Orgc.name')))),
-                'fields' => array(
-                    'Attribute.id', 'Attribute.event_id', 'Attribute.type', 'Attribute.category', 'Attribute.to_ids', 'Attribute.value', 'Attribute.distribution',
-                    'Event.id', 'Event.org_id', 'Event.orgc_id', 'Event.info', 'Event.distribution', 'Event.attribute_count', 'Event.date',
-                )
-            )
-        );
-        $events = array();
-        foreach ($attributes as $attribute) {
-            if (isset($events[$attribute['Event']['id']])) {
-                if ($attribute['Attribute']['to_ids']) {
-                    $events[$attribute['Event']['id']]['to_ids']++;
-                } else {
-                    $events[$attribute['Event']['id']]['no_ids']++;
-                }
-            } else {
-                $events[$attribute['Event']['id']]['Event'] = $attribute['Event'];
-                $events[$attribute['Event']['id']]['to_ids'] = 0;
-                $events[$attribute['Event']['id']]['no_ids'] = 0;
-                if ($attribute['Attribute']['to_ids']) {
-                    $events[$attribute['Event']['id']]['to_ids']++;
-                } else {
-                    $events[$attribute['Event']['id']]['no_ids']++;
-                }
-            }
-        }
-        foreach ($events as $key => $event) {
-            $events[$key]['relevance'] = 100 * $event['to_ids'] / ($event['no_ids'] + $event['to_ids']);
-        }
-        if (!empty($events)) {
-            $events = $this->__subval_sort($events, 'relevance');
-        }
-        return $events;
-    }
-
-    // Sort the array of arrays based on a value of a sub-array
-    private function __subval_sort($a, $subkey)
-    {
-        foreach ($a as $k=>$v) {
-            $b[$k] = strtolower($v[$subkey]);
-        }
-        arsort($b);
-        foreach ($b as $key=>$val) {
-            $c[] = $a[$key];
-        }
-        return $c;
-    }
-
     public function checkComposites()
     {
         if (!self::_isAdmin()) {
diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php
index 440408f79..778d60f4d 100644
--- a/app/Controller/Component/ACLComponent.php
+++ b/app/Controller/Component/ACLComponent.php
@@ -54,7 +54,6 @@ class ACLComponent extends Component
                     'returnAttributes' => array('*'),
                     'rpz' => array('*'),
                     'search' => array('*'),
-                    'searchAlternate' => array('*'),
                     'toggleCorrelation' => array('perm_add'),
                     'text' => array('*'),
                     'toggleToIDS' => array('perm_add'),
diff --git a/app/View/Attributes/search.ctp b/app/View/Attributes/search.ctp
index f5c9a29e5..e94b014b0 100755
--- a/app/View/Attributes/search.ctp
+++ b/app/View/Attributes/search.ctp
@@ -28,10 +28,6 @@
                 'type' => 'checkbox',
                 'label' => __('Only find IOCs flagged as to IDS')
             ));
-            echo $this->Form->input('alternate', array(
-                    'type' => 'checkbox',
-                    'label' => __('Alternate Search Result (Events)')
-            ));
             echo $this->Form->input('first_seen', array(
                 'type' => 'text',
                 'div' => 'input hidden',

From 0198c7b0db426254ed32c350c68807f85134a0a8 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 6 Nov 2021 21:21:40 +0100
Subject: [PATCH 40/45] chg: [internal] Make system setting more secure

---
 app/Console/Command/Task/ConfigLoadTask.php |  5 +--
 app/Controller/AppController.php            | 13 +-----
 app/Model/Server.php                        |  6 +--
 app/Model/SystemSetting.php                 | 47 +++++++++++++++++++++
 4 files changed, 53 insertions(+), 18 deletions(-)

diff --git a/app/Console/Command/Task/ConfigLoadTask.php b/app/Console/Command/Task/ConfigLoadTask.php
index df156011d..b624dc573 100644
--- a/app/Console/Command/Task/ConfigLoadTask.php
+++ b/app/Console/Command/Task/ConfigLoadTask.php
@@ -6,9 +6,8 @@ class ConfigLoadTask extends Shell
         Configure::load('config');
 
         if (Configure::read('MISP.system_setting_db')) {
-            $this->loadModel('SystemSetting');
-            $settings = $this->SystemSetting->getSettings();
-            Configure::write($settings);
+            App::uses('SystemSetting', 'Model');
+            SystemSetting::setGlobalSetting();
         }
     }
 }
diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php
index f9d878382..045d65cf7 100755
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@@ -98,7 +98,8 @@ class AppController extends Controller
     public function beforeFilter()
     {
         if (Configure::read('MISP.system_setting_db')) {
-            $this->_setupSystemSettings();
+            App::uses('SystemSetting', 'Model');
+            SystemSetting::setGlobalSetting();
         }
 
         $this->_setupBaseurl();
@@ -1408,14 +1409,4 @@ class AppController extends Controller
             return true;
         }
     }
-
-    /**
-     * Load system settings from database and set as setting in Configure
-     */
-    protected function _setupSystemSettings()
-    {
-        $this->loadModel('SystemSetting');
-        $settings = $this->SystemSetting->getSettings();
-        Configure::write($settings);
-    }
 }
diff --git a/app/Model/Server.php b/app/Model/Server.php
index 8f7100026..06ac9f7b4 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -2182,7 +2182,7 @@ class Server extends AppModel
             return $errorMessage;
         }
         $oldValue = Configure::read($setting['name']);
-        $fileOnly = isset($setting['file_only']) && $setting['file_only'];
+        $fileOnly = isset($setting['cli_only']) && $setting['cli_only'];
         $settingSaveResult = $this->serverSettingsSaveValue($setting['name'], $value, $fileOnly);
         if ($settingSaveResult) {
             if (SystemSetting::isSensitive($setting['name'])) {
@@ -2223,8 +2223,7 @@ class Server extends AppModel
         if (!$fileOnly && Configure::read('MISP.system_setting_db')) {
             /** @var SystemSetting $systemSetting */
             $systemSetting = ClassRegistry::init('SystemSetting');
-            $systemSetting->setSetting($setting, $value);
-            return true;
+            return $systemSetting->setSetting($setting, $value);
         }
 
         $configFilePath = APP . 'Config' . DS . 'config.php';
@@ -6002,7 +6001,6 @@ class Server extends AppModel
                     'null' => true,
                     'cli_only' => true,
                     'redacted' => true,
-                    'file_only' => true,
                 ],
             ),
             'SecureAuth' => array(
diff --git a/app/Model/SystemSetting.php b/app/Model/SystemSetting.php
index 80ed192d2..281c4bfab 100644
--- a/app/Model/SystemSetting.php
+++ b/app/Model/SystemSetting.php
@@ -48,6 +48,47 @@ class SystemSetting extends AppModel
 
     public $primaryKey = 'setting';
 
+    // Blocked setting that cannot be saved or fetched from DB. The same as cli_only settings.
+    const BLOCKED_SETTINGS = [
+        'Security.encryption_key',
+        'Security.disable_local_feed_access',
+        'GnuPG.binary',
+        'MISP.python_bin',
+        'MISP.ca_path',
+        'MISP.tmpdir',
+        'MISP.system_setting_db',
+        'MISP.attachments_dir',
+    ];
+
+    // Allow to set config values just for these categories
+    const ALLOWED_CATEGORIES = [
+        'MISP',
+        'Security',
+        'GnuPG',
+        'SMIME',
+        'Proxy',
+        'SecureAuth',
+        'Session',
+        'Plugin',
+        'debug',
+        'site_admin_debug',
+    ];
+
+    /**
+     * Set config values from database into global Configure class
+     */
+    public static function setGlobalSetting()
+    {
+        $systemSetting = ClassRegistry::init('SystemSetting');
+        $settings = $systemSetting->getSettings();
+        foreach ($settings as $settingName => $settingValue) {
+            $firstPart = explode('.', $settingName)[0];
+            if (in_array($firstPart, self::ALLOWED_CATEGORIES, true) && !in_array($settingName, self::BLOCKED_SETTINGS, true)) {
+                Configure::write($settingName, $settingValue);
+            }
+        }
+    }
+
     /**
      * @return array
      */
@@ -66,6 +107,11 @@ class SystemSetting extends AppModel
      */
     public function setSetting($setting, $value)
     {
+        $firstPart = explode('.', $setting)[0];
+        if (!in_array($firstPart, self::ALLOWED_CATEGORIES, true) || in_array($setting, self::BLOCKED_SETTINGS, true)) {
+            return false; // blocked setting
+        }
+
         $value = JsonTool::encode($value);
 
         // If encryption is enabled and setting name contains `password` or `apikey` string, encrypt value to protect it
@@ -81,6 +127,7 @@ class SystemSetting extends AppModel
         if (!$valid) {
             throw new Exception("Could not save system setting `$setting` because of validation errors: " . JsonTool::encode($this->validationErrors));
         }
+        return true;
     }
 
     /**

From e844f3584771a5a029b7c9086498e7cafe22a0f1 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 6 Nov 2021 22:12:48 +0100
Subject: [PATCH 41/45] chg: [internal] Delete system setting when value is
 empty

---
 app/Model/Behavior/AuditLogBehavior.php | 2 ++
 app/Model/SystemSetting.php             | 7 +++++++
 2 files changed, 9 insertions(+)

diff --git a/app/Model/Behavior/AuditLogBehavior.php b/app/Model/Behavior/AuditLogBehavior.php
index ad9751318..bf3d20f11 100644
--- a/app/Model/Behavior/AuditLogBehavior.php
+++ b/app/Model/Behavior/AuditLogBehavior.php
@@ -254,6 +254,8 @@ class AuditLogBehavior extends ModelBehavior
             }
             $id = $modelName === 'AttributeTag' ? $model->data[$model->alias]['attribute_id'] : $model->data[$model->alias]['event_id'];
             $modelName = $modelName === 'AttributeTag' ? 'Attribute' : 'Event';
+        } else if ($modelName === 'SystemSetting') {
+            $id = 0;
         }
 
         $this->auditLog()->insert(['AuditLog' => [
diff --git a/app/Model/SystemSetting.php b/app/Model/SystemSetting.php
index 281c4bfab..b789f1e59 100644
--- a/app/Model/SystemSetting.php
+++ b/app/Model/SystemSetting.php
@@ -112,6 +112,13 @@ class SystemSetting extends AppModel
             return false; // blocked setting
         }
 
+        if ($value === '' || $value === null) {
+            if ($this->hasAny(['SystemSetting.setting' => $setting])) {
+                return $this->delete($setting); // delete the whole setting when value is empty
+            }
+            return true;
+        }
+
         $value = JsonTool::encode($value);
 
         // If encryption is enabled and setting name contains `password` or `apikey` string, encrypt value to protect it

From 3459a09943bfeedf3e492e23033d2db557b2bdcf Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 7 Nov 2021 10:40:01 +0100
Subject: [PATCH 42/45] chg: [internal] App model cleanup

---
 app/Model/AppModel.php | 196 ++++++++++++++++++-----------------------
 app/Model/Event.php    |   2 -
 app/Model/Feed.php     |   1 -
 app/Model/Server.php   |   2 -
 app/Model/Template.php |   5 --
 5 files changed, 88 insertions(+), 118 deletions(-)

diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php
index 1731683da..243c52c2b 100644
--- a/app/Model/AppModel.php
+++ b/app/Model/AppModel.php
@@ -23,11 +23,11 @@
 App::uses('Model', 'Model');
 App::uses('LogableBehavior', 'Assets.models/behaviors');
 App::uses('RandomTool', 'Tools');
+App::uses('FileAccessTool', 'Tools');
+App::uses('JsonTool', 'Tools');
 
 class AppModel extends Model
 {
-    public $name;
-
     /** @var PubSubTool */
     private static $loadedPubSubTool;
 
@@ -39,7 +39,7 @@ class AppModel extends Model
 
     private $__profiler = array();
 
-    public $elasticSearchClient = false;
+    public $elasticSearchClient;
 
     /** @var AttachmentTool|null */
     private $attachmentTool;
@@ -47,8 +47,6 @@ class AppModel extends Model
     public function __construct($id = false, $table = null, $ds = null)
     {
         parent::__construct($id, $table, $ds);
-
-        $this->name = get_class($this);
         $this->findMethods['column'] = true;
     }
 
@@ -247,13 +245,13 @@ class AppModel extends Model
         $result = $this->Feed->addDefaultFeeds($feeds);
         $this->Log->create();
         $entry = array(
-                'org' => 'SYSTEM',
-                'model' => 'Server',
-                'model_id' => 0,
-                'email' => 'SYSTEM',
-                'action' => 'update_database',
-                'user_id' => 0,
-                'title' => 'Added new default feeds.'
+            'org' => 'SYSTEM',
+            'model' => 'Server',
+            'model_id' => 0,
+            'email' => 'SYSTEM',
+            'action' => 'update_database',
+            'user_id' => 0,
+            'title' => 'Added new default feeds.'
         );
         if ($result) {
             $entry['change'] = 'Feeds added: ' . $feedNames;
@@ -1789,14 +1787,14 @@ class AppModel extends Model
         if ($flagStop && $errorCount > 0) {
             $this->Log->create();
             $this->Log->save(array(
-                    'org' => 'SYSTEM',
-                    'model' => 'Server',
-                    'model_id' => 0,
-                    'email' => 'SYSTEM',
-                    'action' => 'update_database',
-                    'user_id' => 0,
-                    'title' => __('Issues executing the SQL query for %s', $command),
-                    'change' => __('Database updates stopped as some errors occurred and the stop flag is enabled.')
+                'org' => 'SYSTEM',
+                'model' => 'Server',
+                'model_id' => 0,
+                'email' => 'SYSTEM',
+                'action' => 'update_database',
+                'user_id' => 0,
+                'title' => __('Issues executing the SQL query for %s', $command),
+                'change' => __('Database updates stopped as some errors occurred and the stop flag is enabled.')
             ));
             return false;
         }
@@ -1862,14 +1860,14 @@ class AppModel extends Model
             }
             $this->Log->create();
             $this->Log->save(array(
-                    'org' => 'SYSTEM',
-                    'model' => 'Server',
-                    'model_id' => 0,
-                    'email' => 'SYSTEM',
-                    'action' => 'update_database',
-                    'user_id' => 0,
-                    'title' => ($result ? 'Removed index ' : 'Failed to remove index ') . $icr['STATISTICS']['INDEX_NAME'] . ' from ' . $table,
-                    'change' => ($result ? 'Removed index ' : 'Failed to remove index ') . $icr['STATISTICS']['INDEX_NAME'] . ' from ' . $table,
+                'org' => 'SYSTEM',
+                'model' => 'Server',
+                'model_id' => 0,
+                'email' => 'SYSTEM',
+                'action' => 'update_database',
+                'user_id' => 0,
+                'title' => ($result ? 'Removed index ' : 'Failed to remove index ') . $icr['STATISTICS']['INDEX_NAME'] . ' from ' . $table,
+                'change' => ($result ? 'Removed index ' : 'Failed to remove index ') . $icr['STATISTICS']['INDEX_NAME'] . ' from ' . $table,
             ));
         }
     }
@@ -1901,14 +1899,14 @@ class AppModel extends Model
         }
         $this->Log->create();
         $this->Log->save(array(
-                'org' => 'SYSTEM',
-                'model' => 'Server',
-                'model_id' => 0,
-                'email' => 'SYSTEM',
-                'action' => 'update_database',
-                'user_id' => 0,
-                'title' => ($result ? 'Added index ' : 'Failed to add index ') . $field . ' to ' . $table . ($duplicate ? ' (index already set)' : $errorMessage),
-                'change' => ($result ? 'Added index ' : 'Failed to add index ') . $field . ' to ' . $table . ($duplicate ? ' (index already set)' : $errorMessage),
+            'org' => 'SYSTEM',
+            'model' => 'Server',
+            'model_id' => 0,
+            'email' => 'SYSTEM',
+            'action' => 'update_database',
+            'user_id' => 0,
+            'title' => ($result ? 'Added index ' : 'Failed to add index ') . $field . ' to ' . $table . ($duplicate ? ' (index already set)' : $errorMessage),
+            'change' => ($result ? 'Added index ' : 'Failed to add index ') . $field . ' to ' . $table . ($duplicate ? ' (index already set)' : $errorMessage),
         ));
         $additionResult = array('success' => $result || $duplicate);
         if (!$result) {
@@ -1991,7 +1989,8 @@ class AppModel extends Model
     }
 
     // Try to create a table with a BIGINT(20)
-    public function seenOnAttributeAndObjectPreUpdate() {
+    public function seenOnAttributeAndObjectPreUpdate()
+    {
         $sqlArray[] = "CREATE TABLE IF NOT EXISTS testtable (
             `testfield` BIGINT(6) NULL DEFAULT NULL
         ) ENGINE=InnoDB DEFAULT CHARSET=utf8;";
@@ -2009,10 +2008,6 @@ class AppModel extends Model
         }
     }
 
-    public function failingPreUpdate() {
-        throw new Exception('Yolo fail');
-    }
-
     public function runUpdates($verbose = false, $useWorker = true, $processId = false)
     {
         $this->AdminSetting = ClassRegistry::init('AdminSetting');
@@ -2059,14 +2054,14 @@ class AppModel extends Model
                 if ($this->isUpdateLocked()) { // prevent creation of useless workers
                     $this->Log->create();
                     $this->Log->save(array(
-                            'org' => 'SYSTEM',
-                            'model' => 'Server',
-                            'model_id' => 0,
-                            'email' => 'SYSTEM',
-                            'action' => 'update_db_worker',
-                            'user_id' => 0,
-                            'title' => __('Issues executing run_updates'),
-                            'change' => __('Database updates are locked. Worker not spawned')
+                        'org' => 'SYSTEM',
+                        'model' => 'Server',
+                        'model_id' => 0,
+                        'email' => 'SYSTEM',
+                        'action' => 'update_db_worker',
+                        'user_id' => 0,
+                        'title' => __('Issues executing run_updates'),
+                        'change' => __('Database updates are locked. Worker not spawned')
                     ));
                     if (!empty($job)) { // if multiple prio worker is enabled, want to mark them as done
                         $job['Job']['progress'] = 100;
@@ -2114,14 +2109,14 @@ class AppModel extends Model
                 if ($this->isUpdateLocked()) {
                     $this->Log->create();
                     $this->Log->save(array(
-                            'org' => 'SYSTEM',
-                            'model' => 'Server',
-                            'model_id' => 0,
-                            'email' => 'SYSTEM',
-                            'action' => 'update_db_worker',
-                            'user_id' => 0,
-                            'title' => __('Issues executing run_updates'),
-                            'change' => __('Updates are locked. Stopping worker gracefully')
+                        'org' => 'SYSTEM',
+                        'model' => 'Server',
+                        'model_id' => 0,
+                        'email' => 'SYSTEM',
+                        'action' => 'update_db_worker',
+                        'user_id' => 0,
+                        'title' => __('Issues executing run_updates'),
+                        'change' => __('Updates are locked. Stopping worker gracefully')
                     ));
                     if (!empty($job)) {
                         $job['Job']['progress'] = 100;
@@ -2379,6 +2374,10 @@ class AppModel extends Model
         }
     }
 
+    /**
+     * @param string $db_version
+     * @return array
+     */
     protected function findUpgrades($db_version)
     {
         $updates = array();
@@ -2421,21 +2420,21 @@ class AppModel extends Model
             $Job = ClassRegistry::init('Job');
             $Job->create();
             $data = array(
-                    'worker' => 'default',
-                    'job_type' => 'generate correlation',
-                    'job_input' => 'All attributes',
-                    'status' => 0,
-                    'retries' => 0,
-                    'org' => 'ADMIN',
-                    'message' => 'Job created.',
+                'worker' => 'default',
+                'job_type' => 'generate correlation',
+                'job_input' => 'All attributes',
+                'status' => 0,
+                'retries' => 0,
+                'org' => 'ADMIN',
+                'message' => 'Job created.',
             );
             $Job->save($data);
             $jobId = $Job->id;
             $process_id = CakeResque::enqueue(
-                    'default',
-                    'AdminShell',
-                    array('jobGenerateCorrelation', $jobId),
-                    true
+                'default',
+                'AdminShell',
+                array('jobGenerateCorrelation', $jobId),
+                true
             );
             $Job->saveField('process_id', $process_id);
         }
@@ -2502,26 +2501,20 @@ class AppModel extends Model
     public function getKafkaPubTool()
     {
         if (!$this->loadedKafkaPubTool) {
-            $this->loadKafkaPubTool();
+            App::uses('KafkaPubTool', 'Tools');
+            $kafkaPubTool = new KafkaPubTool();
+            $rdkafkaIni = Configure::read('Plugin.Kafka_rdkafka_config');
+            $kafkaConf = array();
+            if (!empty($rdkafkaIni)) {
+                $kafkaConf = parse_ini_file($rdkafkaIni);
+            }
+            $brokers = Configure::read('Plugin.Kafka_brokers');
+            $kafkaPubTool->initTool($brokers, $kafkaConf);
+            $this->loadedKafkaPubTool = $kafkaPubTool;
         }
         return $this->loadedKafkaPubTool;
     }
 
-    public function loadKafkaPubTool()
-    {
-        App::uses('KafkaPubTool', 'Tools');
-        $kafkaPubTool = new KafkaPubTool();
-        $rdkafkaIni = Configure::read('Plugin.Kafka_rdkafka_config');
-        $kafkaConf = array();
-        if (!empty($rdkafkaIni)) {
-            $kafkaConf = parse_ini_file($rdkafkaIni);
-        }
-        $brokers = Configure::read('Plugin.Kafka_brokers');
-        $kafkaPubTool->initTool($brokers, $kafkaConf);
-        $this->loadedKafkaPubTool = $kafkaPubTool;
-        return true;
-    }
-
     public function publishKafkaNotification($topicName, $data, $action = false)
     {
         $kafkaTopic = $this->kafkaTopic($topicName);
@@ -2649,15 +2642,6 @@ class AppModel extends Model
         }
     }
 
-    public function getRowCount($table = false)
-    {
-        if (empty($table)) {
-            $table = $this->table;
-        }
-        $table_data = $this->query("show table status like '" . $table . "'");
-        return $table_data[0]['TABLES']['Rows'];
-    }
-
     public function benchmarkCustomAdd($valueToAdd = 0, $name = 'default', $customName = 'custom')
     {
         if (empty($this->__profiler[$name]['custom'][$customName])) {
@@ -2720,9 +2704,8 @@ class AppModel extends Model
     {
         static $versionArray;
         if ($versionArray === null) {
-            $file = new File(ROOT . DS . 'VERSION.json');
-            $versionArray = $this->jsonDecode($file->read());
-            $file->close();
+            $content = FileAccessTool::readFromFile(ROOT . DS . 'VERSION.json');
+            $versionArray = JsonTool::decode($content);
         }
         return $versionArray;
     }
@@ -2923,14 +2906,14 @@ class AppModel extends Model
             $this->Log = ClassRegistry::init('Log');
             $this->Log->create();
             $entry = array(
-                    'org' => 'SYSTEM',
-                    'model' => 'Server',
-                    'model_id' => 0,
-                    'email' => 'SYSTEM',
-                    'action' => 'update_database',
-                    'user_id' => 0,
-                    'title' => 'Bumped the timestamps of locked events containing object references.',
-                    'change' => sprintf('Event timestamps updated: %s; Object timestamps updated: %s', count($event_ids), count($object_ids))
+                'org' => 'SYSTEM',
+                'model' => 'Server',
+                'model_id' => 0,
+                'email' => 'SYSTEM',
+                'action' => 'update_database',
+                'user_id' => 0,
+                'title' => 'Bumped the timestamps of locked events containing object references.',
+                'change' => sprintf('Event timestamps updated: %s; Object timestamps updated: %s', count($event_ids), count($object_ids))
             );
             $this->Log->save($entry);
         }
@@ -3088,10 +3071,7 @@ class AppModel extends Model
         $message .= "\n";
 
         do {
-            $message .= sprintf("[%s] %s",
-                get_class($exception),
-                $exception->getMessage()
-            );
+            $message .= sprintf("[%s] %s", get_class($exception), $exception->getMessage());
             $message .= "\nStack Trace:\n" . $exception->getTraceAsString();
             $exception = $exception->getPrevious();
         } while ($exception !== null);
diff --git a/app/Model/Event.php b/app/Model/Event.php
index cd43ba679..9d6a371dc 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -1,9 +1,7 @@
 <?php
 App::uses('AppModel', 'Model');
 App::uses('CakeEmail', 'Network/Email');
-App::uses('FileAccessTool', 'Tools');
 App::uses('AttachmentTool', 'Tools');
-App::uses('JsonTool', 'Tools');
 App::uses('TmpFileTool', 'Tools');
 App::uses('SendEmailTemplate', 'Tools');
 
diff --git a/app/Model/Feed.php b/app/Model/Feed.php
index c2e0a0555..19d959815 100644
--- a/app/Model/Feed.php
+++ b/app/Model/Feed.php
@@ -2,7 +2,6 @@
 App::uses('AppModel', 'Model');
 App::uses('RandomTool', 'Tools');
 App::uses('TmpFileTool', 'Tools');
-App::uses('FileAccessTool', 'Tools');
 App::uses('AttributeValidationTool', 'Tools');
 
 class Feed extends AppModel
diff --git a/app/Model/Server.php b/app/Model/Server.php
index a9b12985c..ddbc6248a 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -2,8 +2,6 @@
 App::uses('AppModel', 'Model');
 App::uses('GpgTool', 'Tools');
 App::uses('ServerSyncTool', 'Tools');
-App::uses('FileAccessTool', 'Tools');
-App::uses('JsonTool', 'Tools');
 App::uses('SystemSetting', 'Model');
 
 /**
diff --git a/app/Model/Template.php b/app/Model/Template.php
index 4bc516960..44c92c70d 100644
--- a/app/Model/Template.php
+++ b/app/Model/Template.php
@@ -72,9 +72,4 @@ class Template extends AppModel
             return false;
         }
     }
-
-    public function generateRandomFileName()
-    {
-        return (new RandomTool())->random_str(false, 12);
-    }
 }

From af234a006c82dfaa3a5241b7e492f2d4830fdf8f Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 7 Nov 2021 14:36:16 +0100
Subject: [PATCH 43/45] chg: [internal] AppController cleanup

---
 app/Controller/AppController.php              |  6 +--
 .../IndexTable/Fields/target_event.ctp        | 50 +++++++++----------
 app/View/Helper/DataPathCollectorHelper.php   | 32 ------------
 3 files changed, 28 insertions(+), 60 deletions(-)
 delete mode 100644 app/View/Helper/DataPathCollectorHelper.php

diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php
index 045d65cf7..d21298b21 100755
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@@ -31,7 +31,7 @@ class AppController extends Controller
      */
     public $defaultModel = '';
 
-    public $helpers = array('OrgImg', 'FontAwesome', 'UserName', 'DataPathCollector');
+    public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
 
     private $__queryVersion = '131';
     public $pyMispVersion = '2.4.148';
@@ -347,7 +347,7 @@ class AppController extends Controller
                     $this->User->Server->updateDatabase('cleanSessionTable');
                 }
             }
-            if (Configure::read('site_admin_debug') && (Configure::read('debug') < 2)) {
+            if (Configure::read('site_admin_debug') && Configure::read('debug') < 2) {
                 Configure::write('debug', 1);
             }
         }
@@ -378,7 +378,7 @@ class AppController extends Controller
             if (!empty($homepage)) {
                 $this->set('homepage', $homepage);
             }
-            if (version_compare(phpversion(), '8.0') >= 0) {
+            if (PHP_MAJOR_VERSION >= 8) {
                 $this->Flash->error(__('WARNING: MISP is currently running under PHP 8.0, which is unsupported. Background jobs will fail, so please contact your administrator to run a supported PHP version (such as 7.4)'));
             }
         }
diff --git a/app/View/Elements/genericElements/IndexTable/Fields/target_event.ctp b/app/View/Elements/genericElements/IndexTable/Fields/target_event.ctp
index 0422c8813..fe69b77a4 100644
--- a/app/View/Elements/genericElements/IndexTable/Fields/target_event.ctp
+++ b/app/View/Elements/genericElements/IndexTable/Fields/target_event.ctp
@@ -1,33 +1,33 @@
 <?php
-    $data = $this->DataPathCollector->extract($row, $field['data_path']);
-    if ($data['Feed.enabled']) {
-        if (in_array($data['Feed.source_format'], array('freetext', 'csv'))) {
-            if ($data['Feed.fixed_event']) {
-                if (!empty($data['Feed.event_error'])) {
+$feed = $row['Feed'];
+if ($feed['enabled']) {
+    if (in_array($feed['source_format'], array('freetext', 'csv'))) {
+        if ($feed['fixed_event']) {
+            if (!empty($feed['event_error'])) {
+                echo sprintf(
+                    '<span class="red bold">%s</span>',
+                    __('Error: Invalid event!')
+                );
+            } else {
+                if ($feed['event_id']) {
                     echo sprintf(
-                        '<span class="red bold">%s</span>',
-                        __('Error: Invalid event!')
+                        '<a href="%s/events/view/%s">%s</a>',
+                        $baseurl,
+                        h($feed['event_id']),
+                        __('Fixed event %s', h($feed['event_id']))
                     );
                 } else {
-                    if ($data['Feed.event_id']) {
-                        echo sprintf(
-                            '<a href="%s/events/view/%s">%s</a>',
-                            $baseurl,
-                            h($data['Feed.event_id']),
-                            __('Fixed event %s', h($data['Feed.event_id']))
-                        );
-                    } else {
-                        echo __('New fixed event');
-                    }
+                    echo __('New fixed event');
                 }
-            } else {
-                echo sprintf(
-                    '<span class="bold red" title="%s">%s</span>',
-                    __('New event each pull can lead to potentially endlessly growing correlation tables. Only use this setting if you are sure that the data in the feed will mostly be completely distinct between each individual pull, otherwise use fixed events. Generally this setting is NOT recommended.'),
-                    __('New event each pull')
-                );
             }
+        } else {
+            echo sprintf(
+                '<span class="bold red" title="%s">%s</span>',
+                __('New event each pull can lead to potentially endlessly growing correlation tables. Only use this setting if you are sure that the data in the feed will mostly be completely distinct between each individual pull, otherwise use fixed events. Generally this setting is NOT recommended.'),
+                __('New event each pull')
+            );
         }
-    } else {
-        echo __('Feed not enabled');
     }
+} else {
+    echo __('Feed not enabled');
+}
diff --git a/app/View/Helper/DataPathCollectorHelper.php b/app/View/Helper/DataPathCollectorHelper.php
deleted file mode 100644
index 8b6305a7b..000000000
--- a/app/View/Helper/DataPathCollectorHelper.php
+++ /dev/null
@@ -1,32 +0,0 @@
-<?php
-    /*
-     * Helper used to extract variables from an array based on path
-     * Used by the index factories
-     *
-     */
-    App::uses('AppHelper', 'View/Helper');
-        class DataPathCollectorHelper extends AppHelper {
-            public function extract($data, $data_path, $options = array())
-            {
-                $result = array();
-                if (!is_array($data_path)) {
-                    $data_path = array($data_path);
-                }
-                foreach ($data_path as $path) {
-                    $temp = Hash::extract($data, $path);
-                    if (is_array($temp)) {
-                        if (count($temp) > 1) {
-                            $temp = implode(', ', $temp);
-                        } else {
-                            if (count($temp) > 0) {
-                                $temp = $temp[0];
-                            } else {
-                                $temp = '';
-                            }
-                        }
-                    }
-                    $result[$path] = $temp;
-                }
-                return $result;
-            }
-        }

From c8738c2be4fe6cf5feb60cf975faafd6cca22639 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 7 Nov 2021 14:51:26 +0100
Subject: [PATCH 44/45] fix: [internal] Remove unused helper

---
 app/View/Helper/XmlOutputHelper.php | 24 ------------------------
 1 file changed, 24 deletions(-)
 delete mode 100644 app/View/Helper/XmlOutputHelper.php

diff --git a/app/View/Helper/XmlOutputHelper.php b/app/View/Helper/XmlOutputHelper.php
deleted file mode 100644
index 2f80d237a..000000000
--- a/app/View/Helper/XmlOutputHelper.php
+++ /dev/null
@@ -1,24 +0,0 @@
-<?php
-App::uses('AppHelper', 'View/Helper');
-    class XmlOutputHelper extends AppHelper {
-        public function recursiveEcho($array) {
-            foreach ($array as $k => $v) {
-                if (is_array($v)) {
-                    if (empty($v)) echo '<' . $k . '/>';
-                    else {
-                        foreach ($v as $element) {
-                            echo '<' . $k . '>';
-                            $this->recursiveEcho($element);
-                            echo '</' . $k . '>';
-                        }
-                    }
-                } else {
-                    if ($v === false) $v = 0;
-                    if ($v === "" || $v === null) echo '<' . $k . '/>';
-                    else {
-                        echo '<' . $k . '>' . $v . '</' . $k . '>';
-                    }
-                }
-            }
-        }
-    }

From df5ab6a14466692886c75b22be986a43f0721c78 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 7 Nov 2021 14:51:42 +0100
Subject: [PATCH 45/45] chg: [internal] Code style

---
 app/Controller/AppController.php | 44 ++++++++++++++++----------------
 1 file changed, 22 insertions(+), 22 deletions(-)

diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php
index d21298b21..c9c454a1a 100755
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@@ -69,30 +69,30 @@ class AppController extends Controller
     }
 
     public $components = array(
-            'Session',
-            'Auth' => array(
-                'authError' => 'Unauthorised access.',
-                'authenticate' => array(
-                    'Form' => array(
-                        'passwordHasher' => 'BlowfishConstant',
-                        'fields' => array(
-                            'username' => 'email'
-                        )
+        'Session',
+        'Auth' => array(
+            'authError' => 'Unauthorised access.',
+            'authenticate' => array(
+                'Form' => array(
+                    'passwordHasher' => 'BlowfishConstant',
+                    'fields' => array(
+                        'username' => 'email'
                     )
                 )
-            ),
-            'Security',
-            'ACL',
-            'CompressedRequestHandler',
-            'RestResponse',
-            'Flash',
-            'Toolbox',
-            'RateLimit',
-            'IndexFilter',
-            'Deprecation',
-            'RestSearch',
-            'CRUD'
-            //,'DebugKit.Toolbar'
+            )
+        ),
+        'Security',
+        'ACL',
+        'CompressedRequestHandler',
+        'RestResponse',
+        'Flash',
+        'Toolbox',
+        'RateLimit',
+        'IndexFilter',
+        'Deprecation',
+        'RestSearch',
+        'CRUD'
+        //,'DebugKit.Toolbar'
     );
 
     public function beforeFilter()