Merge branch '2.4' of github.com:MISP/MISP into 2.4

- helps administrators to easily see what went wrong in terms of timeouts / oom issues

.github/workflows/main.yml

@@ -6,9 +6,9 @@ name: misp
 # events but only for the 2.4 and develop branches
 on:
   push:
-    branches: [ 2.4, develop, misp-stix, taxii ]
+    branches: [ '2.4', develop, misp-stix, taxii ]
   pull_request:
-    branches: [ 2.4, develop, misp-stix ]
+    branches: [ '2.4', develop, misp-stix ]
 
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
@@ -62,186 +62,166 @@ jobs:
           php_version: ${{ matrix.php }}
         run: |
           sudo apt-get -y update
-          # Repo is missing for unknown reason
-          LC_ALL=C.UTF-8 sudo apt-add-repository ppa:ondrej/php -y
-          if [[ $php_version == "7.2" ]]; then
-            # hotfix due to: https://bugs.php.net/bug.php?id=81640 TODO: remove after libpcre2-8-0:10.36 gets to stable channel
-            sudo apt-get --fix-broken install
-          fi
           sudo apt-get -y install curl python3 python3-pip python3-virtualenv apache2 libapache2-mod-php$php_version
 
       # Runs a set of commands using the runners shell
       - name: Install deps
         run: |
-              sudo chown $USER:www-data $HOME/.composer
-              pushd app
-              sudo -H -u $USER composer config --no-plugins allow-plugins.composer/installers true
-              sudo -H -u $USER composer install --no-progress
-              popd
-              cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php
-              # Set perms
-              sudo chown -R $USER:www-data `pwd`
-              sudo chmod -R 775 `pwd`
-              sudo chmod -R g+ws `pwd`/app/tmp
-              sudo chmod -R g+ws `pwd`/app/tmp/cache
-              sudo chmod -R g+ws `pwd`/app/tmp/cache/persistent
-              sudo chmod -R g+ws `pwd`/app/tmp/cache/models
-              sudo chmod -R g+ws `pwd`/app/tmp/logs
-              sudo chmod -R g+ws `pwd`/app/files
-              sudo chmod -R g+ws `pwd`/app/files/scripts/tmp
-              sudo chown -R $USER:www-data `pwd`
-              # Resque perms
-              sudo chown -R $USER:www-data `pwd`/app/Plugin/CakeResque/tmp
-              sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp
-              # install MySQL
-              sudo chmod -R 777 `pwd`/INSTALL
-              mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';"
-              mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant usage on *.* to misp@'%' identified by 'blah';"
-              mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant all privileges on misp.* to misp@'%';"
-              mysql -h 127.0.0.1 --port 3306 -u misp -pblah misp < INSTALL/MYSQL.sql
-              # configure apache virtual hosts
-              sudo chmod -R 777 `pwd`/build
-              sudo mkdir -p /etc/apache2/sites-available
-              sudo cp -f build/github-action-ci-apache /etc/apache2/sites-available/misp.conf
-              sudo sed -e "s?%GITHUB_WORKSPACE%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.conf
-              sudo sed -e "s?%HOST%?${HOST}?g" --in-place /etc/apache2/sites-available/misp.conf
-              sudo a2dissite 000-default
-              sudo a2ensite misp.conf
-              cat /etc/apache2/sites-enabled/misp.conf
-              sudo a2enmod rewrite
-              sudo systemctl restart apache2
-              # MISP configuration
-              sudo chmod -R 777 `pwd`/travis
-              sudo cp app/Config/bootstrap.default.php app/Config/bootstrap.php
-              sudo cp travis/database.php app/Config/database.php
-              sudo cp app/Config/core.default.php app/Config/core.php
-              sudo cp app/Config/config.default.php app/Config/config.php
-              sudo cp travis/email.php app/Config/email.php
-              # Ensure the perms
-              sudo chown -R $USER:www-data `pwd`/app/Config
-              sudo chmod -R 777 `pwd`/app/Config
-              # GPG setup
-              sudo mkdir `pwd`/.gnupg
-              # /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS
-              sudo cp -a /dev/urandom /dev/random
-              sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/travis/gpg
-              sudo gpg --list-secret-keys --homedir `pwd`/.gnupg
-              # change perms
-              sudo chown -R $USER:www-data `pwd`
-              sudo chown -R www-data:www-data `pwd`/.gnupg
-              sudo chmod -R 700 `pwd`/.gnupg
-              sudo usermod -a -G www-data $USER
-              sudo chmod -R 777 `pwd`/app/Plugin/CakeResque/tmp/
-              # Ensure the perms of config files
-              sudo chown -R $USER:www-data `pwd`/app/Config
-              sudo chmod -R 777 `pwd`/app/Config
-              sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.server_settings_skip_backup_rotate" 1'
-              sudo chown -R $USER:www-data `pwd`/app/Config
-              sudo chmod -R 777 `pwd`/app/Config
+          sudo chown $USER:www-data $HOME/.composer
+          pushd app
+          composer config --no-plugins allow-plugins.composer/installers true
+          composer install --no-progress
+          popd
+          cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php
 
-              # fix perms (?)
-              namei -m  /home/runner/work
-              sudo chmod +x /home/runner/work
-              sudo chmod +x /home/runner
-              sudo chmod +x /home
-              sudo chmod +x /
+          # Set perms
+          sudo chown -R $USER:www-data `pwd`
+          sudo chmod -R 775 `pwd`
+          sudo chmod -R g+ws `pwd`/app/tmp
+          sudo chmod -R g+ws `pwd`/app/tmp/cache
+          sudo chmod -R g+ws `pwd`/app/tmp/cache/persistent
+          sudo chmod -R g+ws `pwd`/app/tmp/cache/models
+          sudo chmod -R g+ws `pwd`/app/tmp/logs
+          sudo chmod -R g+ws `pwd`/app/files
+          sudo chmod -R g+ws `pwd`/app/files/scripts/tmp
+          sudo chown -R $USER:www-data `pwd`
 
-      - name: DB Update
-        run: |
-              sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.osuser" $USER'
-              sudo -E su $USER -c 'app/Console/cake Admin runUpdates'
-              sudo -E su $USER -c 'app/Console/cake Admin schemaDiagnostics'
+          # Resque perms
+          sudo chown -R $USER:www-data `pwd`/app/Plugin/CakeResque/tmp
+          sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp
 
-      - name: Configure MISP
-        run: |
-              sudo -u $USER app/Console/cake userInit -q | sudo tee ./key.txt
-              echo "AUTH=`cat key.txt`" >> $GITHUB_ENV
-              sudo -u $USER app/Console/cake Admin setSetting "Session.autoRegenerate" 0
-              sudo -u $USER app/Console/cake Admin setSetting "Session.timeout" 600
-              sudo -u $USER app/Console/cake Admin setSetting "Session.cookieTimeout" 3600
-              sudo -u $USER app/Console/cake Admin setSetting "MISP.host_org_id" 1
-              sudo -u $USER app/Console/cake Admin setSetting "MISP.email" "info@admin.test"
-              sudo -u $USER app/Console/cake Admin setSetting "MISP.disable_emailing" false
-              sudo -u $USER app/Console/cake Admin setSetting --force "debug" true
-              sudo -u $USER app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false
-              sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1"
-              sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_port" 6379
-              sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_database" 13
-              sudo -u $USER app/Console/cake Admin setSetting "MISP.redis_password" ""
-              sudo -u $USER app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test"
-              sudo -u $USER app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg"
-              sudo -u $USER app/Console/cake Admin setSetting "GnuPG.password" "travistest"
-              sudo -u $USER app/Console/cake Admin setSetting "MISP.download_gpg_from_homedir" 1
+          # Fill database with basic MISP schema
+          mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';"
+          mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant usage on *.* to misp@'%' identified by 'blah';"
+          mysql -h 127.0.0.1 --port 3306 -u root -pbar -e "grant all privileges on misp.* to misp@'%';"
+          mysql -h 127.0.0.1 --port 3306 -u misp -pblah misp < INSTALL/MYSQL.sql
 
-      - name: Configure ZMQ
-        run: |
-              sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_host" "127.0.0.1"
-              sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_port" 6379
-              sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_database" 1
-              sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_password" ""
-              sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" 1
-              sudo -u $USER app/Console/cake Admin setSetting "Plugin.ZeroMQ_audit_notifications_enable" 1
+          # configure apache virtual hosts
+          sudo mkdir -p /etc/apache2/sites-available
+          sudo cp -f build/github-action-ci-apache /etc/apache2/sites-available/misp.conf
+          sudo sed -e "s?%GITHUB_WORKSPACE%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.conf
+          sudo sed -e "s?%HOST%?${HOST}?g" --in-place /etc/apache2/sites-available/misp.conf
+          sudo a2dissite 000-default
+          sudo a2ensite misp.conf
+          cat /etc/apache2/sites-enabled/misp.conf
+          sudo a2enmod rewrite
+          sudo systemctl start --no-block apache2
 
-      - name: Update Galaxies
-        run: sudo -E su $USER -c 'app/Console/cake Admin updateGalaxies'
+          # MISP configuration
+          sudo cp app/Config/bootstrap.default.php app/Config/bootstrap.php
+          sudo cp build/database.php app/Config/database.php
+          sudo cp app/Config/core.default.php app/Config/core.php
+          sudo cp app/Config/config.default.php app/Config/config.php
+          sudo cp build/email.php app/Config/email.php
 
-      - name: Update Taxonomies
-        run: sudo -E su $USER -c 'app/Console/cake Admin updateTaxonomies'
+          # GPG setup
+          sudo mkdir `pwd`/.gnupg
+          # /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS
+          sudo cp -a /dev/urandom /dev/random
+          sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/build/gpg
+          sudo gpg --list-secret-keys --homedir `pwd`/.gnupg
 
-      - name: Update Warninglists
-        run: sudo -E su $USER -c 'app/Console/cake Admin updateWarningLists --verbose'
+          # change perms
+          sudo chown -R $USER:www-data `pwd`
+          sudo chown -R www-data:www-data `pwd`/.gnupg
+          sudo chmod -R 700 `pwd`/.gnupg
+          sudo usermod -a -G www-data $USER
+          sudo chmod -R 777 `pwd`/app/Plugin/CakeResque/tmp/
+          # Ensure the perms of config files
+          sudo chown -R $USER:www-data `pwd`/app/Config
+          sudo chmod -R 777 `pwd`/app/Config
+          app/Console/cake Admin setSetting "MISP.server_settings_skip_backup_rotate" 1
+          sudo chown -R $USER:www-data `pwd`/app/Config
+          sudo chmod -R 777 `pwd`/app/Config
 
-      - name: Update Noticelists
-        run: sudo -E su $USER -c 'app/Console/cake Admin updateNoticeLists'
-
-      - name: Update Object Templates
-        run: sudo -E su $USER -c 'app/Console/cake Admin updateObjectTemplates 1'
-
-      - name: Turn MISP live
-        run: sudo -E su $USER -c 'app/Console/cake Live 1'
-
-      - name: Check if Redis is ready
-        run: sudo -E su $USER -c 'app/Console/cake Admin redisReady'
-
-      - name: Start workers
-        run: |
-              sudo chmod +x app/Console/worker/start.sh
-              sudo -u www-data 'app/Console/worker/start.sh'
+          # fix perms (?)
+          namei -m /home/runner/work
+          sudo chmod +x /home/runner/work
+          sudo chmod +x /home/runner
+          sudo chmod +x /home
+          sudo chmod +x /
 
       - name: Python setup
         run: |
-              sudo chmod 777 ./key.txt
-              sudo chmod -R 777 ./tests
-              # Start workers
-              # Dirty install python stuff
-              python3 -m virtualenv -p python3 ./venv
-              sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$GITHUB_WORKSPACE/venv/bin/python"'
-              . ./venv/bin/activate
-              export PYTHONPATH=$PYTHONPATH:./app/files/scripts
-              pip install ./PyMISP[fileobjects,email] ./app/files/scripts/python-stix ./app/files/scripts/cti-python-stix2 pyzmq redis plyara pytest
-              deactivate
+          # Dirty install python stuff
+          python3 -m virtualenv -p python3 ./venv
+          app/Console/cake Admin setSetting "MISP.python_bin" "$GITHUB_WORKSPACE/venv/bin/python"
+          . ./venv/bin/activate
+          export PYTHONPATH=$PYTHONPATH:./app/files/scripts
+          pip install ./PyMISP[fileobjects,email] ./app/files/scripts/python-stix ./app/files/scripts/cti-python-stix2 pyzmq redis plyara pytest
+          deactivate
+
+      - name: DB Update
+        run: |
+          app/Console/cake Admin setSetting "MISP.osuser" $USER
+          app/Console/cake Admin runUpdates
+          app/Console/cake Admin schemaDiagnostics
+
+      - name: Configure MISP
+        run: |
+          app/Console/cake User init | sudo tee ./key.txt
+          echo "AUTH=`cat key.txt`" >> $GITHUB_ENV
+          app/Console/cake Admin setSetting "Session.autoRegenerate" 0
+          app/Console/cake Admin setSetting "Session.timeout" 600
+          app/Console/cake Admin setSetting "Session.cookieTimeout" 3600
+          app/Console/cake Admin setSetting "MISP.host_org_id" 1
+          app/Console/cake Admin setSetting "MISP.email" "info@admin.test"
+          app/Console/cake Admin setSetting "MISP.disable_emailing" false
+          app/Console/cake Admin setSetting --force "debug" true
+          app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false
+          app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1"
+          app/Console/cake Admin setSetting "MISP.redis_port" 6379
+          app/Console/cake Admin setSetting "MISP.redis_database" 13
+          app/Console/cake Admin setSetting "MISP.redis_password" ""
+          app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test"
+          app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg"
+          app/Console/cake Admin setSetting "GnuPG.password" "travistest"
+          app/Console/cake Admin setSetting "MISP.download_gpg_from_homedir" 1
+          app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_host" "127.0.0.1"
+          app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_port" 6379
+          app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_database" 1
+          app/Console/cake Admin setSetting "Plugin.ZeroMQ_redis_password" ""
+          app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" 1
+          app/Console/cake Admin setSetting "Plugin.ZeroMQ_audit_notifications_enable" 1
+
+      - name: Update JSON
+        run: app/Console/cake Admin updateJSON
+
+      - name: Turn MISP live
+        run: app/Console/cake Admin live 1
+
+      - name: Check if Redis is ready
+        run: app/Console/cake Admin redisReady
+
+      - name: Start workers
+        run: |
+          sudo chmod +x app/Console/worker/start.sh
+          sudo -u www-data 'app/Console/worker/start.sh'
 
       - name: Test if apache is working
         run: |
-              sudo systemctl status apache2 --no-pager -l
-              sudo apache2ctl -S
-              curl http://${HOST}
-              sudo chmod -R 777 PyMISP
-              pushd PyMISP
-              echo 'url = "http://'${HOST}'"' >> tests/keys.py
-              echo 'key = "'${AUTH}'"' >> tests/keys.py
-              cat tests/keys.py
-              popd
-              . ./venv/bin/activate
-              pushd tests
-              bash ./build-test.sh
-              popd
-              deactivate
+          sudo systemctl status apache2 --no-pager -l
+          sudo apache2ctl -S
+          curl -sS http://${HOST}
+
+      - name: Check if dependencies working as expected
+        run: |
+          sudo chmod -R 777 PyMISP
+          pushd PyMISP
+          echo 'url = "http://'${HOST}'"' >> tests/keys.py
+          echo 'key = "'${AUTH}'"' >> tests/keys.py
+          cat tests/keys.py
+          popd
+          . ./venv/bin/activate
+          pushd tests
+          bash ./build-test.sh
+          popd
+          deactivate
 
       - name: Run PHP tests
         run: |
-              ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/
-              sudo -u www-data ./app/Vendor/bin/phpunit app/Test/
+          ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ -e php,ctp app/
+          sudo -u www-data ./app/Vendor/bin/phpunit app/Test/
 
       - name: Clone test files
         uses: actions/checkout@v4
@@ -249,45 +229,48 @@ jobs:
           repository: viper-framework/viper-test-files
           path: PyMISP/tests/viper-test-files
 
-
       - name: Run tests
         run: |
-              pushd tests
-              ./curl_tests_GH.sh $AUTH $HOST
-              popd
+          pushd tests
+          ./curl_tests_GH.sh $AUTH $HOST
+          popd
 
-              sudo chmod -R g+ws `pwd`/app/tmp/logs
+          sudo chmod -R g+ws `pwd`/app/tmp/logs
 
-              . ./venv/bin/activate
-              pushd PyMISP
-              cp tests/keys.py .
-              python -m pytest -v --durations=0 tests/test_mispevent.py
-              python -m pytest -v --durations=0 tests/testlive_comprehensive.py
-              popd
-              python tests/testlive_security.py -v
-              python tests/testlive_sync.py
-              python tests/testlive_comprehensive_local.py -v
-              cp PyMISP/tests/keys.py PyMISP/examples/events/
-              pushd PyMISP/examples/events/
-              python ./create_massive_dummy_events.py -l 5 -a 30
-              popd
-              python tools/misp-feed/validate.py
-              deactivate
+          . ./venv/bin/activate
+          pushd PyMISP
+          cp tests/keys.py .
+          python -m pytest -v --durations=0 tests/test_mispevent.py
+          python -m pytest -v --durations=0 tests/testlive_comprehensive.py
+          popd
+          python tests/testlive_comprehensive_local.py -v
+          python tests/testlive_sync.py -v
+          python tests/testlive_security.py -v
+          cp PyMISP/tests/keys.py PyMISP/examples/events/
+          pushd PyMISP/examples/events/
+          python ./create_massive_dummy_events.py -l 5 -a 30
+          popd
+          pip install jsonschema
+          python tools/misp-feed/validate.py
+          deactivate
 
       - name: Check requirements.txt
         run: python tests/check_requirements.py
 
-      - name: Logs
+      - name: System logs
         if: ${{ always() }}
         # update logs_test.sh when adding more logsources here
         run: |
-              tail -n +1 `pwd`/app/tmp/logs/*
-              tail -n +1 /var/log/apache2/*.log
+          tail -n +1 `pwd`/app/tmp/logs/*
+          tail -n +1 /var/log/apache2/*.log
 
-              sudo -u $USER app/Console/cake Log export /tmp/logs.json.gz --without-changes
-              zcat /tmp/logs.json.gz
+      - name: Application logs
+        if: ${{ always() }}
+        run: |
+          app/Console/cake Log export /tmp/logs.json.gz --without-changes
+          zcat /tmp/logs.json.gz
 
       - name: Errors in Logs
         if: ${{ always() }}
         run: |
-              ./tests/logs_tests.sh
+          ./tests/logs_tests.sh

.gitignore

@@ -24,7 +24,6 @@ tools/mkdocs
 /.idea
 .DS_Store
 /.htaccess
-/app/Vendor
 /README
 /app/tmp/GPG*
 /app/tmp/sessions/sess_*
@@ -40,6 +39,7 @@ tools/mkdocs
 /app/tmp/cache/feeds/*.etag
 app/Lib/EventWarning/Custom/*
 !app/Lib/EventWarning/Custom/empty
+!/app/files/certs/empty
 !/app/files/feed-metadata
 !/app/files/empty
 !/app/files/scripts/
@@ -69,7 +69,6 @@ app/Lib/EventWarning/Custom/*
 /app/files/scripts/stix2/*
 !/app/files/scripts/stix2/misp2stix2*.py
 !/app/files/scripts/stix2/stix2misp*.py
-!/app/files/empty
 /app/files/terms/*
 !/app/files/terms/empty
 !/app/files/browscap
@@ -83,10 +82,20 @@ app/Lib/EventWarning/Custom/*
 !/app/webroot/img/orgs/MISP.png
 !/app/webroot/img/orgs/NATO.png
 !/app/webroot/img/orgs/NCIRC.png
+/app/files/img/custom/*
+!/app/files/img/custom/empty
+!/app/files/img/orgs
+/app/files/img/orgs/*
+!/app/files/img/orgs/ADMIN.png
+!/app/files/img/orgs/MIL.be.png
+!/app/files/img/orgs/MISP.png
+!/app/files/img/orgs/NATO.png
+!/app/files/img/orgs/NCIRC.png
 /app/Config/bootstrap.php
 /app/Config/database.php
 /app/Config/core.php
 /app/Config/config.php
+/app/Config/hmac_key.php
 /app/Console/Command/training.json
 /app/Lib/cakephp
 /app/webroot/gpg.asc

.travis.yml

@@ -1,195 +0,0 @@
-language: php
-
-php:
-    - 7.2
-    - 7.3
-    - 7.4
-    - nightly
-
-services:
-    - redis
-
-sudo: required
-dist: bionic
-
-addons:
-    mariadb: '10.2'
-    hosts:
-        - misp.local
-        - localhost
-
-before_install:
-    - git config --global user.name "TravisCI"
-    - export PATH="$HOME/.local/bin:$PATH"
-
-install:
-    - date
-    - sudo apt-get -y update
-    # Install haveged, because Travis lacks entropy.
-    - sudo apt-get -y install haveged python3 python3-venv python3-pip python3-dev python3-nose python3-redis python3-lxml python3-dateutil python3-msgpack libxml2-dev libzmq3-dev zlib1g-dev apache2 curl php-mysql php-dev php-cli libapache2-mod-php libfuzzy-dev php-mbstring libonig4 php-json php-xml php-opcache php-readline php-redis php-gnupg php-gd
-    - sudo pip3 install --upgrade pip setuptools requests
-    - sudo pip3 install --upgrade -r requirements.txt
-    - sudo pip3 install --upgrade -r requirements-dev.txt
-    - pip3 install --user poetry
-    - phpenv rehash
-    - sudo mkdir $HOME/.composer ; sudo chown $USER:www-data $HOME/.composer
-    - pushd app
-    - sudo -H -u $USER php composer.phar install --no-progress
-    - sudo phpenmod redis
-    - sudo phpenmod gnupg
-    - popd
-    - cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php
-    # Set perms
-    - sudo chown -R $USER:www-data `pwd`
-    - sudo chmod -R 775 `pwd`
-    - sudo chmod -R g+ws `pwd`/app/tmp
-    - sudo chmod -R g+ws `pwd`/app/tmp/cache
-    - sudo chmod -R g+ws `pwd`/app/tmp/cache/persistent
-    - sudo chmod -R g+ws `pwd`/app/tmp/cache/models
-    - sudo chmod -R g+ws `pwd`/app/tmp/logs
-    - sudo chmod -R g+ws `pwd`/app/files
-    - sudo chmod -R g+ws `pwd`/app/files/scripts/tmp
-    - sudo chown -R $USER:www-data `pwd`
-    # Resque perms
-    - sudo chown -R $USER:www-data `pwd`/app/Plugin/CakeResque/tmp
-    - sudo chmod -R 755 `pwd`/app/Plugin/CakeResque/tmp
-    # install MySQL
-    - sudo chmod -R 777 `pwd`/INSTALL
-    - mysql -u root -e "SET GLOBAL sql_mode = 'STRICT_ALL_TABLES';"
-    - mysql -u root -e 'create database misp;'
-    - mysql -u root -e "grant usage on *.* to misp@localhost identified by 'blah'";
-    - mysql -u root -e "grant all privileges on misp.* to misp@localhost;"
-    - mysql -u misp -pblah misp < INSTALL/MYSQL.sql
-    # configure apache virtual hosts
-    - sudo chmod -R 777 `pwd`/build
-    - sudo mkdir -p /etc/apache2/sites-available
-    - sudo cp -f build/travis-ci-apache /etc/apache2/sites-available/misp.local.conf
-    - sudo sed -e "s?%TRAVIS_BUILD_DIR%?$(pwd)?g" --in-place /etc/apache2/sites-available/misp.local.conf
-    - sudo a2dissite 000-default
-    - sudo a2ensite misp.local.conf
-    - sudo a2enmod rewrite
-    - sudo service apache2 restart
-    # MISP configuration
-    - sudo chmod -R 777 `pwd`/travis
-    - sudo cp app/Config/bootstrap.default.php app/Config/bootstrap.php
-    - sudo cp travis/database.php app/Config/database.php
-    - sudo cp app/Config/core.default.php app/Config/core.php
-    - sudo cp app/Config/config.default.php app/Config/config.php
-    - sudo cp travis/email.php app/Config/email.php
-    # Ensure the perms
-    - sudo chown -R $USER:www-data `pwd`/app/Config
-    - sudo chmod -R 770 `pwd`/app/Config
-    # GPG setup
-    - sudo mkdir `pwd`/.gnupg
-    # /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS
-    - sudo cp -a /dev/urandom /dev/random
-    - sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/travis/gpg
-    - sudo gpg --list-secret-keys --homedir `pwd`/.gnupg
-    # change perms
-    - sudo chown -R $USER:www-data `pwd`
-    - sudo chmod +x /home/travis/build
-    - sudo chmod +x /home/travis
-    - sudo chmod +x /home
-    - sudo chmod -R 770 `pwd`/.gnupg
-    # Get authkey
-    - sudo usermod -a -G www-data $USER
-    - sudo -E su $USER -c 'app/Console/cake Admin runUpdates'
-    - sudo -E su $USER -c 'app/Console/cake userInit -q | sudo tee ./key.txt'
-    - sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.autoRegenerate" 0'
-    - sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.timeout" 600'
-    - sudo -E su $USER -c 'app/Console/cake Admin setSetting "Session.cookieTimeout" 3600'
-    - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.host_org_id" 1'
-    - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.email" "info@admin.test"'
-    - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.disable_emailing" false'
-    - sudo -E su $USER -c 'app/Console/cake Admin setSetting "debug" true'
-    - sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.CustomAuth_disable_logout" false'
-    - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_host" "127.0.0.1"'
-    - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_port" 6379'
-    - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_database" 13'
-    - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_password" ""'
-    - sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test"'
-    - sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg"'
-    - sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.password" "travistest"'
-    - sudo -E su $USER -c 'app/Console/cake Admin updateGalaxies'
-    - sudo -E su $USER -c 'app/Console/cake Admin updateTaxonomies'
-    - sudo -E su $USER -c 'app/Console/cake Admin updateWarningLists'
-    - sudo -E su $USER -c 'app/Console/cake Admin updateNoticeLists'
-    - sudo -E su $USER -c 'app/Console/cake Admin updateObjectTemplates 1'
-    - sudo -E su $USER -c 'app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" true'
-    - sudo -E su $USER -c 'app/Console/cake Live 1'
-    - sudo chmod 777 ./key.txt
-    - sudo chmod -R 777 ./tests
-    # Start workers
-    - sudo chmod +x app/Console/worker/start.sh
-    - sudo -E su $USER -c 'app/Console/worker/start.sh &'
-    - sleep 10
-    # Dirty install python stuff
-    - virtualenv -p python3.6 ./venv
-    - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$TRAVIS_BUILD_DIR/venv/bin/python"'
-    - . ./venv/bin/activate
-    - pushd cti-python-stix2
-    - pip install .
-    - popd
-    - pushd PyMISP
-    - pip install .[fileobjects]
-    - popd
-    - pip install stix zmq redis plyara
-    - deactivate
-
-before_script:
-    - curl http://misp.local
-    - AUTH=`cat key.txt`
-    - sudo chmod -R 777 PyMISP
-    - pushd PyMISP
-    - echo 'url = "http://misp.local"' >> tests/keys.py
-    - echo 'key = "'${AUTH}'"' >> tests/keys.py
-    - cat tests/keys.py
-    - popd
-
-script:
-    - ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/
-    - ./app/Vendor/bin/phpunit app/Test/ComplexTypeToolTest.php
-    - ./app/Vendor/bin/phpunit app/Test/JSONConverterToolTest.php
-    # Ensure the perms
-    - sudo chown -R $USER:www-data `pwd`/app/Config
-    - sudo chmod -R 770 `pwd`/app/Config
-    - pushd tests
-    - ./curl_tests.sh $AUTH
-    - popd
-    - pushd PyMISP
-    - git submodule init
-    - git submodule update
-    - travis_retry poetry install -E fileobjects -E openioc -E virustotal -E docs -E pdfexport
-    - poetry run python tests/testlive_comprehensive.py
-    - poetry run python tests/test_mispevent.py
-    - popd
-    - cp PyMISP/tests/keys.py PyMISP/examples/events/
-    - pushd PyMISP/examples/events/
-    - poetry run python ./create_massive_dummy_events.py -l 5 -a 30
-    - popd
-    - python3 tools/misp-feed/validate.py
-
-after_failure:
-    - curl http://misp.local
-    - cat /etc/apache2/sites-available/misp.local.conf
-    - sudo tail -n +1 `pwd`/app/tmp/logs/*
-    - sudo ls -l /var/log/apache2
-    - sudo cat /var/log/apache2/error.log
-    - sudo cat /var/log/apache2/misp.local_error.log
-    - sudo cat /var/log/apache2/misp.local_access.log
-    - pwd
-
-notifications:
-  webhooks:
-    urls:
-      - https://webhooks.gitter.im/e/05e30284086a8e948d31
-    on_success: change  # options: [always|never|change] default: always
-    on_failure: always  # options: [always|never|change] default: always
-    on_start: never     # options: [always|never|change] default: always
-
-after_success:
-    - sudo tail -n +1 `pwd`/app/tmp/logs/*
-    - coveralls
-    - coverage report
-    - coverage xml
-    - codecov

INSTALL/INSTALL.sh

@@ -3628,6 +3628,7 @@ armv7l-ubuntu-bionic
 armv7l-ubuntu-focal
 aarch64-ubuntu-focal
 aarch64-ubuntu-hirsute
+aarch64-ubuntu-jammy
 "
 
 # Check if we actually support this configuration

INSTALL/INSTALL.sh.sha1

@@ -1 +1 @@
-8624b1d834a4c958b16dce32ceae88c3d1dc15d7  INSTALL.sh
+9be7870e15e262913fc6a007d93d09d672675a7c  INSTALL.sh

INSTALL/INSTALL.sh.sha256

@@ -1 +1 @@
-d7b9e370c85b53bbf7b0f81f5c263b6ab2e534f59b40a6499277f39407ff194a  INSTALL.sh
+ee53f64a2996c551ffcccf4a928dc4a67cb5d373026b430e14df6b72a984c42d  INSTALL.sh

INSTALL/INSTALL.sh.sha384

@@ -1 +1 @@
-ef5b68e9d0d634c2cadd4fd9b1e5c2a93dbd938f488417f67a2b9c87e8867cb0200293a150cdaeda369e7fdc476eec2b  INSTALL.sh
+9f2ebadefb74bff235d86913bc825da045ff67ae23c9833683ff0780bbcbf63fefaa056ee169a97fd1c1765a483e7af4  INSTALL.sh

INSTALL/INSTALL.sh.sha512

@@ -1 +1 @@
-1445710924bc029647cc5aa0ebffd0a3b6ddaf39d26b5a0e951fffeef621677c59470f250ecfb6059c9e200951f98d4f21646f152d6f8931438531f9516a8748  INSTALL.sh
+c600dd0b2a98b570bfc66128b3637b55c2414147ea5d92c7cd5670b23f0862df3828b6e3e172777e08e2f810a88a7619f24bc7ff4a522e90f469136f791cfe14  INSTALL.sh

INSTALL/INSTALL.tpl.sh

@@ -867,6 +867,7 @@ armv7l-ubuntu-bionic
 armv7l-ubuntu-focal
 aarch64-ubuntu-focal
 aarch64-ubuntu-hirsute
+aarch64-ubuntu-jammy
 "
 
 # Check if we actually support this configuration

INSTALL/MYSQL.sql

@@ -1428,7 +1428,7 @@ CREATE TABLE IF NOT EXISTS `users` (
 -- Table structure for table `user_login_profiles`
 --
 
-CREATE TABLE `user_login_profiles` (
+CREATE TABLE IF NOT EXISTS `user_login_profiles` (
   `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
   `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
   `user_id` int(11) NOT NULL,
@@ -1698,4 +1698,4 @@ INSERT IGNORE INTO `org_blocklists` (`org_uuid`, `created`, `org_name`, `comment
 
 INSERT IGNORE INTO `admin_settings` (`setting`, `value`) VALUES 
 ('fix_login', NOW()),
-('default_role', 3);
+('default_role', 3);

PyMISP

@@ -1 +1 @@
-Subproject commit 190f82923687cebaf06138724e7c03239e529cb4
+Subproject commit 8b4f98ac4c2e6c8cc1dba064f937dac816b67d0f

README.md

@@ -46,6 +46,8 @@ The objective of MISP is to foster the sharing of structured information within
 </tr>
 </table>
 
+[![CLA FREE initiative](https://raw.githubusercontent.com/ossbase-org/ossbase.org/main/logos/cla-free-small.png)](https://ossbase.org/initiatives/cla-free/)
+
 Core functions
 ------------------
 - An **efficient IOC and indicators** database, allowing to store technical and non-technical information about malware samples, incidents, attackers and intelligence.

VERSION.json

@@ -1 +1 @@
-{"major":2, "minor":4, "hotfix":182}
+{"major":2, "minor":4, "hotfix":192}

app/Config/bootstrap.default.php

@@ -180,3 +180,7 @@ CakeLog::config('error', array(
 CakePlugin::loadAll(array(
 	'CakeResque' => array('bootstrap' => true)
 ));
+
+
+// Enable the additional exception logging for certain failures (timeouts, out of memory, etc)
+Configure::write('Exception.renderer', 'AppExceptionRenderer');

app/Console/Command/AdminShell.php

@@ -46,11 +46,11 @@ class AdminShell extends AppShell
             'help' => __('Update the JSON definition of taxonomies.'),
         ));
         $parser->addSubcommand('setSetting', [
-            'help' => __('Set setting in PHP config file.'),
+            'help' => __('Set setting in MISP config'),
             'parser' => [
                 'arguments' => [
                     'name' => ['help' => __('Setting name'), 'required' => true],
-                    'value' => ['help' => __('Setting value'), 'required' => true],
+                    'value' => ['help' => __('Setting value')],
                 ],
                 'options' => [
                     'force' => [
@@ -72,7 +72,7 @@ class AdminShell extends AppShell
             'help' => __('Set if MISP instance is live and accessible for users.'),
             'parser' => [
                 'arguments' => [
-                    'state' => ['help' => __('Set Live state')],
+                    'state' => ['help' => __('Set Live state (boolean). If not provided, current state will be printed.')],
                 ],
             ],
         ]);
@@ -85,6 +85,14 @@ class AdminShell extends AppShell
                 ],
             ],
         ]);
+        $parser->addSubcommand('isEncryptionKeyValid', [
+            'help' => __('Check if current encryption key is valid.'),
+            'parser' => [
+                'options' => [
+                    'encryptionKey' => ['help' => __('Encryption key to test. If not provided, current key will be used.')],
+                ],
+            ],
+        ]);
         $parser->addSubcommand('dumpCurrentDatabaseSchema', [
             'help' => __('Dump current database schema to JSON file.'),
         ]);
@@ -109,6 +117,20 @@ class AdminShell extends AppShell
         $parser->addSubcommand('configLint', [
             'help' => __('Check if settings has correct value.'),
         ]);
+        $parser->addSubcommand('createZmqConfig', [
+            'help' => __('Create config file for ZeroMQ server.'),
+        ]);
+        $parser->addSubcommand('scanAttachment', [
+            'help' => __('Scan attachments with AV.'),
+            'parser' => [
+                'arguments' => [
+                    'type' => ['help' => __('all, Attribute or ShadowAttribute'), 'required' => true],
+                    'attributeId' => ['help' => __('ID to scan.')],
+                    'jobId' => ['help' => __('Job ID')],
+
+                ],
+            ],
+        ]);
         return $parser;
     }
 
@@ -280,16 +302,22 @@ class AdminShell extends AppShell
     public function updateJSON()
     {
         $this->out('Updating all JSON structures.');
-        $results = $this->Server->updateJSON();
-        foreach ($results as $type => $result) {
+        $overallSuccess = true;
+        foreach ($this->Server->updateJSON() as $type => $result) {
             $type = Inflector::pluralize(Inflector::humanize($type));
-            if ($result !== false) {
-                $this->out(__('%s updated.', $type));
+            if ($result['success']) {
+                $this->out(__('%s updated in %.2f seconds.', $type, $result['duration']));
             } else {
                 $this->out(__('Could not update %s.', $type));
+                $this->out($result['result']);
+                $overallSuccess = false;
             }
         }
-        $this->out('All JSON structures updated. Thank you and have a very safe and productive day.');
+        if ($overallSuccess) {
+            $this->out('All JSON structures updated. Thank you and have a very safe and productive day.');
+        } else {
+            $this->error('Some structure could no be updated');
+        }
     }
 
     public function updateGalaxies()
@@ -485,32 +513,47 @@ class AdminShell extends AppShell
                 }
             }
         }
-        echo json_encode($result, JSON_PRETTY_PRINT) . PHP_EOL;
+        $this->out($this->json($result));
   }
 
     public function setSetting()
     {
-        list($setting_name, $value) = $this->args;
-        if ($value === 'false') {
-            $value = 0;
-        } elseif ($value === 'true') {
-            $value = 1;
-        }
-        if ($this->params['null']) {
+        list($settingName) = $this->args;
+
+        if ($this->params['null'] && isset($this->args[1])) {
+            $this->error(__('Trying to set setting to null value, but value was provided.'));
+        } else if ($this->params['null']) {
             $value = null;
+        } elseif (isset($this->args[1])) {
+            $value = $this->args[1];
+        } else {
+            $this->error(__('No setting value provided.'));
         }
-        $cli_user = array('id' => 0, 'email' => 'SYSTEM', 'Organisation' => array('name' => 'SYSTEM'));
-        if (empty($setting_name) || ($value === null && !$this->params['null'])) {
-            die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Set setting'] . PHP_EOL);
-        }
-        $setting = $this->Server->getSettingData($setting_name);
+
+        $setting = $this->Server->getSettingData($settingName);
         if (empty($setting)) {
-            $message =  'Invalid setting "' . $setting_name . '". Please make sure that the setting that you are attempting to change exists and if a module parameter, the modules are running.' . PHP_EOL;
+            $message = 'Invalid setting "' . $settingName . '". Please make sure that the setting that you are attempting to change exists and if a module parameter, the modules are running.' . PHP_EOL;
             $this->error(__('Setting change rejected.'), $message);
         }
-        $result = $this->Server->serverSettingsEditValue($cli_user, $setting, $value, $this->params['force']);
+
+        // Convert value to boolean or to int
+        if ($value !== null) {
+            if ($setting['type'] === 'boolean') {
+                $value = $this->toBoolean($value);
+            } else if ($setting['type'] === 'numeric') {
+                if (is_numeric($value)) {
+                    $value = (int)$value;
+                } elseif ($value === 'true' || $value === 'false') {
+                    $value = $value === 'true' ? 1 : 0; // special case for `debug` setting
+                } else {
+                    $this->error(__('Setting "%s" change rejected.', $settingName), __('Provided value %s is not a number.', $value));
+                }
+            }
+        }
+
+        $result = $this->Server->serverSettingsEditValue('SYSTEM', $setting, $value, $this->params['force']);
         if ($result === true) {
-            $this->out(__('Setting "%s" changed to %s', $setting_name, is_string($value) ? '"' . $value . '"' : (string)$value));
+            $this->out(__('Setting "%s" changed to %s', $settingName, is_string($value) ? '"' . $value . '"' : json_encode($value)));
         } else {
             $message = __("The setting change was rejected. MISP considers the requested setting value as invalid and would lead to the following error:\n\n\"%s\"\n\nIf you still want to force this change, please supply the --force argument.\n", $result);
             $this->error(__('Setting change rejected.'), $message);
@@ -579,9 +622,9 @@ class AdminShell extends AppShell
         try {
             $redis = RedisTool::init();
             for ($i = 0; $i < 10; $i++) {
-                $persistence = $redis->info('persistence');
-                if (isset($persistence['loading']) && $persistence['loading']) {
-                    $this->out('Redis is still loading...');
+                $pong = $redis->ping();
+                if ($pong !== true) {
+                    $this->out('Redis is still loading... ' . $pong);
                     sleep(1);
                 } else {
                     break;
@@ -648,6 +691,8 @@ class AdminShell extends AppShell
      */
     public function change_authkey()
     {
+        $this->deprecated('cake user change_authkey [user_id]');
+
         if (empty($this->args[0])) {
             echo 'MISP apikey command line tool' . PHP_EOL . 'To assign a new random API key for a user: ' . APP . 'Console/cake Admin change_authkey [user_email]' . PHP_EOL . 'To assign a fixed API key: ' . APP . 'Console/cake Admin change_authkey [user_email] [authkey]' . PHP_EOL;
             die();
@@ -787,6 +832,8 @@ class AdminShell extends AppShell
      */
     public function UserIP()
     {
+        $this->deprecated('cake user user_ips [user_id]');
+
         if (empty($this->args[0])) {
             die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get IPs for user ID'] . PHP_EOL);
         }
@@ -814,6 +861,8 @@ class AdminShell extends AppShell
      */
     public function IPUser()
     {
+        $this->deprecated('cake user ip_user [ip]');
+
         if (empty($this->args[0])) {
             die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get user ID for user IP'] . PHP_EOL);
         }
@@ -839,8 +888,8 @@ class AdminShell extends AppShell
     public function scanAttachment()
     {
         $input = $this->args[0];
-        $attributeId = isset($this->args[1]) ? $this->args[1] : null;
-        $jobId = isset($this->args[2]) ? $this->args[2] : null;
+        $attributeId = $this->args[1] ?? null;
+        $jobId = $this->args[2] ?? null;
 
         $this->loadModel('AttachmentScan');
         $result = $this->AttachmentScan->scan($input, $attributeId, $jobId);
@@ -951,7 +1000,7 @@ class AdminShell extends AppShell
             $newStatus = $this->toBoolean($this->args[0]);
             $overallSuccess = false;
             try {
-                $redis = $this->Server->setupRedisWithException();
+                $redis = RedisTool::init();
                 if ($newStatus) {
                     $redis->del('misp:live');
                     $this->out('Set live status to True in Redis.');
@@ -980,7 +1029,7 @@ class AdminShell extends AppShell
         } else {
             $this->out('Current status:');
             $this->out('PHP Config file: ' . (Configure::read('MISP.live') ? 'True' : 'False'));
-            $newStatus = $this->Server->setupRedisWithException()->get('misp:live');
+            $newStatus = RedisTool::init()->get('misp:live');
             $this->out('Redis: ' . ($newStatus !== '0' ? 'True' : 'False'));
         }
     }
@@ -1031,6 +1080,27 @@ class AdminShell extends AppShell
         $this->out(__('New encryption key "%s" saved into config file.', $new));
     }
 
+    public function isEncryptionKeyValid()
+    {
+        $encryptionKey = $this->params['encryptionKey'] ?? null;
+        if ($encryptionKey === null) {
+            $encryptionKey = Configure::read('Security.encryption_key');
+        }
+        if (!$encryptionKey) {
+            $this->error('No encryption key provided');
+        }
+
+        /** @var SystemSetting $systemSetting */
+        $systemSetting = ClassRegistry::init('SystemSetting');
+
+        try {
+            $systemSetting->isEncryptionKeyValid($encryptionKey);
+            $this->Server->isEncryptionKeyValid($encryptionKey);
+        } catch (Exception $e) {
+            $this->error($e->getMessage(), __('Probably provided encryption key is invalid'));
+        }
+    }
+
     public function redisMemoryUsage()
     {
         $redis = RedisTool::init();
@@ -1240,4 +1310,10 @@ class AdminShell extends AppShell
             $this->Job->saveField('message', __('Database truncated: ' . $table));
         }
     }
+
+    public function createZmqConfig()
+    {
+        $this->Server->getPubSubTool()->createConfigFile();
+        $this->err("Config file created in " . PubSubTool::SCRIPTS_TMP);
+    }
 }

app/Console/Command/AppShell.php

@@ -18,6 +18,7 @@
 
 App::uses('AppModel', 'Model');
 App::uses('BackgroundJobsTool', 'Tools');
+App::uses('BenchmarkTool', 'Tools');
 
 require_once dirname(__DIR__) . '/../Model/Attribute.php';   // FIXME workaround bug where Vendor/symfony/polyfill-php80/Resources/stubs/Attribute.php is loaded instead
 
@@ -31,16 +32,25 @@ require_once dirname(__DIR__) . '/../Model/Attribute.php';   // FIXME workaround
  */
 abstract class AppShell extends Shell
 {
-    public $tasks = array('ConfigLoad');
-
     /** @var BackgroundJobsTool */
     private $BackgroundJobsTool;
 
     public function initialize()
     {
-        $this->ConfigLoad = $this->Tasks->load('ConfigLoad');
-        $this->ConfigLoad->execute();
-
+        $configLoad = $this->Tasks->load('ConfigLoad');
+        $configLoad->execute();
+        if (Configure::read('Plugin.Benchmarking_enable')) {
+            $Benchmark = new BenchmarkTool(ClassRegistry::init('User'));
+            $start_time = $Benchmark->startBenchmark();
+            register_shutdown_function(function () use ($start_time, $Benchmark) {
+                $Benchmark->stopBenchmark([
+                    'user' => 0,
+                    'controller' => 'Shell::' . $this->modelClass,
+                    'action' => $this->command,
+                    'start_time' => $start_time
+                ]);
+            });
+        }
         parent::initialize();
     }
 
@@ -84,6 +94,15 @@ abstract class AppShell extends Shell
         }
     }
 
+    /**
+     * @param string $newCommand
+     * @return void
+     */
+    protected function deprecated($newCommand)
+    {
+        $this->err("<warning>Warning: This method is deprecated. Next time please use `$newCommand`.</warning>");
+    }
+
     /**
      * @return BackgroundJobsTool
      * @throws Exception

app/Console/Command/AuthkeyShell.php

@@ -12,7 +12,7 @@ class AuthkeyShell extends AppShell {
 
     public function main()
     {
-        $this->err('This method is deprecated. Next time please use `cake user change_authkey [user] [authkey]` command.');
+        $this->deprecated('cake user change_authkey [user] [authkey]');
 
         if (!isset($this->args[0]) || empty($this->args[0])) echo 'MISP authkey reset command line tool.' . PHP_EOL . 'To assign a new authkey for a user:' . PHP_EOL . APP . 'Console/cake Authkey [email] [auth_key | optional]' . PHP_EOL;
         else {

app/Console/Command/BaseurlShell.php

@@ -11,7 +11,7 @@ class BaseurlShell extends AppShell {
 
     public function main()
     {
-        $this->err('This method is deprecated. Next time please use `cake admin setSetting MISP.baseurl [baseurl]` command.');
+        $this->deprecated('cake admin setSetting MISP.baseurl [baseurl]');
 
         $baseurl = $this->args[0];
         $result = $this->Server->testBaseURL($baseurl);

app/Console/Command/EventShell.php

@@ -53,10 +53,21 @@ class EventShell extends AppShell
         $parser->addSubcommand('mergeTags', [
             'help' => __('Merge tags'),
             'parser' => [
-                'arguments' => array(
+                'arguments' => [
                     'source' => ['help' => __('Source tag ID or name. Source tag will be deleted.'), 'required' => true],
                     'destination' => ['help' => __('Destination tag ID or name.'), 'required' => true],
-                )
+                ],
+            ],
+        ]);
+        $parser->addSubcommand('reportValidationIssuesAttributes', [
+            'help' => __('Report validation issues on attributes'),
+        ]);
+        $parser->addSubcommand('normalizeIpAddress', [
+            'help' => __('Normalize IP address format in old events'),
+            'parser' => [
+                'options' => [
+                    'dry-run' => ['help' => __('Just show what changes will be made.'), 'boolean' => true],
+                ],
             ],
         ]);
         return $parser;
@@ -636,18 +647,28 @@ class EventShell extends AppShell
         }
     }
 
-    /**
-     * @param int $userId
-     * @return array
-     */
-    private function getUser($userId)
+    public function reportValidationIssuesAttributes()
     {
-        $user = $this->User->getAuthUser($userId, true);
-        if (empty($user)) {
-            $this->error("User with ID $userId does not exist.");
+        foreach ($this->Event->Attribute->reportValidationIssuesAttributes() as $validationIssue) {
+            echo $this->json($validationIssue) . "\n";
+        }
+    }
+
+    public function normalizeIpAddress()
+    {
+        $dryRun = $this->param('dry-run');
+
+        $count = 0;
+        foreach ($this->Event->Attribute->normalizeIpAddress($dryRun) as $attribute) {
+            $count++;
+            echo JsonTool::encode($attribute) . "\n";
+        }
+
+        if ($dryRun) {
+            $this->err(__n("%s attribute to fix", "%s attributes to fix", $count, $count));
+        } else {
+            $this->err(__n("%s attribute fixed", "%s attributes fixed", $count, $count));
         }
-        Configure::write('CurrentUserId', $user['id']); // for audit logging purposes
-        return $user;
     }
 
     public function generateTopCorrelations()
@@ -668,4 +689,18 @@ class EventShell extends AppShell
             $this->Job->save($job);
         }
     }
+
+    /**
+     * @param int $userId
+     * @return array
+     */
+    private function getUser($userId)
+    {
+        $user = $this->User->getAuthUser($userId, true);
+        if (empty($user)) {
+            $this->error("User with ID $userId does not exist.");
+        }
+        Configure::write('CurrentUserId', $user['id']); // for audit logging purposes
+        return $user;
+    }
 }

app/Console/Command/LiveShell.php

@@ -1,8 +1,9 @@
 <?php
-/*
+/**
  * Enable/disable misp
  *
  * arg0 = [0|1]
+ * @deprecated Use AdminShell::live instead
  */
 class LiveShell extends AppShell {
 
@@ -10,6 +11,8 @@ class LiveShell extends AppShell {
 
     public function main()
     {
+        $this->deprecated('cake admin live [0|1]');
+
         $live = $this->args[0];
         if ($live != 0 && $live != 1) {
             echo 'Invalid parameters. Usage: /var/www/MISP/app/Console/cake Live [0|1]';

app/Console/Command/Ls22Shell.php

@@ -20,7 +20,8 @@ class Ls22Shell extends AppShell
                 $this->__servers[] = [
                     'Server' => [
                         'url' => trim($fields[0]),
-                        'authkey' => trim($fields[2])
+                        'authkey' => trim($fields[2]),
+                        'self_signed' => true,
                     ]
                 ];
             }
@@ -30,6 +31,7 @@ class Ls22Shell extends AppShell
     public function getOptionParser()
     {
         $this->stdout->styles('green', array('text' => 'green'));
+        $this->stdout->styles('black', array('text' => 'black'));
 
         $parser = parent::getOptionParser();
         $parser->addSubcommand('enableTaxonomy', [
@@ -414,6 +416,13 @@ class Ls22Shell extends AppShell
 
     public function scores()
     {
+        $MITIGATION_DETECTION_OBJECT_UUIDs = [
+            'b5acf82e-ecca-4868-82fe-9dbdf4d808c3', # yara
+            '3c177337-fb80-405a-a6c1-1b2ddea8684a', # suricata
+            'aa21a3cd-ab2c-442a-9999-a5e6626591ec', # sigma
+            '6bce7d01-dbec-4054-b3c2-3655a19382e2', # script
+            '35b4dd03-4fa9-4e0e-97d8-a2867b11c956', # yabin
+        ];
         $results = [];
         $this->__getInstances($this->param('instances'));
         $server = null;
@@ -464,6 +473,7 @@ class Ls22Shell extends AppShell
             $params = [
                 'org' => $org_id,
                 'includeWarninglistHits' => true,
+                // 'includeAnalystData' => true,
             ];
             if (!empty($time_range)) {
                 $params['publish_timestamp'] = $time_range;
@@ -486,6 +496,8 @@ class Ls22Shell extends AppShell
                 'warnings' => 0,
                 'events_extended' => 0,
                 'extending_events' => 0,
+                'mitigation_detection_rules_count' => 0,
+                'analyst_data_count' => 0,
             ];
             foreach ($events['response'] as $event) {
                 $event_uuid_per_org[$event['Event']['uuid']] = $event['Event']['Orgc']['name'];
@@ -511,6 +523,11 @@ class Ls22Shell extends AppShell
                         }
                     }
                 }
+
+                #  ['Note' => 0, 'Opinion' => 0, 'Relationship' => 0,]
+                $analystDataCount = $this->countAnalystData($event['Event'], $org_name);
+                $results[$org_name]['analyst_data_count'] = $analystDataCount['Note'] + $analystDataCount['Opinion'] + $analystDataCount['Relationship'];
+
                 foreach ($event['Event']['Attribute'] as $attribute) {
                     if (!empty($attribute['referenced_by'])) {
                         $results[$org_name]['connected_elements'] +=1;
@@ -533,6 +550,9 @@ class Ls22Shell extends AppShell
                     foreach ($event['Event']['Object'] as $object) {
                         $results[$org_name]['attribute_count'] += count($object['Attribute']);
                         $results[$org_name]['object_count'] += 1;
+                        if (in_array($object['template_uuid'], $MITIGATION_DETECTION_OBJECT_UUIDs)) {
+                            $results[$org_name]['mitigation_detection_rules_count'] += 1;
+                        }
                         if (!empty($object['ObjectReference'])) {
                             $results[$org_name]['connected_elements'] += 1;
                         }
@@ -547,6 +567,9 @@ class Ls22Shell extends AppShell
                                 }
                             }
                         }
+                        if (!empty($attribute['warnings'])) {
+                            $results[$org_name]['warnings'] += 1;
+                        }
                     }
                 }
 
@@ -579,13 +602,24 @@ class Ls22Shell extends AppShell
                 } else {
                     $results[$k]['metrics']['warnings'] = 0;
                 }
+                $results[$k]['metrics']['mitigation_detection_rules'] = 100 * ($result['mitigation_detection_rules_count'] / ($result['event_count']));
                 $results[$k]['metrics']['connectedness'] = 100 * ($result['connected_elements'] / ($result['attribute_count'] + $result['object_count']));
                 $results[$k]['metrics']['attack_weight'] = 100 * (2*($result['attack']) + $result['attribute_attack']) / ($result['attribute_count'] + $result['object_count']);
                 $results[$k]['metrics']['other_weight'] = 100 * (2*($result['other']) + $result['attribute_other']) / ($result['attribute_count'] + $result['object_count']);
-                $results[$k]['metrics']['collaboration'] = 100 * ((2*$result['events_extended'] + $result['extending_events']) / $result['event_count']);
-                $results[$k]['metrics']['collaboration'] = 100 * (2*(2*$result['events_extended'] + $result['extending_events']) / $result['event_count']);
+                // $results[$k]['metrics']['collaboration'] = 100 * ((2*$result['events_extended'] + $result['extending_events']) / $result['event_count']);
+                // $results[$k]['metrics']['collaboration'] = 100 * (2*(2*$result['events_extended'] + $result['extending_events']) / $result['event_count']);
+                $results[$k]['metrics']['collaboration'] = 100 * (($result['events_extended'] + $result['extending_events']));
+                
+                # Math magic to give lots of points of you extend or have your events extended. You quickly get point, but it slows down very quick
+                if (($result['events_extended'] + $result['extending_events']) == 0) {
+                    $results[$k]['metrics']['collaboration'] = 0;
+                } else {
+                    $results[$k]['metrics']['collaboration'] = min(5*log(($result['events_extended'] + $result['extending_events']), 1.17), 100);
+                }
+
+                $results[$k]['metrics']['collaboration_analyst'] = $result['analyst_data_count'] > 0 ? 100 : 0;
             }
-            foreach (['connectedness', 'attack_weight', 'other_weight', 'warnings', 'collaboration'] as $metric) {
+            foreach (['connectedness',  'mitigation_detection_rules', 'attack_weight', 'other_weight', 'warnings', 'collaboration', 'collaboration_analyst'] as $metric) {
                 if (empty($results[$k]['metrics'][$metric])) {
                     $results[$k]['metrics'][$metric] = 0;
                 }
@@ -594,18 +628,21 @@ class Ls22Shell extends AppShell
                 }
             }
             $results[$k]['score'] = round(
-                    20 * $results[$k]['metrics']['warnings'] +
-                    20 * $results[$k]['metrics']['connectedness'] +
+                    15 * $results[$k]['metrics']['warnings'] +
+                    15 * $results[$k]['metrics']['mitigation_detection_rules'] +
+                    10 * $results[$k]['metrics']['connectedness'] +
                     40 * $results[$k]['metrics']['attack_weight'] +
                     10 * $results[$k]['metrics']['other_weight'] +
+                    // 7 * $results[$k]['metrics']['collaboration'] + 3 * $results[$k]['metrics']['collaboration_analyst']
                     10 * $results[$k]['metrics']['collaboration']
                 ) / 100;
             $scores[$k]['total'] = $results[$k]['score'];
-            $scores[$k]['warnings'] = round(20 * $results[$k]['metrics']['warnings']);
-            $scores[$k]['connectedness'] = round(20 * $results[$k]['metrics']['connectedness']);
+            $scores[$k]['warnings'] = round(15 * $results[$k]['metrics']['warnings']);
+            $scores[$k]['mitigation_detection_rules'] = round(15 * $results[$k]['metrics']['mitigation_detection_rules']);
+            $scores[$k]['connectedness'] = round(10 * $results[$k]['metrics']['connectedness']);
             $scores[$k]['attack_weight'] = round(40 * $results[$k]['metrics']['attack_weight']);
             $scores[$k]['other_weight'] = round(10 * $results[$k]['metrics']['other_weight']);
-            $scores[$k]['collaboration'] = round(10 * $results[$k]['metrics']['collaboration']);
+            $scores[$k]['collaboration'] = round(7 * $results[$k]['metrics']['collaboration']) + round(3 * $results[$k]['metrics']['collaboration_analyst']);
         }
         arsort($scores, SORT_DESC);
         $this->out(str_repeat('=', 128), 1, Shell::NORMAL);
@@ -618,20 +655,22 @@ class Ls22Shell extends AppShell
         $this->out(str_repeat('=', 128), 1, Shell::NORMAL);
         foreach ($scores as $org => $score) {
             $score_string[0] = str_repeat('█', round($score['warnings']/100));
-            $score_string[1] = str_repeat('█', round($score['connectedness']/100));
-            $score_string[2] = str_repeat('█', round($score['attack_weight']/100));
-            $score_string[3] = str_repeat('█', round($score['other_weight']/100));
-            $score_string[4] = str_repeat('█', round($score['collaboration']/100));
+            $score_string[1] = str_repeat('█', round($score['mitigation_detection_rules']/100));
+            $score_string[2] = str_repeat('█', round($score['connectedness']/100));
+            $score_string[3] = str_repeat('█', round($score['attack_weight']/100));
+            $score_string[4] = str_repeat('█', round($score['other_weight']/100));
+            $score_string[5] = str_repeat('█', round($score['collaboration']/100));
             $this->out(sprintf(
                 '| %s | %s | %s |',
                 str_pad($org, 10, ' ', STR_PAD_RIGHT),
                 sprintf(
-                    '<error>%s</error><warning>%s</warning><question>%s</question><info>%s</info><green>%s</green>%s',
+                    '<error>%s</error><black>%s</black><warning>%s</warning><question>%s</question><info>%s</info><green>%s</green>%s',
                     $score_string[0],
                     $score_string[1],
                     $score_string[2],
                     $score_string[3],
                     $score_string[4],
+                    $score_string[5],
                     str_repeat(' ', 100 - mb_strlen(implode('', $score_string)))
                 ),
                 str_pad($score['total'] . '%', 8, ' ', STR_PAD_RIGHT)
@@ -639,8 +678,9 @@ class Ls22Shell extends AppShell
         }
         $this->out(str_repeat('=', 128), 1, Shell::NORMAL);
         $this->out(sprintf(
-            '| Legend: %s %s %s %s %s |',
+            '| Legend: %s %s %s %s %s %s |',
             '<error>█: Warnings</error>',
+            '<black>█: Detection/Mitigation Rules</black>',
             '<warning>█: Connectedness</warning>',
             '<question>█: ATT&CK context</question>',
             '<info>█: Other Context</info>',
@@ -650,4 +690,30 @@ class Ls22Shell extends AppShell
         $this->out(str_repeat('=', 128), 1, Shell::NORMAL);
         file_put_contents(APP . 'tmp/report.json', json_encode($results, JSON_PRETTY_PRINT));
     }
+
+    private function countAnalystData($data, $orgc_name): array {
+        $analystTypes = ['Note', 'Opinion', 'Relationship'];
+        $counts = [
+            'Note' => 0,
+            'Opinion' => 0,
+            'Relationship' => 0,
+        ];
+
+        foreach ($analystTypes as $type) {
+            if (!empty($data[$type])) {
+                foreach ($data[$type] as $entry) {
+                    if ($entry['Orgc']['name'] == $orgc_name) {
+                        $counts[$type] += 1;
+                    }
+                }
+                foreach ($data[$type] as $child) {
+                    $nestedCounts = $this->countAnalystData($child, $orgc_name);
+                    $counts['Note'] += $nestedCounts['Note'];
+                    $counts['Opinion'] += $nestedCounts['Opinion'];
+                    $counts['Relationship'] += $nestedCounts['Relationship'];
+                }
+            }
+        }
+        return $counts;
+    }
 }

app/Console/Command/OrganisationShell.php

@@ -0,0 +1,64 @@
+<?php
+
+/**
+ * @property User $User
+ * @property Log $Log
+ * @property UserLoginProfile $UserLoginProfile
+ */
+class OrganisationShell extends AppShell
+{
+    public $uses = ['Organisation'];
+
+    public function getOptionParser()
+    {
+        $parser = parent::getOptionParser();
+        $parser->addSubcommand('list', [
+            'help' => __('Get list of organisations.'),
+            'parser' => [
+                'arguments' => [
+                    'filter' => ['help' => __('Filter the list by name.'), 'required' => false],
+                    'local' => ['help' => __('Filter the list by local/known organisations.'), 'required' => false],
+                ],
+                'options' => [
+                    'json' => ['help' => __('Output as JSON.'), 'boolean' => true],
+                ],
+            ]
+        ]);
+        return $parser;
+    }
+
+    public function list()
+    {
+        $filter = $this->args[0] ?? null;
+        $local = isset($this->args[1]) ? $this->args[1] : null;
+        $conditions = [];
+        if ($filter) {
+            $conditions = ['OR' => [
+                'Organisation.name LIKE' => "%$filter%",
+                'Organisation.uuid LIKE' => "%$filter%"
+            ]];
+        }
+        if ($local !== null) {
+            $conditions['OR'][] = [
+                'Organisation.local' => $local
+            ];
+        }
+        $organisations = $this->Organisation->find('all', [
+            'recursive' => -1,
+            'conditions' => $conditions
+        ]);
+        if ($this->params['json']) {
+            $this->out($this->json($organisations));
+        } else {
+            foreach ($organisations as $organisation) {
+                $this->out(sprintf(
+                    '%d. [%s] %s',
+                    $organisation['Organisation']['id'],
+                    $organisation['Organisation']['uuid'],
+                    $organisation['Organisation']['name']
+                ));
+            }
+            $this->out(count($organisations) . ' hits.');
+        }
+    }
+}

app/Console/Command/PasswordShell.php

@@ -12,7 +12,7 @@ class PasswordShell extends AppShell {
 
     public function main()
     {
-        $this->err('This method is deprecated. Next time please use `cake user change_pw [user] [password]` command.');
+        $this->deprecated('cake user change_pw [user] [password]');
 
         if (!isset($this->args[0]) || empty($this->args[0]) || !isset($this->args[1]) || empty($this->args[1])) echo 'MISP password reset command line tool.' . PHP_EOL . 'To assign a new password for a user:' . PHP_EOL . APP . 'Console/cake Password [email] [password]' . PHP_EOL;
         else {

app/Console/Command/RoleShell.php

@@ -0,0 +1,56 @@
+<?php
+
+/**
+ * @property User $User
+ * @property Log $Log
+ * @property UserLoginProfile $UserLoginProfile
+ */
+class RoleShell extends AppShell
+{
+    public $uses = ['Role'];
+
+    public function getOptionParser()
+    {
+        $parser = parent::getOptionParser();
+        $parser->addSubcommand('list', [
+            'help' => __('Get list of the roles.'),
+            'parser' => [
+                'arguments' => [
+                    'filter' => ['help' => __('Filter list by name.'), 'required' => false],
+                ],
+                'options' => [
+                    'json' => ['help' => __('Output as JSON.'), 'boolean' => true],
+                ],
+            ]
+        ]);
+        return $parser;
+    }
+
+    public function list()
+    {
+        $filter = $this->args[0] ?? null;
+        if ($filter) {
+            $conditions = ['OR' => [
+                'Role.name LIKE' => "%$filter%"
+            ]];
+        } else {
+            $conditions = [];
+        }
+        $roles = $this->Role->find('all', [
+            'recursive' => -1,
+            'conditions' => $conditions
+        ]);
+        if ($this->params['json']) {
+            $this->out($this->json($roles));
+        } else {
+            foreach ($roles as $role) {
+                $this->out(sprintf(
+                    '%d. %s',
+                    $role['Role']['id'],
+                    $role['Role']['name']
+                ));
+            }
+        }
+        $this->out(count($roles) . ' hits.');
+    }
+}

app/Console/Command/ServerShell.php

@@ -85,6 +85,7 @@ class ServerShell extends AppShell
 
         $userId = $this->args[0];
         $user = $this->getUser($userId);
+        Configure::write('CurrentUserId', $userId);
 
         if (!empty($this->args[1])) {
             $technique = $this->args[1];
@@ -124,11 +125,11 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Pull'] . PHP_EOL);
         }
-
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $serverId = $this->args[1];
         $server = $this->getServer($serverId);
+        Configure::write('CurrentUserId', $userId);
         if (!empty($this->args[2])) {
             $technique = $this->args[2];
         } else {
@@ -143,10 +144,14 @@ class ServerShell extends AppShell
         if (!empty($this->args[4]) && $this->args[4] === 'force') {
             $force = true;
         }
+
+        // Try to enable garbage collector as pulling events can use a lot of memory
+        gc_enable();
+
         try {
             $result = $this->Server->pull($user, $technique, $server, $jobId, $force);
             if (is_array($result)) {
-                $message = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4]);
+                $message = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled, %s analyst data pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4], $result[5]);
                 $this->Job->saveStatus($jobId, true, $message);
             } else {
                 $message = __('ERROR: %s', $result);
@@ -164,11 +169,12 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Push'] . PHP_EOL);
         }
-
+        
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $serverId = $this->args[1];
         $server = $this->getServer($serverId);
+        Configure::write('CurrentUserId', $userId);
         $technique = empty($this->args[2]) ? 'full' : $this->args[2];
         if (!empty($this->args[3])) {
             $jobId = $this->args[3];
@@ -203,6 +209,7 @@ class ServerShell extends AppShell
         $userId = $this->args[0];
         $user = $this->getUser($userId);
 
+        Configure::write('CurrentUserId', $userId);
         $technique = isset($this->args[1]) ? $this->args[1] : 'full';
 
         $servers = $this->Server->find('list', array(
@@ -366,7 +373,7 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Fetch feeds as local data'] . PHP_EOL);
         }
-
+        
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $feedId = $this->args[1];
@@ -422,7 +429,7 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Cache server'] . PHP_EOL);
         }
-
+        
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $scope = $this->args[1];
@@ -485,7 +492,7 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Cache feeds for quick lookups'] . PHP_EOL);
         }
-
+        
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $scope = $this->args[1];
@@ -731,6 +738,7 @@ class ServerShell extends AppShell
 
     public function sendPeriodicSummaryToUsers()
     {
+        
         $periods = $this->__getPeriodsForToday();
         $start_time = time();
         echo __n('Started periodic summary generation for the %s period', 'Started periodic summary generation for periods: %s', count($periods), implode(', ', $periods)) . PHP_EOL;
@@ -796,7 +804,7 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Push Taxii'] . PHP_EOL);
         }
-
+        
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $serverId = $this->args[1];

app/Console/Command/StartWorkerShell.php

@@ -37,24 +37,32 @@ class StartWorkerShell extends AppShell
 
     public function main()
     {
+        $pid = getmypid();
+        if ($pid === false) {
+            throw new RuntimeException("Could not get current process ID");
+        }
+
         $this->worker = new Worker(
             [
-                'pid' => getmypid(),
+                'pid' => $pid,
                 'queue' => $this->args[0],
                 'user' => ProcessTool::whoami(),
             ]
         );
 
         $this->maxExecutionTime = (int)$this->params['maxExecutionTime'];
+        $queue = $this->worker->queue();
+        $backgroundJobTool = $this->getBackgroundJobsTool();
 
-        CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - starting to process background jobs...");
+        CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$queue}] - starting to process background jobs...");
 
         while (true) {
             $this->checkMaxExecutionTime();
 
-            $job = $this->getBackgroundJobsTool()->dequeue($this->worker->queue());
+            $job = $backgroundJobTool->dequeue($queue);
             if ($job) {
                 $this->runJob($job);
+                $backgroundJobTool->removeFromRunning($this->worker, $job);
             }
         }
     }
@@ -64,7 +72,7 @@ class StartWorkerShell extends AppShell
      */
     private function runJob(BackgroundJob $job)
     {
-        CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}...");
+        CakeLog::info("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - launching job with ID: {$job->id()}");
 
         try {
             $job->setStatus(BackgroundJob::STATUS_RUNNING);
@@ -73,12 +81,16 @@ class StartWorkerShell extends AppShell
             CakeLog::info("[JOB ID: {$job->id()}] - started command `$command`.");
             $this->getBackgroundJobsTool()->update($job);
 
-            $job->run();
+            $start = microtime(true);
+            $job->run(function (array $status) use ($job) {
+                $this->getBackgroundJobsTool()->markAsRunning($this->worker, $job, $status['pid']);
+            });
+            $duration = number_format(microtime(true) - $start, 3, '.', '');
 
             if ($job->status() === BackgroundJob::STATUS_COMPLETED) {
-                CakeLog::info("[JOB ID: {$job->id()}] - completed.");
+                CakeLog::info("[JOB ID: {$job->id()}] - successfully completed in $duration seconds.");
             } else {
-                CakeLog::error("[JOB ID: {$job->id()}] - failed with error code {$job->returnCode()}. STDERR: {$job->error()}. STDOUT: {$job->output()}.");
+                CakeLog::error("[JOB ID: {$job->id()}] - failed with error code {$job->returnCode()} after $duration seconds. STDERR: {$job->error()}. STDOUT: {$job->output()}.");
             }
         } catch (Exception $exception) {
             CakeLog::error("[WORKER PID: {$this->worker->pid()}][{$this->worker->queue()}] - job ID: {$job->id()} failed with exception: {$exception->getMessage()}");

app/Console/Command/StatisticsShell.php

@@ -303,10 +303,10 @@ class StatisticsShell extends AppShell {
         $this->out(json_encode([
             'events' => $this->Event->find('count'),
             'attributes' => $this->Event->Attribute->find('count',
-                ['conditions' => ['Attribute.deleted' => 0], 'recursive' => -1]
+                ['recursive' => -1]
             ),
             'objects' => $this->Event->Object->find('count',
-                ['conditions' => ['Object.deleted' => 0], 'recursive' => -1]
+                ['recursive' => -1]
             ),
             'correlations' => $this->Correlation->find('count') / 2,
             'users' => $this->User->find('count',

app/Console/Command/Task/ConfigLoadTask.php

@@ -3,8 +3,6 @@ class ConfigLoadTask extends Shell
 {
     public function execute()
     {
-        Configure::load('config');
-
         if (Configure::read('MISP.system_setting_db')) {
             App::uses('SystemSetting', 'Model');
             SystemSetting::setGlobalSetting();

app/Console/Command/UserInitShell.php

@@ -1,7 +1,13 @@
 <?php
+
+/**
+ * @deprecated
+ */
 class UserInitShell extends AppShell {
 	public $uses = array('User', 'Role', 'Organisation', 'Server', 'ConnectionManager');
 	public function main() {
+        $this->deprecated('cake user init');
+
 		if (!Configure::read('Security.salt')) {
 			$this->loadModel('Server');
 			$this->Server->serverSettingsSaveValue('Security.salt', $this->User->generateRandomPassword(32));

app/Console/Command/UserShell.php

@@ -3,10 +3,11 @@
 /**
  * @property User $User
  * @property Log $Log
+ * @property UserLoginProfile $UserLoginProfile
  */
 class UserShell extends AppShell
 {
-    public $uses = ['User', 'Log'];
+    public $uses = ['User', 'Log', 'UserLoginProfile'];
 
     public function getOptionParser()
     {
@@ -15,23 +16,45 @@ class UserShell extends AppShell
             'help' => __('Get list of user accounts.'),
             'parser' => [
                 'arguments' => [
-                    'userId' => ['help' => __('User ID or e-mail address.'), 'required' => true],
+                    'userId' => ['help' => __('User ID or e-mail address to filter.'), 'required' => false],
                 ],
                 'options' => [
                     'json' => ['help' => __('Output as JSON.'), 'boolean' => true],
                 ],
             ]
         ]);
+        $parser->addSubcommand('create', [
+            'help' => __('Create a new user account.'),
+            'parser' => [
+                'arguments' => [
+                    'email' => ['help' => __('E-mail address (also used as the username.'), 'required' => true],
+                    'role_id' => ['help' => __('Role ID of the user. For a list of available roles, use `cake Roles list`.'), 'required' => true],
+                    'org_id' => ['help' => __('Organisation under which the user should be created'), 'required' => true],
+                    'password' => ['help' => __('Enter a password to assign to the user (optional) - if none is set, the user will receive a temporary password.')]
+                ],
+                'options' => [
+                    'json' => ['help' => __('Output as JSON.'), 'boolean' => true],
+                ],
+            ]
+        ]);
+        $parser->addSubcommand('init', [
+            'help' => __('Create default role, organisation and user when not exists.'),
+        ]);
         $parser->addSubcommand('authkey', [
             'help' => __('Get information about given authkey.'),
             'parser' => [
                 'arguments' => [
-                    'authkey' => ['help' => __('Authentication key. If not provide, it will be read from STDIN.')],
+                    'authkey' => ['help' => __('Authentication key. If not provided, it will be read from STDIN.')],
                 ],
             ]
         ]);
         $parser->addSubcommand('authkey_valid', [
             'help' => __('Check if given authkey by STDIN is valid.'),
+            'parser' => [
+                'options' => [
+                    'disableStdLog' => ['help' => __('Do not show logs in STDOUT or STDERR.'), 'boolean' => true],
+                ],
+            ],
         ]);
         $parser->addSubcommand('block', [
             'help' => __('Immediately block user.'),
@@ -73,6 +96,15 @@ class UserShell extends AppShell
                 ],
             ],
         ]);
+        $parser->addSubcommand('change_role', [
+            'help' => __('Change user role.'),
+            'parser' => [
+                'arguments' => [
+                    'userId' => ['help' => __('User ID or e-mail address.'), 'required' => true],
+                    'new_role' => ['help' => __('Role ID or Role name.'), 'required' => true],
+                ]
+            ],
+        ]);
         $parser->addSubcommand('change_authkey', [
             'help' => __('Change authkey. When advanced authkeys are enabled, old authkeys will be disabled.'),
             'parser' => [
@@ -104,6 +136,14 @@ class UserShell extends AppShell
                 ],
             ],
         ]);
+        $parser->addSubcommand('ip_country', [
+            'help' => __('Get country for given IP address'),
+            'parser' => [
+                'arguments' => [
+                    'ip' => ['help' => __('IPv4 or IPv6 address.'), 'required' => true],
+                ]
+            ],
+        ]);
         $parser->addSubcommand('require_password_change_for_old_passwords', [
             'help' => __('Trigger forced password change on next login for users with an old (older than x days) password.'),
             'parser' => [
@@ -121,7 +161,7 @@ class UserShell extends AppShell
 
     public function list()
     {
-        $userId = isset($this->args[0]) ? $this->args[0] : null;
+        $userId = $this->args[0] ?? null;
         if ($userId) {
             $conditions = ['OR' => [
                 'User.id' => $userId,
@@ -163,13 +203,66 @@ class UserShell extends AppShell
         }
     }
 
+    public function create()
+    {
+        if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
+            $this->err('Invalid input. Usage: `User create [email] [role_id] [org_id] [password:optional]`');
+        }
+        $user = [
+            'email' => $this->args[0],
+            'role_id' => $this->args[1],
+            'org_id' => $this->args[2],
+            'change_pw' => true
+        ];
+        if (!empty($this->args[3])) {
+            $user['password'] = $this->args[3];
+            $user['confirm_password'] = $this->args[3];
+            $user['change_pw'] = true;
+        }
+        $this->User->create();
+        $result = $this->User->save($user);
+        // do not fetch sensitive or big values
+        $schema = $this->User->schema();
+        unset($schema['authkey']);
+        unset($schema['password']);
+        unset($schema['gpgkey']);
+        unset($schema['certif_public']);
+
+        $fields = array_keys($schema);
+        $fields[] = 'Role.*';
+        $fields[] = 'Organisation.*';
+
+        $user = $this->User->find('first', [
+            'recursive' => -1,
+            'fields' => $fields,
+            'conditions' => ['User.id' => $this->User->id],
+            'contain' => ['Organisation', 'Role', 'UserSetting'],
+        ]);
+        if ($this->params['json']) {
+            $this->out($this->json($user));
+        } else {
+            $this->out('User created.');
+        }
+    }
+
+    public function init()
+    {
+        if (!Configure::read('Security.salt')) {
+            $this->loadModel('Server');
+            $this->Server->serverSettingsSaveValue('Security.salt', $this->User->generateRandomPassword(32));
+        }
+
+        $authKey = $this->User->init();
+        if ($authKey === null) {
+            $this->err('Script aborted: MISP instance already initialised.');
+        } else {
+            $this->out($authKey);
+        }
+    }
+
     public function authkey()
     {
-        if (isset($this->args[0])) {
-            $authkey = $this->args[0];
-        } else {
-            $authkey = fgets(STDIN); // read line from STDIN
-        }
+        $authkey = $this->args[0] ?? fgets(STDIN);
         $authkey = trim($authkey);
         if (strlen($authkey) !== 40) {
             $this->error('Authkey has not valid format.');
@@ -212,26 +305,37 @@ class UserShell extends AppShell
      */
     public function authkey_valid()
     {
+        if ($this->params['disableStdLog']) {
+            $this->_useLogger(false);
+        }
+
         $cache = [];
-        do {
+        $randomKey = random_bytes(16);
+        $advancedAuthKeysEnabled = (bool)Configure::read('Security.advanced_authkeys');
+
+        while (true) {
             $authkey = fgets(STDIN); // read line from STDIN
             $authkey = trim($authkey);
             if (strlen($authkey) !== 40) {
-                fwrite(STDOUT, "0\n");  // authkey is not in valid format
+                echo "0\n";  // authkey is not in valid format
+                $this->log("Authkey in incorrect format provided, expected 40 chars long string, $authkey provided.", LOG_WARNING);
                 continue;
             }
-            $time = time();
+
             // Generate hash from authkey to not store raw authkey in memory
-            $keyHash = hash('sha256', $authkey, true);
+            $keyHash = sha1($authkey . $randomKey, true);
+
+            // If authkey is in cache and is fresh, use info from cache
+            $time = time();
             if (isset($cache[$keyHash]) && $cache[$keyHash][1] > $time) {
-                fwrite(STDOUT, $cache[$keyHash][0] ? "1\n" : "0\n");
+                echo $cache[$keyHash][0] ? "1\n" : "0\n";
                 continue;
             }
 
             $user = false;
             for ($i = 0; $i < 5; $i++) {
                 try {
-                    if (Configure::read('Security.advanced_authkeys')) {
+                    if ($advancedAuthKeysEnabled) {
                         $user = $this->User->AuthKey->getAuthUserByAuthKey($authkey);
                     } else {
                         $user = $this->User->getAuthUserByAuthkey($authkey);
@@ -249,11 +353,34 @@ class UserShell extends AppShell
                 }
             }
 
-            $user = (bool)$user;
-            // Cache results for 5 seconds
-            $cache[$keyHash] = [$user, $time + 5];
-            fwrite(STDOUT, $user ? "1\n" : "0\n");
-        } while (true);
+            if (!$user) {
+                $valid = null;
+            } else if ($user['disabled']) {
+                $valid = false;
+            } else {
+                $valid = true;
+            }
+
+            echo $valid ? "1\n" : "0\n";
+
+            if ($valid) {
+                // Cache results for 60 seconds if key is valid
+                $cache[$keyHash] = [true, $time + 60];
+            } else {
+                // Cache results for 5 seconds if key is invalid
+                $cache[$keyHash] = [false, $time + 5];
+
+                $start = substr($authkey, 0, 4);
+                $end = substr($authkey, -4);
+                $authKeyForLog = $start . str_repeat('*', 32) . $end;
+
+                if ($valid === false) {
+                    $this->log("Authkey $authKeyForLog belongs to user {$user['id']} that is disabled.", LOG_WARNING);
+                } else {
+                    $this->log("Authkey $authKeyForLog is invalid or expired.", LOG_WARNING);
+                }
+            }
+        }
     }
 
     public function block()
@@ -296,7 +423,7 @@ class UserShell extends AppShell
 
         $conditions = ['User.disabled' => false]; // fetch just not disabled users
 
-        $userId = isset($this->args[0]) ? $this->args[0] : null;
+        $userId = $this->args[0] ?? null;
         if ($userId) {
             $conditions['OR'] = [
                 'User.id' => $userId,
@@ -355,7 +482,7 @@ class UserShell extends AppShell
         }
         $user = $this->getUser($userId);
 
-        # validate new authentication key if provided
+        // validate new authentication key if provided
         if (!empty($newkey) && (strlen($newkey) != 40 || !ctype_alnum($newkey))) {
             $this->error('The new auth key needs to be 40 characters long and only alphanumeric.');
         }
@@ -381,6 +508,35 @@ class UserShell extends AppShell
         }
     }
 
+    public function change_role()
+    {
+        list($userId, $newRole) = $this->args;
+        $user = $this->getUser($userId);
+
+        if (is_numeric($newRole)) {
+            $conditions = ['Role.id' => $newRole];
+        } else {
+            $conditions = ['Role.name' => $newRole];
+        }
+
+        $newRoleFromDb = $this->User->Role->find('first', [
+            'conditions' => $conditions,
+            'fields' => ['Role.id'],
+        ]);
+
+        if (empty($newRoleFromDb)) {
+            $this->error("Role `$newRole` not found.");
+        }
+
+        if ($newRoleFromDb['Role']['id'] == $user['role_id']) {
+            $this->error("Role `$newRole` is already assigned to {$user['email']}.");
+        }
+
+        $this->User->updateField($user, 'role_id', $newRoleFromDb['Role']['id']);
+
+        $this->out("Role changed from `{$user['role_id']}` to `{$newRoleFromDb['Role']['id']}`.");
+    }
+
     public function user_ips()
     {
         list($userId) = $this->args;
@@ -390,7 +546,7 @@ class UserShell extends AppShell
             $this->out('<warning>Storing user IP addresses is disabled.</warning>');
         }
 
-        $ips = $this->User->setupRedisWithException()->smembers('misp:user_ip:' . $user['id']);
+        $ips = RedisTool::init()->smembers('misp:user_ip:' . $user['id']);
 
         if ($this->params['json']) {
             $this->out($this->json($ips));
@@ -413,36 +569,50 @@ class UserShell extends AppShell
             $this->out('<warning>Storing user IP addresses is disabled.</warning>');
         }
 
-        $userId = $this->User->setupRedisWithException()->get('misp:ip_user:' . $ip);
+        $userId = RedisTool::init()->get('misp:ip_user:' . $ip);
         if (empty($userId)) {
             $this->out('No hits.');
             $this->_stop();
         }
 
-        $user = $this->User->find('first', array(
+        $user = $this->User->find('first', [
             'recursive' => -1,
-            'conditions' => array('User.id' => $userId),
+            'conditions' => ['User.id' => $userId],
             'fields' => ['id', 'email'],
-        ));
+        ]);
 
         if (empty($user)) {
             $this->error("User with ID $userId doesn't exists anymore.");
         }
 
+        $ipCountry = $this->UserLoginProfile->countryByIp($ip);
+
         if ($this->params['json']) {
             $this->out($this->json([
                 'ip' => $ip,
                 'id' => $user['User']['id'],
                 'email' => $user['User']['email'],
+                'country' => $ipCountry,
             ]));
         } else {
-            $this->out(sprintf(
-                '%s==============================%sIP: %s%s==============================%sUser #%s: %s%s==============================%s',
-                PHP_EOL, PHP_EOL, $ip, PHP_EOL, PHP_EOL, $user['User']['id'], $user['User']['email'], PHP_EOL, PHP_EOL
-            ));
+            $this->hr();
+            $this->out("IP: $ip (country $ipCountry)");
+            $this->hr();
+            $this->out("User #{$user['User']['id']}: {$user['User']['email']}");
+            $this->hr();
         }
     }
 
+    public function ip_country()
+    {
+        list($ip) = $this->args;
+        if (!filter_var($ip, FILTER_VALIDATE_IP)) {
+            $this->error("IP `$ip` is not valid IPv4 or IPv6 address");
+        }
+
+        $this->out($this->UserLoginProfile->countryByIp($ip));
+    }
+
     public function require_password_change_for_old_passwords()
     {
         list($days) = $this->args;
@@ -499,7 +669,7 @@ class UserShell extends AppShell
     }
 
     /**
-     * @param string|int $userId
+     * @param string|int $userId User ID or User e-mail
      * @return array
      */
     private function getUser($userId)

app/Console/Command/WorkerShell.php

@@ -0,0 +1,119 @@
+<?php
+
+declare(strict_types=1);
+
+/**
+ * @property Job $Job
+ */
+class WorkerShell extends AppShell
+{
+    public $uses = ['Job'];
+
+    public function getOptionParser(): ConsoleOptionParser
+    {
+        $parser = parent::getOptionParser();
+        $parser->addSubcommand('showQueues', [
+            'help' => __('Show jobs in worker queues'),
+        ]);
+        $parser->addSubcommand('flushQueue', [
+            'help' => __('Flush jobs in given queue'),
+            'parser' => [
+                'arguments' => [
+                    'queue' => ['help' => __('Queue name'), 'required' => true],
+                ],
+            ],
+        ]);
+        $parser->addSubcommand('showJobStatus', [
+            'help' => __('Show job status'),
+            'parser' => [
+                'arguments' => [
+                    'job_id' => ['help' => __('Job ID (ID or UUID)'), 'required' => true],
+                ],
+            ],
+        ]);
+        return $parser;
+    }
+
+    /**
+     * @throws RedisException
+     * @throws JsonException
+     */
+    public function showQueues()
+    {
+        $tool = $this->getBackgroundJobsTool();
+        $runningJobs = $tool->runningJobs();
+
+        foreach (BackgroundJobsTool::VALID_QUEUES as $queue) {
+            $this->out("{$queue}:\t{$tool->getQueueSize($queue)}");
+            $queueJobs = $runningJobs[$queue] ?? [];
+            foreach ($queueJobs as $jobId => $data) {
+                $this->out(" - $jobId (" . JsonTool::encode($data) .")");
+            }
+       }
+    }
+
+    public function flushQueue()
+    {
+        $queue = $this->args[0];
+        try {
+            $this->getBackgroundJobsTool()->clearQueue($queue);
+        } catch (InvalidArgumentException $e) {
+            $this->error($e->getMessage());
+        }
+    }
+
+    public function showJobStatus()
+    {
+        $processId = $this->args[0];
+        if (is_numeric($processId)) {
+            $job = $this->Job->find('first', [
+                'conditions' => ['Job.id' => $processId],
+                'recursive' => -1,
+            ]);
+            if (!$job) {
+                $this->error('Job not found', "Job with ID {$processId} not found");
+            }
+
+            $this->out($this->json($job['Job']));
+            $processId = $job['Job']['process_id'];
+        }
+
+        if (!Validation::uuid($processId)) {
+            $this->error('Job not found', "Job ID must be number or UUID, '$processId' given");
+        }
+
+        $jobStatus = $this->getBackgroundJobsTool()->getJob($processId);
+        if (!$jobStatus) {
+            $this->error('Job not found', "Job with UUID {$processId} not found");
+        }
+
+        $jobStatus = $jobStatus->jsonSerialize();
+
+        foreach (['createdAt', 'updatedAt'] as $timeField) {
+            if (isset($jobStatus[$timeField])) {
+                $jobStatus[$timeField] = date('c', $jobStatus[$timeField]);
+            }
+        }
+
+        if (isset($jobStatus['status'])) {
+            $jobStatus['status'] = $this->jobStatusToString($jobStatus['status']);
+        }
+
+        $this->out($this->json($jobStatus));
+    }
+
+    private function jobStatusToString(int $jobStatus)
+    {
+        switch ($jobStatus) {
+            case Job::STATUS_WAITING:
+                return 'waiting';
+            case Job::STATUS_RUNNING:
+                return 'running';
+            case Job::STATUS_FAILED:
+                return 'failed';
+            case Job::STATUS_COMPLETED:
+                return 'completed';
+        }
+        throw new InvalidArgumentException("Invalid job status $jobStatus");
+    }
+}

app/Console/Command/WorkflowShell.php

@@ -17,6 +17,9 @@ class WorkflowShell extends AppShell {
         $data = JsonTool::decode($this->args[1]);
         $logging = JsonTool::decode($this->args[2]);
         $jobId = $this->args[3];
+        if (!empty($this->args[4])) {
+            Configure::write('CurrentUserId', JsonTool::decode($this->args[4]));
+        }
 
         $blockingErrors = [];
         $executionSuccess = $this->Workflow->executeWorkflowForTrigger($trigger_id, $data, $blockingErrors);

app/Controller/AnalystDataBlocklistsController.php

@@ -0,0 +1,103 @@
+<?php
+App::uses('AppController', 'Controller');
+
+class AnalystDataBlocklistsController extends AppController
+{
+    public $components = array('Session', 'RequestHandler', 'BlockList');
+
+    public $paginate = array(
+            'limit' => 60,
+            'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 entries <- no we won't, this is the max a user van view/page.
+            'order' => array(
+                'AnalystDataBlocklist.created' => 'DESC'
+            ),
+    );
+
+    public function index()
+    {
+        $passedArgsArray = array();
+        $passedArgs = $this->passedArgs;
+        $params = array();
+        $validParams = array('analyst_data_uuid', 'comment', 'analyst_data_info', 'analyst_data_orgc');
+        foreach ($validParams as $validParam) {
+            if (!empty($this->params['named'][$validParam])) {
+                $params[$validParam] = $this->params['named'][$validParam];
+            }
+        }
+        if (!empty($this->params['named']['searchall'])) {
+            $params['AND']['OR'] = array(
+                'analyst_data_uuid' => $this->params['named']['searchall'],
+                'comment' => $this->params['named']['searchall'],
+                'analyst_data_info' => $this->params['named']['searchall'],
+                'analyst_data_orgc' => $this->params['named']['searchall']
+            );
+        }
+        $this->set('passedArgs', json_encode($passedArgs));
+        $this->set('passedArgsArray', $passedArgsArray);
+        return $this->BlockList->index($this->_isRest(), $params);
+    }
+
+    public function add()
+    {
+        $this->set('action', 'add');
+        return $this->BlockList->add($this->_isRest());
+    }
+
+    public function edit($id)
+    {
+        $this->set('action', 'edit');
+        return $this->BlockList->edit($this->_isRest(), $id);
+    }
+
+    public function delete($id)
+    {
+        if (Validation::uuid($id)) {
+            $entry = $this->AnalystDataBlocklist->find('first', array(
+                'conditions' => array('analyst_data_uuid' => $id)
+            ));
+            if (empty($entry)) {
+                throw new NotFoundException(__('Invalid blocklist entry'));
+            }
+            $id = $entry['AnalystDataBlocklist']['id'];
+        }
+        return $this->BlockList->delete($this->_isRest(), $id);
+    }
+
+    public function massDelete()
+    {
+        if ($this->request->is('post') || $this->request->is('put')) {
+            if (!isset($this->request->data['AnalystDataBlocklist'])) {
+                $this->request->data = array('AnalystDataBlocklist' => $this->request->data);
+            }
+            $ids = $this->request->data['AnalystDataBlocklist']['ids'];
+            $analyst_data_ids = json_decode($ids, true);
+            if (empty($analyst_data_ids)) {
+                throw new NotFoundException(__('Invalid Analyst Data IDs.'));
+            }
+            $result = $this->AnalystDataBlocklist->deleteAll(array('AnalystDataBlocklist.id' => $analyst_data_ids));
+            if ($result) {
+                if ($this->_isRest()) {
+                    return $this->RestResponse->saveSuccessResponse('AnalystDataBlocklist', 'Deleted', $ids, $this->response->type());
+                } else {
+                    $this->Flash->success('Blocklist entry removed');
+                    $this->redirect(array('controller' => 'AnalystDataBlocklist', 'action' => 'index'));
+                }
+            } else {
+                $error = __('Failed to delete Analyst Data from AnalystDataBlocklist. Error: ') . PHP_EOL . h($result);
+                if ($this->_isRest()) {
+                    return $this->RestResponse->saveFailResponse('AnalystDataBlocklist', 'Deleted', false, $error, $this->response->type());
+                } else {
+                    $this->Flash->error($error);
+                    $this->redirect(array('controller' => 'AnalystDataBlocklists', 'action' => 'index'));
+                }
+            }
+        } else {
+            $ids = json_decode($this->request->query('ids'), true);
+            if (empty($ids)) {
+                throw new NotFoundException(__('Invalid analyst data IDs.'));
+
+            }
+            $this->set('analyst_data_ids', $ids);
+        }
+    }
+}

app/Controller/AnalystDataController.php

@@ -0,0 +1,333 @@
+<?php
+App::uses('AppController', 'Controller');
+
+class AnalystDataController extends AppController
+{
+
+    public $components = ['Session', 'RequestHandler'];
+
+    public $paginate = [
+        'limit' => 60,
+        'order' => []
+    ];
+
+    public $uses = [
+        'Opinion',
+        'Note',
+        'Relationship'
+    ];
+
+    private $__valid_types = [
+        'Opinion',
+        'Note',
+        'Relationship'
+    ];
+
+    // public $modelSelection = 'Note';
+
+    private function _setViewElements()
+    {
+        $dropdownData = [];
+        $this->loadModel('Event');
+        $dropdownData['distributionLevels'] = $this->Event->distributionLevels;
+        $this->set('initialDistribution', Configure::read('MISP.default_event_distribution'));
+        $dropdownData['sgs'] = $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1);
+        $dropdownData['valid_targets'] = array_combine($this->AnalystData->valid_targets, $this->AnalystData->valid_targets);
+        $this->set(compact('dropdownData'));
+        $this->set('modelSelection', $this->modelSelection);
+        $this->set('distributionLevels', $this->Event->distributionLevels);
+        App::uses('LanguageRFC5646Tool', 'Tools');
+        $this->set('languageRFC5646', ['' => __('- No language -'), LanguageRFC5646Tool::getLanguages()]);
+    }
+    
+    public function add($type = 'Note', $object_uuid = null, $object_type = null)
+    {
+        $this->__typeSelector($type);
+        if (!empty($object_uuid)) {
+            $this->request->data[$this->modelSelection]['object_uuid'] = $object_uuid;
+        }
+        if (!empty($object_type)) {
+            $this->request->data[$this->modelSelection]['object_type'] = $object_type;
+        }
+        
+        if (empty($this->request->data[$this->modelSelection]['object_type']) && !empty($this->request->data[$this->modelSelection]['object_uuid'])) {
+            $this->request->data[$this->modelSelection]['object_type'] = $this->AnalystData->deduceType($object_uuid);
+        }
+        $params = [];
+        $this->CRUD->add($params);
+        if ($this->restResponsePayload) {
+            return $this->restResponsePayload;
+        }
+        $this->_setViewElements();
+        if ($type == 'Relationship') {
+            $this->set('existingRelations', $this->AnalystData->getExistingRelationships());
+        }
+        $this->set('menuData', array('menuList' => 'analyst_data', 'menuItem' => 'add_' . strtolower($type)));
+        $this->render('add');
+    }
+
+    public function edit($type = 'Note', $id)
+    {
+        if ($type === 'all' && Validation::uuid($id)) {
+            $this->loadModel('AnalystData');
+            $type = $this->AnalystData->deduceType($id);
+        }
+        $this->__typeSelector($type);
+        if (!is_numeric($id) && Validation::uuid($id)) {
+            $id = $this->AnalystData->getIDFromUUID($type, $id);
+        }
+
+        $this->set('id', $id);
+        $conditions = $this->AnalystData->buildConditions($this->Auth->user());
+        $params = [
+            'fields' => $this->AnalystData->getEditableFields(),
+            'conditions' => $conditions,
+            'afterFind' => function(array $analystData): array {
+                $canEdit = $this->ACL->canEditAnalystData($this->Auth->user(), $analystData, $this->modelSelection);
+                if (!$canEdit) {
+                    throw new MethodNotAllowedException(__('You are not authorised to do that.'));
+                }
+                return $analystData;
+            },
+            'beforeSave' => function(array $analystData): array {
+                $analystData[$this->modelSelection]['modified'] = date('Y-m-d H:i:s');
+                return $analystData;
+            }
+        ];
+        $this->CRUD->edit($id, $params);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }
+        $this->_setViewElements();
+        if ($type == 'Relationship') {
+            $this->set('existingRelations', $this->AnalystData->getExistingRelationships());
+        }
+        $this->set('menuData', array('menuList' => 'analyst_data', 'menuItem' => 'edit'));
+        $this->render('add');
+    }
+
+    public function delete($type = 'Note', $id, $hard=true)
+    {
+        if ($type === 'all' && Validation::uuid($id)) {
+            $this->loadModel('AnalystData');
+            $type = $this->AnalystData->deduceType($id);
+        }
+        $this->__typeSelector($type);
+        if (!is_numeric($id) && Validation::uuid($id)) {
+            $id = $this->AnalystData->getIDFromUUID($type, $id);
+        }
+
+        $params = [
+            'afterFind' => function(array $analystData) {
+                $canEdit = $this->ACL->canEditAnalystData($this->Auth->user(), $analystData, $this->modelSelection);
+                if (!$canEdit) {
+                    throw new MethodNotAllowedException(__('You are not authorised to do that.'));
+                }
+                return $analystData;
+            },
+            'afterDelete' => function($deletedAnalystData) use ($hard) {
+                if (empty($hard)) {
+                    return;
+                }
+                $type = $this->AnalystData->deduceAnalystDataType($deletedAnalystData);
+                $info = '- Unsupported analyst type -';
+                if ($type === 'Note') {
+                    $info = $deletedAnalystData[$type]['note'];
+                } else if ($type === 'Opinion') {
+                    $info = sprintf('%s/100 :: %s', $deletedAnalystData[$type]['opinion'], $deletedAnalystData[$type]['comment']);
+                } else if ($type === 'Relationship') {
+                    $info = sprintf('-- %s --> %s :: %s', $deletedAnalystData[$type]['relationship_type'] ?? '[undefined]', $deletedAnalystData[$type]['related_object_type'], $deletedAnalystData[$type]['related_object_uuid']);
+                }
+                $this->AnalystDataBlocklist = ClassRegistry::init('AnalystDataBlocklist');
+                $this->AnalystDataBlocklist->create();
+                if (!empty($deletedAnalystData[$type]['orgc_uuid'])) {
+                    if (!empty($deletedAnalystData[$type]['Orgc'])) {
+                        $orgc = $deletedAnalystData[$type];
+                    } else {
+                        $orgc = $this->Orgc->find('first', array(
+                            'conditions' => ['Orgc.uuid' => $deletedAnalystData[$type]['orgc_uuid']],
+                            'recursive' => -1,
+                            'fields' => ['Orgc.name'],
+                        ));
+                    }
+                } else {
+                    $orgc = ['Orgc' => ['name' => 'MISP']];
+                }
+                $this->AnalystDataBlocklist->save(['analyst_data_uuid' => $deletedAnalystData[$type]['uuid'], 'analyst_data_info' => $info, 'analyst_data_orgc' => $orgc['Orgc']['name']]);
+            }
+        ];
+        $this->CRUD->delete($id, $params);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }
+        
+    }
+
+    public function view($type = 'Note', $id)
+    {
+        if ($type === 'all' && Validation::uuid($id)) {
+            $this->loadModel('AnalystData');
+            $type = $this->AnalystData->getAnalystDataTypeFromUUID($id);
+        }
+        $this->__typeSelector($type);
+        if (!is_numeric($id) && Validation::uuid($id)) {
+            $id = $this->AnalystData->getIDFromUUID($type, $id);
+        }
+
+        if (!$this->IndexFilter->isRest()) {
+            $this->AnalystData->fetchRecursive = true;
+        }
+        $conditions = $this->AnalystData->buildConditions($this->Auth->user());
+        $this->CRUD->view($id, [
+            'conditions' => $conditions,
+            'contain' => ['Org', 'Orgc'],
+            'afterFind' => function(array $analystData) {
+                if (!$this->request->is('ajax')) {
+                    unset($analystData[$this->modelSelection]['_canEdit']);
+                }
+                return $analystData;
+            }
+        ]);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }
+        $this->set('id', $id);
+        $this->loadModel('Event');
+        $this->_setViewElements();
+        $this->set('distributionLevels', $this->Event->distributionLevels);
+        $this->set('shortDist', $this->Event->shortDist);
+        $this->set('menuData', array('menuList' => 'analyst_data', 'menuItem' => 'view'));
+        $this->render('view');
+    }
+
+    public function index($type = 'Note')
+    {
+        $this->__typeSelector($type);
+
+        $conditions = $this->AnalystData->buildConditions($this->Auth->user());
+        $params = [
+            'filters' => ['uuid', 'target_object'],
+            'quickFilters' => ['name'],
+            'conditions' => $conditions,
+            'afterFind' => function(array $data) {
+                foreach ($data as $i => $analystData) {
+                    if (!$this->request->is('ajax')) {
+                        unset($analystData[$this->modelSelection]['_canEdit']);
+                    }
+                }
+                return $data;
+            }
+        ];
+        $this->CRUD->index($params);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }
+        $this->_setViewElements();
+        $this->set('menuData', array('menuList' => 'analyst_data', 'menuItem' => 'index'));
+    }
+
+    public function getRelatedElement($type, $uuid)
+    {
+        $this->__typeSelector('Relationship');
+        $data = $this->AnalystData->getRelatedElement($this->Auth->user(), $type, $uuid);
+        return $this->RestResponse->viewData($data, 'json');
+    }
+
+    public function getChildren($type = 'Note', $uuid, $depth=2)
+    {
+        $this->__typeSelector($type);
+        $data = $this->AnalystData->getChildren($this->Auth->user(), $uuid, $depth);
+        return $this->RestResponse->viewData($data, 'json');
+    }
+
+    public function filterAnalystDataForPush()
+    {
+        if (!$this->request->is('post')) {
+            throw new MethodNotAllowedException(__('This function is only accessible via POST requests.'));
+        }
+
+        $this->loadModel('AnalystData');
+
+        $allIncomingAnalystData = $this->request->data;
+        $allData = $this->AnalystData->filterAnalystDataForPush($allIncomingAnalystData);
+
+        return $this->RestResponse->viewData($allData, $this->response->type());
+    }
+
+    public function indexMinimal()
+    {
+        $this->loadModel('AnalystData');
+        $filters = [];
+        if ($this->request->is('post')) {
+            $filters = $this->request->data;
+        }
+        $options = [];
+        if (!empty($filters['orgc_name'])) {
+            $orgcNames = $filters['orgc_name'];
+            if (!is_array($orgcNames)) {
+                $orgcName = [$orgcNames];
+            }
+            $filterName = 'orgc_uuid';
+            foreach ($orgcNames as $orgcName) {
+                if ($orgcName[0] === '!') {
+                    $orgc = $this->AnalystData->Orgc->fetchOrg(substr($orgcName, 1));
+                    if ($orgc === false) {
+                        continue;
+                    }
+                    $options[]['AND'][] = ["{$filterName} !=" => $orgc['uuid']];
+                } else {
+                    $orgc = $this->AnalystData->Orgc->fetchOrg($orgcName);
+                    if ($orgc === false) {
+                        continue;
+                    }
+                    $options['OR'][] = [$filterName => $orgc['uuid']];
+                }
+            }
+        }
+        $allData = $this->AnalystData->indexMinimal($this->Auth->user(), $options);
+
+        return $this->RestResponse->viewData($allData, $this->response->type());
+    }
+
+    public function pushAnalystData()
+    {
+        if (!$this->Auth->user()['Role']['perm_sync'] || !$this->Auth->user()['Role']['perm_analyst_data']) {
+            throw new MethodNotAllowedException(__('You do not have the permission to do that.'));
+        }
+        if (!$this->_isRest()) {
+            throw new MethodNotAllowedException(__('This action is only accessible via a REST request.'));
+        }
+        if ($this->request->is('post')) {
+            $this->loadModel('AnalystData');
+            $analystData = $this->request->data;
+            $saveResult = $this->AnalystData->captureAnalystData($this->Auth->user(), $analystData);
+            $messageInfo = __('%s imported, %s ignored, %s failed. %s', $saveResult['imported'], $saveResult['ignored'], $saveResult['failed'], !empty($saveResult['errors']) ? implode(', ', $saveResult['errors']) : '');
+            if ($saveResult['success']) {
+                $message = __('Analyst Data imported. ') . $messageInfo;
+                return $this->RestResponse->saveSuccessResponse('AnalystData', 'pushAnalystData', false, $this->response->type(), $message);
+            } else {
+                $message = __('Could not import analyst data. ') . $messageInfo;
+                return $this->RestResponse->saveFailResponse('AnalystData', 'pushAnalystData', false, $message);
+            }
+        }
+    }
+
+    private function __typeSelector($type) {
+        foreach ($this->__valid_types as $vt) {
+            if ($type === $vt) {
+                $this->modelSelection = $vt;
+                $this->loadModel($vt);
+                $this->AnalystData = $this->{$vt};
+                $this->modelClass = $vt;
+                $this->{$vt}->current_user = $this->Auth->user();
+                if (!empty($this->request->data)) {
+                    if (!isset($this->request->data[$type])) {
+                        $this->request->data = [$type => $this->request->data];
+                    }
+                }
+                return $vt;
+            }
+        }
+        throw new MethodNotAllowedException(__('Invalid type.'));
+    }
+}

app/Controller/AppController.php

@@ -33,15 +33,19 @@ class AppController extends Controller
 
     public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
 
-    private $__queryVersion = '157';
-    public $pyMispVersion = '2.4.182';
+    private $__queryVersion = '162';
+    public $pyMispVersion = '2.4.190';
     public $phpmin = '7.2';
     public $phprec = '7.4';
     public $phptoonew = '8.0';
-    public $pythonmin = '3.6';
-    public $pythonrec = '3.7';
     private $isApiAuthed = false;
 
+    /** @var redis */
+    private $redis = null;
+
+    /** @var benchmark_results */
+    private $benchmark_results = null;
+
     public $baseurl = '';
 
     public $restResponsePayload = null;
@@ -59,9 +63,14 @@ class AppController extends Controller
     /** @var ACLComponent */
     public $ACL;
 
+    /** @var BenchmarkComponent */
+    public $Benchmark;
+
     /** @var RestResponseComponent */
     public $RestResponse;
 
+    public $start_time;
+
     public function __construct($request = null, $response = null)
     {
         parent::__construct($request, $response);
@@ -99,15 +108,19 @@ class AppController extends Controller
 
     public function beforeFilter()
     {
-        $controller = $this->request->params['controller'];
-        $action = $this->request->params['action'];
-        if (empty($this->Session->read('creation_timestamp'))) {
-            $this->Session->write('creation_timestamp', time());
-        }
         if (Configure::read('MISP.system_setting_db')) {
             App::uses('SystemSetting', 'Model');
             SystemSetting::setGlobalSetting();
         }
+        
+        $this->User = ClassRegistry::init('User');
+        if (Configure::read('Plugin.Benchmarking_enable')) {
+            App::uses('BenchmarkTool', 'Tools');
+            $this->Benchmark = new BenchmarkTool($this->User);
+            $this->start_time = $this->Benchmark->startBenchmark();
+        }
+        $controller = $this->request->params['controller'];
+        $action = $this->request->params['action'];
 
         $this->_setupBaseurl();
         $this->Auth->loginRedirect = $this->baseurl . '/users/routeafterlogin';
@@ -151,8 +164,6 @@ class AppController extends Controller
             Configure::write('Config.language', 'eng');
         }
 
-        $this->User = ClassRegistry::init('User');
-
         if (!empty($this->request->params['named']['disable_background_processing'])) {
             Configure::write('MISP.background_jobs', 0);
         }
@@ -230,10 +241,15 @@ class AppController extends Controller
         ) {
             // REST authentication
             if ($this->_isRest() || $this->_isAutomation()) {
+                
                 // disable CSRF for REST access
                 $this->Security->csrfCheck = false;
                 $loginByAuthKeyResult = $this->__loginByAuthKey();
                 if ($loginByAuthKeyResult === false || $this->Auth->user() === null) {
+                    if ($this->IndexFilter->isXhr()) {
+                        throw new ForbiddenException('Authentication failed.');
+                    }
+
                     if ($loginByAuthKeyResult === null) {
                         $this->loadModel('Log');
                         $this->Log->createLogEntry('SYSTEM', 'auth_fail', 'User', 0, "Failed API authentication. No authkey was provided.");
@@ -306,6 +322,7 @@ class AppController extends Controller
             $this->set('isAclTagger', $role['perm_tagger']);
             $this->set('isAclGalaxyEditor', !empty($role['perm_galaxy_editor']));
             $this->set('isAclSighting', $role['perm_sighting'] ?? false);
+            $this->set('isAclAnalystDataCreator', $role['perm_analyst_data'] ?? false);
             $this->set('aclComponent', $this->ACL);
             $this->userRole = $role;
 
@@ -362,6 +379,10 @@ class AppController extends Controller
                 }
             }
         }
+        if (Configure::read('MISP.enable_automatic_garbage_collection') && mt_rand(1,100) % 100 == 0) {
+            $this->loadModel('AdminSetting');
+            $this->AdminSetting->garbageCollect();
+        }
     }
 
     public function beforeRender()
@@ -440,8 +461,7 @@ class AppController extends Controller
                     // User found in the db, add the user info to the session
                     if (Configure::read('MISP.log_auth')) {
                         $this->loadModel('Log');
-                        $this->UserLoginProfile = ClassRegistry::init('UserLoginProfile');
-                        $change = $this->UserLoginProfile->_getUserProfile();
+                        $change = $this->User->UserLoginProfile->_getUserProfile();
                         $change['http_method'] = $_SERVER['REQUEST_METHOD'];
                         $change['target'] = $this->request->here;
                         $this->Log->createLogEntry(
@@ -541,13 +561,18 @@ class AppController extends Controller
             }
         }
 
+        $sessionCreationTime = $this->Session->read('creation_timestamp');
+        if (empty($sessionCreationTime)) {
+            $sessionCreationTime = $_SERVER['REQUEST_TIME'] ?? time();
+            $this->Session->write('creation_timestamp', $sessionCreationTime);
+        }
+
         // kill existing sessions for a user if the admin/instance decides so
         // exclude API authentication as it doesn't make sense
-        if (!$this->isApiAuthed && $this->User->checkForSessionDestruction($user['id'])) {
+        if (!$this->isApiAuthed && $this->User->checkForSessionDestruction($user['id'], $sessionCreationTime)) {
             $this->Auth->logout();
             $this->Session->destroy();
-            $message = __('User deauthenticated on administrator request. Please reauthenticate.');
-            $this->Flash->warning($message);
+            $this->Flash->warning(__('User deauthenticated on administrator request. Please reauthenticate.'));
             $this->_redirectToLogin();
             return false;
         }
@@ -564,8 +589,7 @@ class AppController extends Controller
         if ($user['disabled'] || (isset($user['logged_by_authkey']) && $user['logged_by_authkey']) && !$this->User->checkIfUserIsValid($user)) {
             if ($this->_shouldLog('disabled:' . $user['id'])) {
                 $this->Log = ClassRegistry::init('Log');
-                $this->UserLoginProfile = ClassRegistry::init('UserLoginProfile');
-                $change = $this->UserLoginProfile->_getUserProfile();
+                $change = $this->User->UserLoginProfile->_getUserProfile();
                 $this->Log->createLogEntry($user, 'auth_fail', 'User', $user['id'], 'Login attempt by disabled user.', json_encode($change));
             }
 
@@ -585,9 +609,9 @@ class AppController extends Controller
             if ($user['authkey_expiration'] < $time) {
                 if ($this->_shouldLog('expired:' . $user['authkey_id'])) {
                     $this->Log = ClassRegistry::init('Log');
-                    $this->UserLoginProfile = ClassRegistry::init('UserLoginProfile');
-                    $change = $this->UserLoginProfile->_getUserProfile();
-                    $this->Log->createLogEntry($user, 'auth_fail', 'User', $user['id'], "Login attempt by expired auth key {$user['authkey_id']}.", json_encode($change));                }
+                    $change = $this->User->UserLoginProfile->_getUserProfile();
+                    $this->Log->createLogEntry($user, 'auth_fail', 'User', $user['id'], "Login attempt by expired auth key {$user['authkey_id']}.", json_encode($change));
+                }
                 $this->Auth->logout();
                 throw new ForbiddenException('Auth key is expired');
             }
@@ -596,7 +620,7 @@ class AppController extends Controller
         if (!empty($user['allowed_ips'])) {
             App::uses('CidrTool', 'Tools');
             $cidrTool = new CidrTool($user['allowed_ips']);
-            $remoteIp = $this->_remoteIp();
+            $remoteIp = $this->User->_remoteIp();
             if ($remoteIp === null) {
                 $this->Auth->logout();
                 throw new ForbiddenException('Auth key is limited to IP address, but IP address not found');
@@ -604,9 +628,9 @@ class AppController extends Controller
             if (!$cidrTool->contains($remoteIp)) {
                 if ($this->_shouldLog('not_allowed_ip:' . $user['authkey_id'] . ':' . $remoteIp)) {
                     $this->Log = ClassRegistry::init('Log');
-                    $this->UserLoginProfile = ClassRegistry::init('UserLoginProfile');
-                    $change = $this->UserLoginProfile->_getUserProfile();
-                    $this->Log->createLogEntry($user, 'auth_fail', 'User', $user['id'], "Login attempt from not allowed IP address {$remoteIp} for auth key {$user['authkey_id']}.", json_encode($change));                }
+                    $change = $this->User->UserLoginProfile->_getUserProfile();
+                    $this->Log->createLogEntry($user, 'auth_fail', 'User', $user['id'], "Login attempt from not allowed IP address {$remoteIp} for auth key {$user['authkey_id']}.", json_encode($change));
+                }
                 $this->Auth->logout();
                 throw new ForbiddenException('It is not possible to use this Auth key from your IP address');
             }
@@ -625,21 +649,21 @@ class AppController extends Controller
         }
 
         // Check if user accepted terms and conditions
-        if (!$user['termsaccepted'] && !empty(Configure::read('MISP.terms_file')) && !$this->_isControllerAction(['users' => ['terms', 'logout', 'login', 'downloadTerms']])) {
+        if (!$user['termsaccepted'] && !empty(Configure::read('MISP.terms_file')) && !$this->_isControllerAction(['users' => ['terms', 'logout', 'login', 'downloadTerms', 'totp_new', 'email_otp']])) {
             //if ($this->_isRest()) throw new MethodNotAllowedException('You have not accepted the terms of use yet, please log in via the web interface and accept them.');
             $this->redirect(array('controller' => 'users', 'action' => 'terms', 'admin' => false));
             return false;
         }
 
         // Check if user must change password
-        if ($user['change_pw'] && !$this->_isControllerAction(['users' => ['terms', 'change_pw', 'logout', 'login']])) {
+        if ($user['change_pw'] && !$this->_isControllerAction(['users' => ['terms', 'change_pw', 'logout', 'login', 'totp_new', 'email_otp']])) {
             //if ($this->_isRest()) throw new MethodNotAllowedException('Your user account is expecting a password change, please log in via the web interface and change it before proceeding.');
             $this->redirect(array('controller' => 'users', 'action' => 'change_pw', 'admin' => false));
             return false;
         }
 
         // Check if user must read news
-        if (!$this->_isControllerAction(['news' => ['index'], 'users' => ['terms', 'change_pw', 'login', 'logout']])) {
+        if (!$this->_isControllerAction(['news' => ['index'], 'users' => ['terms', 'change_pw', 'login', 'logout', 'totp_new', 'email_otp']])) {
             $this->loadModel('News');
             $latestNewsCreated = $this->News->latestNewsTimestamp();
             if ($latestNewsCreated && $user['newsread'] < $latestNewsCreated) {
@@ -689,7 +713,7 @@ class AppController extends Controller
             return;
         }
 
-        $remoteAddress = $this->_remoteIp();
+        $remoteAddress = $this->User->_remoteIp();
 
         $pipe = $redis->pipeline();
         // keep for 30 days
@@ -732,7 +756,7 @@ class AppController extends Controller
             $includeRequestBody = !empty(Configure::read('MISP.log_paranoid_include_post_body')) || $userMonitoringEnabled;
             /** @var AccessLog $accessLog */
             $accessLog = ClassRegistry::init('AccessLog');
-            $accessLog->logRequest($user, $this->_remoteIp(), $this->request, $includeRequestBody);
+            $accessLog->logRequest($user, $this->User->_remoteIp(), $this->request, $includeRequestBody);
         }
 
         if (
@@ -823,33 +847,53 @@ class AppController extends Controller
 
     private function __rateLimitCheck(array $user)
     {
-        $info = array();
         $rateLimitCheck = $this->RateLimit->check(
             $user,
             $this->request->params['controller'],
-            $this->request->action,
-            $info,
-            $this->response->type()
+            $this->request->params['action'],
         );
-        if (!empty($info)) {
-            $this->RestResponse->setHeader('X-Rate-Limit-Limit', $info['limit']);
-            $this->RestResponse->setHeader('X-Rate-Limit-Remaining', $info['remaining']);
-            $this->RestResponse->setHeader('X-Rate-Limit-Reset', $info['reset']);
+
+        if ($rateLimitCheck) {
+            $headers = [
+                'X-Rate-Limit-Limit' => $rateLimitCheck['limit'],
+                'X-Rate-Limit-Remaining' => $rateLimitCheck['remaining'],
+                'X-Rate-Limit-Reset' => $rateLimitCheck['reset'],
+            ];
+
+            if ($rateLimitCheck['exceeded']) {
+                $response = $this->RestResponse->throwException(
+                    429,
+                    __('Rate limit exceeded.'),
+                    '/' . $this->request->params['controller'] . '/' . $this->request->params['action'],
+                    false,
+                    false,
+                    $headers
+                );
+                $response->send();
+                $this->_stop();
+            } else {
+                $this->RestResponse->headers = array_merge($this->RestResponse->headers, $headers);
+            }
         }
-        if ($rateLimitCheck !== true) {
-            $this->response->header('X-Rate-Limit-Limit', $info['limit']);
-            $this->response->header('X-Rate-Limit-Remaining', $info['remaining']);
-            $this->response->header('X-Rate-Limit-Reset', $info['reset']);
-            $this->response->body($rateLimitCheck);
-            $this->response->statusCode(429);
-            $this->response->send();
-            $this->_stop();
-        }
-        return true;
     }
 
     public function afterFilter()
     {
+        // benchmarking
+        if (Configure::read('Plugin.Benchmarking_enable') && isset($this->Benchmark)) {
+            $this->Benchmark->stopBenchmark([
+                'user' => $this->Auth->user('id'),
+                'controller' => $this->request->params['controller'],
+                'action' => $this->request->params['action'],
+                'start_time' => $this->start_time
+            ]);
+
+            //if ($redis && !$redis->exists('misp:auth_fail_throttling:' . $key)) {
+                //$redis->setex('misp:auth_fail_throttling:' . $key, 3600, 1);
+                //return true;
+            //}
+            
+        }
         if ($this->isApiAuthed && $this->_isRest() && !Configure::read('Security.authkey_keep_session')) {
             $this->Session->destroy();
         }
@@ -878,7 +922,7 @@ class AppController extends Controller
             ConnectionManager::create('default', $db->config);
         }
         $dataSource = $dataSourceConfig['datasource'];
-        if (!in_array($dataSource, ['Database/Mysql', 'Database/Postgres', 'Database/MysqlObserver', 'Database/MysqlExtended'], true)) {
+        if (!in_array($dataSource, ['Database/Mysql', 'Database/Postgres', 'Database/MysqlObserver', 'Database/MysqlExtended', 'Database/MysqlObserverExtended'], true)) {
             throw new Exception('Datasource not supported: ' . $dataSource);
         }
     }
@@ -948,31 +992,6 @@ class AppController extends Controller
         return $this->userRole['perm_site_admin'];
     }
 
-    protected function _getApiAuthUser($key, &$exception)
-    {
-        if (strlen($key) === 40) {
-            // check if the key is valid -> search for users based on key
-            $user = $this->_checkAuthUser($key);
-            if (!$user) {
-                $exception = $this->RestResponse->throwException(
-                    401,
-                    __('This authentication key is not authorized to be used for exports. Contact your administrator.')
-                );
-                return false;
-            }
-        } else {
-            $user = $this->Auth->user();
-            if (!$user) {
-                $exception = $this->RestResponse->throwException(
-                    401,
-                    __('You have to be logged in to do that.')
-                );
-                return false;
-            }
-        }
-        return $user;
-    }
-
     private function __captureParam($data, $param, $value)
     {
         if ($this->modelClass->checkParam($param)) {
@@ -990,6 +1009,14 @@ class AppController extends Controller
      */
     protected function _harvestParameters($options, &$exception = null, $data = [])
     {
+        if (!empty($options['paramArray'])) {
+            if (!in_array('page', $options['paramArray'])) {
+                $options['paramArray'][] = 'page';
+            }
+            if (!in_array('limit', $options['paramArray'])) {
+                $options['paramArray'][] = 'limit';
+            }
+        }
         $request = $options['request'] ?? $this->request;
         if ($request->is('post')) {
             if (empty($request->data)) {
@@ -1012,7 +1039,19 @@ class AppController extends Controller
                     $data = array_merge($data, $temp);
                 } else {
                     foreach ($options['paramArray'] as $param) {
-                        if (isset($temp[$param])) {
+                        if (substr($param, -1) == '*') {
+                            $root = substr($param, 0, strlen($param)-1);
+                            foreach ($temp as $existingParamKey => $v) {
+                                $leftover = substr($existingParamKey, strlen($param)-1);
+                                if (
+                                    $root == substr($existingParamKey, 0, strlen($root)) &&
+                                    preg_match('/^[\w_-. ]+$/', $leftover) == 1
+                                ) {
+                                    $data[$existingParamKey] = $temp[$existingParamKey];
+                                    break;
+                                }
+                            }
+                        } else if (isset($temp[$param])) {
                             $data[$param] = $temp[$param];
                         }
                     }
@@ -1080,6 +1119,23 @@ class AppController extends Controller
 
     protected function _checkAuthUser($authkey)
     {
+        if (Configure::read('Security.api_key_quick_lookup')) {
+            $redis = RedisTool::init();
+            if (file_exists(APP . 'Config/hmac_key.php')) {
+                include(APP . 'Config/hmac_key.php');
+                $hashed_authkey = hash_hmac('sha512', $authkey, $hmac_key);
+                if ($redis && $redis->exists('misp:fast_authkey_lookup:' . $hashed_authkey)) {
+                    $user = RedisTool::deserialize($redis->get('misp:fast_authkey_lookup:' . $hashed_authkey));
+                    if ($user) {
+                        return $user;
+                    }
+                }
+            } else {
+                App::uses('RandomTool', 'Tools');
+                $hmac_key = RandomTool::random_str(true, 40);
+                file_put_contents(APP . 'Config/hmac_key.php', sprintf('<?php%s$hmac_key = \'%s\';', PHP_EOL, $hmac_key));
+            }
+        }
         if (Configure::read('Security.advanced_authkeys')) {
             $user = $this->User->AuthKey->getAuthUserByAuthKey($authkey);
         } else {
@@ -1093,6 +1149,13 @@ class AppController extends Controller
             return false;
         }
         $user['logged_by_authkey'] = true;
+        if (Configure::read('Security.api_key_quick_lookup') && !empty($hmac_key) && $redis) {
+            $expiration = Configure::read('Security.api_key_quick_lookup_expiration') ? Configure::read('Security.api_key_quick_lookup_expiration') : 180;
+            if ($redis) {
+                $hashed_authkey = hash_hmac('sha512', $authkey, $hmac_key);
+                $redis->setex('misp:fast_authkey_lookup:' . $hashed_authkey, $expiration, RedisTool::serialize($user));
+            }
+        }
         return $user;
     }
 
@@ -1130,34 +1193,32 @@ class AppController extends Controller
                 $headerNamespace = '';
             }
             if (isset($server[$headerNamespace . $header]) && !empty($server[$headerNamespace . $header])) {
-                if (Configure::read('Plugin.CustomAuth_only_allow_source') && Configure::read('Plugin.CustomAuth_only_allow_source') !== $this->_remoteIp()) {
+                if (Configure::read('Plugin.CustomAuth_only_allow_source') && Configure::read('Plugin.CustomAuth_only_allow_source') !== $this->User->_remoteIp()) {
                     $this->Log = ClassRegistry::init('Log');
                     $this->Log->createLogEntry(
                         'SYSTEM',
                         'auth_fail',
                         'User',
                         0,
-                        'Failed authentication using external key (' . trim($server[$headerNamespace . $header]) . ') - the user has not arrived from the expected address. Instead the request came from: ' . $this->_remoteIp(),
+                        'Failed authentication using external key (' . trim($server[$headerNamespace . $header]) . ') - the user has not arrived from the expected address. Instead the request came from: ' . $this->User->_remoteIp(),
                         null);
                     $this->__preAuthException($authName . ' authentication failed. Contact your MISP support for additional information at: ' . Configure::read('MISP.contact'));
                 }
-                $temp = $this->_checkExternalAuthUser($server[$headerNamespace . $header]);
-                $user['User'] = $temp;
-                if ($user['User']) {
-                    $this->User->updateLoginTimes($user['User']);
+                $user = $this->_checkExternalAuthUser($server[$headerNamespace . $header]);
+                if ($user) {
+                    $this->User->updateLoginTimes($user);
                     //$this->Session->renew();
-                    $this->Session->write(AuthComponent::$sessionKey, $user['User']);
+                    $this->Session->write(AuthComponent::$sessionKey, $user);
                     if (Configure::read('MISP.log_auth')) {
                         $this->Log = ClassRegistry::init('Log');
-                        $this->UserLoginProfile = ClassRegistry::init('UserLoginProfile');
-                        $change = $this->UserLoginProfile->_getUserProfile();
+                        $change = $this->User->UserLoginProfile->_getUserProfile();
                         $change['http_method'] = $_SERVER['REQUEST_METHOD'];
                         $change['target'] = $this->request->here;
                         $this->Log->createLogEntry(
                             $user,
                             'auth',
                             'User',
-                            $user['User']['id'],
+                            $user['id'],
                             'Successful authentication using ' . $authName . ' key',
                             json_encode($change));
                     }
@@ -1166,8 +1227,7 @@ class AppController extends Controller
                     // User not authenticated correctly
                     // reset the session information
                     $this->Log = ClassRegistry::init('Log');
-                    $this->UserLoginProfile = ClassRegistry::init('UserLoginProfile');
-                    $change = $this->UserLoginProfile->_getUserProfile();
+                    $change = $this->User->UserLoginProfile->_getUserProfile();
                     $this->Log->createLogEntry(
                         'SYSTEM',
                         'auth_fail',
@@ -1299,7 +1359,7 @@ class AppController extends Controller
         $exception = false;
         $filters = $this->_harvestParameters($filterData, $exception, $this->_legacyParams);
         if (empty($filters) && $this->request->is('get')) {
-            throw new InvalidArgumentException(__('Restsearch queries using GET and no parameters are not allowed. If you have passed parameters via a JSON body, make sure you use POST requests.'));
+            throw new BadRequestException(__('Restsearch queries using GET and no parameters are not allowed. If you have passed parameters via a JSON body, make sure you use POST requests.'));
         }
         if (empty($filters['returnFormat'])) {
             $filters['returnFormat'] = 'json';
@@ -1308,13 +1368,8 @@ class AppController extends Controller
         if ($filters === false) {
             return $exception;
         }
-        $key = empty($filters['key']) ? $filters['returnFormat'] : $filters['key'];
-        $user = $this->_getApiAuthUser($key, $exception);
-        if ($user === false) {
-            return $exception;
-        }
 
-        session_write_close(); // Rest search can be longer, so close session to allow concurrent requests
+        $user = $this->_closeSession();
 
         if (isset($filters['returnFormat'])) {
             $returnFormat = $filters['returnFormat'];
@@ -1434,14 +1489,13 @@ class AppController extends Controller
 
     /**
      * @return string|null
+     * @deprecated Use User::_remoteIp() instead
      */
     protected function _remoteIp()
     {
-        $ipHeader = Configure::read('MISP.log_client_ip_header') ?: 'REMOTE_ADDR';
-        return isset($_SERVER[$ipHeader]) ? trim($_SERVER[$ipHeader]) : $_SERVER['REMOTE_ADDR'];
+        return $this->User->_remoteIp();
     }
 
-
     /**
      * @param string $key
      * @return bool Returns true if the same log defined by $key was not stored in last hour
@@ -1502,10 +1556,20 @@ class AppController extends Controller
      * Close session without writing changes to them and return current user.
      * @return array
      */
-    protected function _closeSession()
+    protected function _closeSession($saveSession = false)
     {
         $user = $this->Auth->user();
-        session_abort();
+
+        // Hack to store user info in static AuthComponent::$_user variable to avoid starting session again by calling
+        // $this->Auth->user()
+        AuthComponent::$sessionKey = null;
+        $this->Auth->login($user);
+
+        if ($saveSession) {
+            @session_write_close();
+        } else {
+            session_abort();
+        }
         return $user;
     }
 

app/Controller/AttributesController.php

@@ -33,38 +33,16 @@ class AttributesController extends AppController
     {
         parent::beforeFilter();
 
-        $this->Auth->allow('restSearch');
-        $this->Auth->allow('returnAttributes');
-        $this->Auth->allow('downloadAttachment');
-        $this->Auth->allow('text');
-        $this->Auth->allow('rpz');
-        $this->Auth->allow('bro');
-
         // permit reuse of CSRF tokens on the search page.
         if ('search' === $this->request->params['action']) {
             $this->Security->csrfCheck = false;
         }
         $this->Security->unlockedActions[] = 'getMassEditForm';
         $this->Security->unlockedActions[] = 'search';
+
         if ($this->request->action === 'add_attachment') {
             $this->Security->unlockedFields = array('values');
-        }
-
-        // convert uuid to id if present in the url and overwrite id field
-        if (isset($this->request->params->query['uuid'])) {
-            $params = array(
-                    'conditions' => array('Attribute.uuid' => $this->request->params->query['uuid']),
-                    'recursive' => 0,
-                    'fields' => 'Attribute.id'
-                    );
-            $result = $this->Attribute->find('first', $params);
-            if (isset($result['Attribute']) && isset($result['Attribute']['id'])) {
-                $id = $result['Attribute']['id'];
-                $this->params->addParams(array('pass' => array($id))); // FIXME find better way to change id variable if uuid is found. params->url and params->here is not modified accordingly now
-            }
-        }
-
-        if ($this->request->action === 'viewPicture') {
+        } elseif ($this->request->action === 'viewPicture') {
             $this->Security->doNotGenerateToken = true;
         }
     }
@@ -793,12 +771,14 @@ class AttributesController extends AppController
                 $result = $this->Attribute->save($this->request->data, array('fieldList' => Attribute::EDITABLE_FIELDS));
                 if ($result) {
                     $this->Attribute->AttributeTag->handleAttributeTags($this->Auth->user(), $this->request->data['Attribute'], $attribute['Event']['id'], $capture=true);
+                    $this->Attribute->Event->captureAnalystData($this->Auth->user(), $this->request->data['Attribute'], 'Attribute', $existingAttribute['Attribute']['uuid']);
                 }
                 $this->Attribute->Object->updateTimestamp($existingAttribute['Attribute']['object_id']);
             } else {
                 $result = $this->Attribute->save($this->request->data, array('fieldList' => Attribute::EDITABLE_FIELDS));
                 if ($result) {
                     $this->Attribute->AttributeTag->handleAttributeTags($this->Auth->user(), $this->request->data['Attribute'], $attribute['Event']['id'], $capture=true);
+                    $this->Attribute->Event->captureAnalystData($this->Auth->user(), $this->request->data['Attribute'], 'Attribute', $existingAttribute['Attribute']['uuid']);
                 }
                 if ($this->request->is('ajax')) {
                     $this->autoRender = false;
@@ -1576,10 +1556,11 @@ class AttributesController extends AppController
 
             // Force index for performance reasons see #3321
             if (isset($filters['value'])) {
-                $this->paginate['forceIndexHint'] = '(value1, value2)';
+                $this->paginate['forceIndexHint'] = 'value1, value2';
             }
 
             $this->paginate['conditions'] = $params['conditions'];
+            $this->paginate['ignoreIndexHint'] = 'deleted';
             $attributes = $this->paginate();
             $this->Attribute->attachTagsToAttributes($attributes, ['includeAllTags' => true]);
 
@@ -1917,7 +1898,7 @@ class AttributesController extends AppController
     public function reportValidationIssuesAttributes($eventId = false)
     {
         // search for validation problems in the attributes
-        $this->set('result', $this->Attribute->reportValidationIssuesAttributes($eventId));
+        $this->set('result', iterator_to_array($this->Attribute->reportValidationIssuesAttributes($eventId)));
     }
 
     public function generateCorrelation()
@@ -2671,12 +2652,15 @@ class AttributesController extends AppController
             $fails = 0;
             $this->Taxonomy = ClassRegistry::init('Taxonomy');
             foreach ($idList as $id) {
+                $conditions = $this->__idToConditions($id);
+                $conditions['Attribute.deleted'] = 0;
                 $attribute = $this->Attribute->fetchAttributeSimple($this->Auth->user(), [
-                    'conditions' => array('Attribute.id' => $id, 'Attribute.deleted' => 0),
+                    'conditions' => $conditions,
                 ]);
                 if (empty($attribute)) {
                     throw new NotFoundException(__('Invalid attribute'));
                 }
+                $id = $attribute['Attribute']['id'];
                 if (!$this->__canModifyTag($attribute, $local)) {
                     $fails++;
                     continue;
@@ -3035,4 +3019,35 @@ class AttributesController extends AppController
             ];
         }
     }
+
+    public function viewAnalystData($id, $seed = null)
+    {
+        $this->Attribute->includeAnalystDataRecursive = true;
+        $attribute = $this->Attribute->fetchAttributes(
+            $this->Auth->user(),
+            [
+                'conditions' => $this->__idToConditions($id),
+                'flatten' => true
+            ]
+        );
+        if(empty($attribute)) {
+            throw new NotFoundException(__('Invalid Attribute.'));
+        } else {
+            $attribute[0]['Attribute'] = array_merge_recursive($attribute[0]['Attribute'], $this->Attribute->attachAnalystData($attribute[0]['Attribute']));
+        }
+        if ($this->_isRest()) {
+            $validFields = ['Note', 'Opinion', 'Relationship'];
+            $results = [];
+            foreach ($validFields as $field) {
+                if (!empty($attribute[0]['Attribute'][$field])) {
+                    $results[$field] = $attribute[0]['Attribute'][$field];
+                }
+            }
+            return $this->RestResponse->viewData($results, $this->response->type());
+        }
+        $this->layout = null;
+        $this->set('shortDist', $this->Attribute->shortDist);
+        $this->set('object', $attribute[0]['Attribute']);
+        $this->set('seed', $seed);
+    }
 }

app/Controller/AuditLogsController.php

@@ -91,21 +91,6 @@ class AuditLogsController extends AppController
         ];
     }
 
-    private function __applyAuditACL(array $user)
-    {
-        $acl = [];
-        if (empty($user['Role']['perm_site_admin'])) {
-            if (!empty($user['Role']['perm_admin'])) {
-                // ORG admins can see their own org info
-                $acl = ['AuditLog.org_id' => $user['org_id']];
-            } else {
-                // users can see their own info
-                $acl = ['AuditLog.user_id' => $user['id']];
-            }
-        }
-        return $acl;
-    }
-
     public function admin_index()
     {
         $this->paginate['fields'][] = 'ip';
@@ -134,7 +119,8 @@ class AuditLogsController extends AppController
         ]);
 
         $this->paginate['conditions'] = $this->__searchConditions($params);
-        $acl = $this->__applyAuditACL($this->Auth->user());
+        $user = $this->Auth->user();
+        $acl = $this->__applyAuditAcl($user);
         if ($acl) {
             $this->paginate['conditions']['AND'][] = $acl;
         }
@@ -144,7 +130,7 @@ class AuditLogsController extends AppController
             return $this->RestResponse->viewData($list, 'json');
         }
 
-        $list = $this->__appendModelLinks($list);
+        $list = $this->__appendModelLinks($user, $list);
         foreach ($list as $k => $item) {
             $list[$k]['AuditLog']['action_human'] =  $this->actions[$item['AuditLog']['action']];
         }
@@ -222,13 +208,19 @@ class AuditLogsController extends AppController
 
     public function fullChange($id)
     {
+        $acl = $this->__applyAuditAcl($this->Auth->user());
         $log = $this->AuditLog->find('first', [
-            'conditions' => ['id' => $id],
+            'conditions' => [
+                'AND' => [
+                    $acl,
+                    'id' => $id
+                ]
+            ],
             'recursive' => -1,
             'fields' => ['change', 'action'],
         ]);
         if (empty($log)) {
-            throw new Exception('Log not found.');
+            throw new NotFoundException('Log not found.');
         }
         $this->set('log', $log);
     }
@@ -246,6 +238,21 @@ class AuditLogsController extends AppController
         return $this->RestResponse->viewData($data, $this->response->type());
     }
 
+    private function __applyAuditAcl(array $user)
+    {
+        $acl = [];
+        if (empty($user['Role']['perm_site_admin'])) {
+            if (!empty($user['Role']['perm_admin'])) {
+                // ORG admins can see their own org info
+                $acl = ['AuditLog.org_id' => $user['org_id']];
+            } else {
+                // users can see their own info
+                $acl = ['AuditLog.user_id' => $user['id']];
+            }
+        }
+        return $acl;
+    }
+
     /**
      * @return array
      */
@@ -435,10 +442,11 @@ class AuditLogsController extends AppController
 
     /**
      * Generate link to model view if exists and use has permission to access it.
+     * @param array $user
      * @param array $auditLogs
      * @return array
      */
-    private function __appendModelLinks(array $auditLogs)
+    private function __appendModelLinks(array $user, array $auditLogs)
     {
         $models = [];
         foreach ($auditLogs as $auditLog) {
@@ -449,7 +457,7 @@ class AuditLogsController extends AppController
             }
         }
 
-        $eventIds = isset($models['Event']) ? $models['Event'] : [];
+        $eventIds = $models['Event'] ?? [];
 
         if (isset($models['ObjectReference'])) {
             $this->loadModel('ObjectReference');
@@ -461,11 +469,11 @@ class AuditLogsController extends AppController
 
         if (isset($models['Object']) || isset($objectReferences)) {
             $objectIds = array_unique(array_merge(
-                isset($models['Object']) ? $models['Object'] : [],
+                $models['Object'] ?? [],
                 isset($objectReferences) ? array_values($objectReferences) : []
             ));
             $this->loadModel('MispObject');
-            $conditions = $this->MispObject->buildConditions($this->Auth->user());
+            $conditions = $this->MispObject->buildConditions($user);
             $conditions['Object.id'] = $objectIds;
             $objects = $this->MispObject->find('all', [
                 'conditions' => $conditions,
@@ -473,22 +481,22 @@ class AuditLogsController extends AppController
                 'fields' => ['Object.id', 'Object.event_id', 'Object.uuid', 'Object.deleted'],
             ]);
             $objects = array_column(array_column($objects, 'Object'), null, 'id');
-            $eventIds = array_merge($eventIds, array_column($objects, 'event_id'));
+            array_push($eventIds, ...array_column($objects, 'event_id'));
         }
 
         if (isset($models['Attribute'])) {
             $this->loadModel('Attribute');
-            $attributes = $this->Attribute->fetchAttributesSimple($this->Auth->user(), [
+            $attributes = $this->Attribute->fetchAttributesSimple($user, [
                 'conditions' => ['Attribute.id' => array_unique($models['Attribute'])],
                 'fields' => ['Attribute.id', 'Attribute.event_id', 'Attribute.uuid', 'Attribute.deleted'],
             ]);
             $attributes = array_column(array_column($attributes, 'Attribute'), null, 'id');
-            $eventIds = array_merge($eventIds, array_column($attributes, 'event_id'));
+            array_push($eventIds, ...array_column($attributes, 'event_id'));
         }
 
         if (isset($models['ShadowAttribute'])) {
             $this->loadModel('ShadowAttribute');
-            $conditions = $this->ShadowAttribute->buildConditions($this->Auth->user());
+            $conditions = $this->ShadowAttribute->buildConditions($user);
             $conditions['AND'][] = ['ShadowAttribute.id' => array_unique($models['ShadowAttribute'])];
             $shadowAttributes = $this->ShadowAttribute->find('all', [
                 'conditions' => $conditions,
@@ -496,12 +504,12 @@ class AuditLogsController extends AppController
                 'contain' => ['Event', 'Attribute'],
             ]);
             $shadowAttributes = array_column(array_column($shadowAttributes, 'ShadowAttribute'), null, 'id');
-            $eventIds = array_merge($eventIds, array_column($shadowAttributes, 'event_id'));
+            array_push($eventIds, ...array_column($shadowAttributes, 'event_id'));
         }
 
         if (!empty($eventIds)) {
             $this->loadModel('Event');
-            $conditions = $this->Event->createEventConditions($this->Auth->user());
+            $conditions = $this->Event->createEventConditions($user);
             $conditions['Event.id'] = array_unique($eventIds);
             $events = $this->Event->find('list', [
                 'conditions' => $conditions,

app/Controller/BenchmarksController.php

@@ -0,0 +1,123 @@
+<?php
+App::uses('AppController', 'Controller');
+
+class BenchmarksController extends AppController
+{
+    public $components = array('Session', 'RequestHandler');
+
+    public $paginate = [
+        'limit' => 60,
+        'maxLimit' => 9999,
+
+    ];
+
+    public function beforeFilter()
+    {
+        parent::beforeFilter();
+    }
+
+    public function index()
+    {
+        $this->set('menuData', ['menuList' => 'admin', 'menuItem' => 'index']);
+        $this->loadModel('User');
+        App::uses('BenchmarkTool', 'Tools');
+        $this->Benchmark = new BenchmarkTool($this->User);
+        $passedArgs = $this->passedArgs;
+        $this->paginate['order'] = 'value';
+        $defaults = [
+            'days' => null,
+            'average' => false,
+            'aggregate' => false,
+            'scope' => null,
+            'field' => null,
+            'key' => null,
+            'quickFilter' => null
+        ];
+        $filters = $this->IndexFilter->harvestParameters(array_keys($defaults));
+        foreach ($defaults as $key => $value) {
+            if (!isset($filters[$key])) {
+                $filters[$key] = $defaults[$key];
+            }
+        }
+        $temp = $this->Benchmark->getAllTopLists(
+            $filters['days'] ?? null,
+            $filters['limit'] ?? 100,
+            $filters['average'] ?? null,
+            $filters['aggregate'] ?? null
+        );
+        $settings = $this->Benchmark->getSettings();
+        $units = $this->Benchmark->getUnits();
+        $this->set('settings', $settings);
+        $data = [];
+        $userLookup = [];
+        foreach ($temp as $scope => $t) {
+            if (!empty($filters['scope']) && $filters['scope'] !== 'all' && $scope !== $filters['scope']) {
+                continue;
+            }
+            foreach ($t as $field => $t2) {
+                if (!empty($filters['field']) && $filters['field'] !== 'all' && $field !== $filters['field']) {
+                    continue;
+                }
+                foreach ($t2 as $date => $t3) {
+                    foreach ($t3 as $key => $value) {
+                        if ($scope == 'user') {
+                            if ($key === 'SYSTEM') {
+                                $text = 'SYSTEM';
+                            } else if (isset($userLookup[$key])) {
+                                $text = $userLookup[$key];
+                            } else {
+                                $user = $this->User->find('first', [
+                                    'fields' => ['User.id', 'User.email'],
+                                    'recursive' => -1,
+                                    'conditions' => ['User.id' => $key]
+                                ]);
+                                if (empty($user)) {
+                                    $text = '(' . $key . ') ' . __('Invalid user');
+                                } else {
+                                    $text = '(' . $key . ') ' . $user['User']['email'];
+                                }
+                                $userLookup[$key] = $text;
+                            }
+                        } else {
+                            $text = $key;
+                        }
+                        if (!empty($filters['quickFilter'])) {
+                            $q = strtolower($filters['quickFilter']);
+                            if (
+                                strpos(strtolower($scope), $q) === false &&
+                                strpos(strtolower($field), $q) === false &&
+                                strpos(strtolower($key), $q) === false &&
+                                strpos(strtolower($value), $q) === false &&
+                                strpos(strtolower($date), $q) === false &&
+                                strpos(strtolower($text), $q) === false
+                            ) {
+                                continue;
+                            }
+                        }
+                        if (empty($filters['key']) || $key == $filters['key']) {
+                            $data[] = [
+                                'scope' => $scope,
+                                'field' => $field,
+                                'date' => $date,
+                                'key' => $key,
+                                'text' => $text,
+                                'value' => $value,
+                                'unit' => $units[$field]
+                            ];    
+                        }
+                    }
+                }
+            }
+        }
+        if ($this->_isRest()) {
+            return $this->RestResponse->viewData($data, $this->response->type());
+        }
+        App::uses('CustomPaginationTool', 'Tools');
+        $customPagination = new CustomPaginationTool();
+        $customPagination->truncateAndPaginate($data, $this->params, $this->modelClass, true);
+        $this->set('data', $data);
+        $this->set('passedArgs', json_encode($passedArgs));
+        $this->set('filters', $filters);
+    }
+
+}

app/Controller/CollectionElementsController.php

@@ -0,0 +1,146 @@
+<?php
+
+use PHPUnit\Framework\MockObject\InvalidMethodNameException;
+
+App::uses('AppController', 'Controller');
+
+class CollectionElementsController extends AppController
+{
+
+    public $components = ['Session', 'RequestHandler'];
+
+    public $paginate = [
+        'limit' => 60,
+        'order' => []
+    ];
+
+    public $uses = [
+    ];
+    
+    public function add($collection_id)
+    {   
+        $this->CollectionElement->Collection->current_user = $this->Auth->user();
+        if (!$this->CollectionElement->Collection->mayModify($this->Auth->user('id'), intval($collection_id))) {
+            throw new MethodNotAllowedException(__('Invalid Collection or insuficient privileges'));
+        }
+        $this->CRUD->add([
+            'beforeSave' => function (array $collectionElement) use ($collection_id) {
+                $collectionElement['CollectionElement']['collection_id'] = intval($collection_id);
+                return $collectionElement;
+            }
+        ]);
+        if ($this->restResponsePayload) {
+            return $this->restResponsePayload;
+        }
+        $dropdownData = [
+            'types' => array_combine($this->CollectionElement->valid_types, $this->CollectionElement->valid_types)
+        ];
+        $this->set(compact('dropdownData'));
+        $this->set('menuData', array('menuList' => 'collections', 'menuItem' => 'add_element'));
+    }
+
+    public function delete($element_id)
+    {
+        $collectionElement = $this->CollectionElement->find('first', [
+            'recursive' => -1,
+            'conditions' => [
+                'CollectionElement.id' => $element_id
+            ]
+        ]);
+        $collection_id = $collectionElement['CollectionElement']['collection_id'];
+        if (!$this->CollectionElement->Collection->mayModify($this->Auth->user('id'), $collection_id)) {
+            throw new MethodNotAllowedException(__('Invalid Collection or insuficient privileges'));
+        }
+        $this->CRUD->delete($element_id);
+        if ($this->restResponsePayload) {
+            return $this->restResponsePayload;
+        }
+    }
+
+    public function index($collection_id)
+    {
+        $this->set('menuData', array('menuList' => 'collections', 'menuItem' => 'index'));
+        if (!$this->CollectionElement->Collection->mayView($this->Auth->user('id'), intval($collection_id))) {
+            throw new NotFoundException(__('Invalid collection or no access.'));
+        }
+        $params = [
+            'filters' => ['uuid', 'type', 'name'],
+            'quickFilters' => ['name'],
+            'conditions' => ['collection_id' => $collection_id]
+        ];
+        $this->loadModel('Event');
+        $this->set('distributionLevels', $this->Event->distributionLevels);
+        $this->CRUD->index($params);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }
+    }
+
+    public function addElementToCollection($element_type, $element_uuid)
+    {
+        if ($this->request->is('get')) {
+            $validCollections = $this->CollectionElement->Collection->find('list', [
+                'recursive' => -1,
+                'fields' => ['Collection.id', 'Collection.name'],
+                'conditions' => ['Collection.orgc_id' => $this->Auth->user('org_id')]
+            ]);
+            if (empty($validCollections)) {
+                if ($this->request->is('ajax')) {
+                    return $this->redirect(['controller' => 'collections', 'action' => 'add']);
+                }
+                throw new NotFoundException(__('You don\'t have any collections yet. Make sure you create one first before you can start adding elements.'));
+            }
+            $dropdownData = [
+                'collections' => $validCollections
+            ];
+            $this->set(compact('dropdownData'));
+        } else if ($this->request->is('post')) {
+            if (!isset($this->request->data['CollectionElement'])) {
+                $this->request->data = ['CollectionElement' => $this->request->data];
+            }
+            if (!isset($this->request->data['CollectionElement']['collection_id'])) {
+                throw new NotFoundException(__('No collection_id specified.'));
+            }
+            $collection_id = intval($this->request->data['CollectionElement']['collection_id']);
+            if (!$this->CollectionElement->Collection->mayModify($this->Auth->user('id'), $collection_id)) {
+                throw new NotFoundException(__('Invalid collection or not authorized.'));
+            }
+            $description = empty($this->request->data['CollectionElement']['description']) ? '' : $this->request->data['CollectionElement']['description'];
+            $dataToSave = [
+                'CollectionElement' => [
+                    'element_uuid' => $element_uuid,
+                    'element_type' => $element_type,
+                    'description' => $description,
+                    'collection_id' => $collection_id
+                ]
+            ];
+            $this->CollectionElement->create();
+            $error = '';
+            try {
+                $result = $this->CollectionElement->save($dataToSave);
+            } catch (PDOException $e) {
+                if ($e->errorInfo[0] == 23000) {
+                    $error = __(' Element already in Collection.');
+                }
+            }
+            
+            if ($result) {
+                $message = __('Element added to the Collection.');
+                if ($this->IndexFilter->isRest()) {
+                    return $this->RestResponse->saveSuccessResponse('CollectionElements', 'addElementToCollection', false, $this->response->type(), $message);
+                } else {
+                    $this->Flash->success($message);
+                    $this->redirect(Router::url($this->referer(), true));
+                }
+            } else {
+                $message = __('Element could not be added to the Collection.%s', $error);
+                if ($this->IndexFilter->isRest()) {
+                    return $this->RestResponse->saveFailResponse('CollectionElements', 'addElementToCollection', false, $message, $this->response->type());
+                } else {
+                    $this->Flash->error($message);
+                    $this->redirect(Router::url($this->referer(), true));
+                }
+            }
+        }
+    }
+}

app/Controller/CollectionsController.php

@@ -0,0 +1,171 @@
+<?php
+App::uses('AppController', 'Controller');
+
+class CollectionsController extends AppController
+{
+
+    public $components = ['Session', 'RequestHandler'];
+
+    public $paginate = [
+        'limit' => 60,
+        'order' => []
+    ];
+
+    public $uses = [
+    ];
+
+    private $valid_types = [
+        'campaign',
+        'intrusion_set',
+        'named_threat',
+        'other',
+        'research'
+    ];
+    
+    public function add()
+    {   
+        $this->Collection->current_user = $this->Auth->user();
+        $params = [];
+        if ($this->request->is('post')) {
+            $data = $this->request->data;
+            $params = [
+                'afterSave' => function (array $collection) use ($data) {
+                    $this->Collection->CollectionElement->captureElements($collection);
+                    return $collection;
+                }
+            ];
+        }
+        $this->CRUD->add($params);
+        if ($this->restResponsePayload) {
+            return $this->restResponsePayload;
+        }
+        $this->set('menuData', array('menuList' => 'collections', 'menuItem' => 'add'));
+        $this->loadModel('Event');
+        $dropdownData = [
+            'types' => array_combine($this->valid_types, $this->valid_types),
+            'distributionLevels' => $this->Event->distributionLevels,
+            'sgs' => $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1)  
+        ];
+        $this->set('initialDistribution', Configure::read('MISP.default_event_distribution'));
+        $this->set(compact('dropdownData'));
+        $this->render('add');
+    }
+
+    public function edit($id)
+    {
+        $this->Collection->current_user = $this->Auth->user();
+        if (!$this->Collection->mayModify($this->Auth->user('id'), $id)) {
+            throw new MethodNotAllowedException(__('Invalid Collection or insuficient privileges'));
+        }
+        $params = [];
+        if ($this->request->is('post') || $this->request->is('put')) {
+            $oldCollection = $this->Collection->find('first', [
+                'recursive' => -1,
+                'conditions' => ['Collection.id' => intval($id)]
+            ]);
+            if (empty($oldCollection)) {
+                throw new NotFoundException(__('Invalid collection.'));
+            }
+            if (empty($this->request->data['Collection'])) {
+                $this->request->data = ['Collection' => $this->request->data];
+            }
+            $data = $this->request->data;
+            if (
+                isset($data['Collection']['modified']) &&
+                $data['Collection']['modified'] <= $oldCollection['Collection']['modified']
+            ) {
+                throw new ForbiddenException(__('Collection received older or same as local version.'));
+            }
+            $params = [
+                'afterSave' => function (array &$collection) use ($data) {
+                    $collection = $this->Collection->CollectionElement->captureElements($collection);
+                    return $collection;
+                }
+            ];
+        }
+        $this->set('id', $id);
+        $this->CRUD->edit($id, $params);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }
+        $this->set('menuData', array('menuList' => 'collections', 'menuItem' => 'edit'));
+        $this->loadModel('Event');
+        $dropdownData = [
+            'types' => $this->valid_types,
+            'distributionLevels' => $this->Event->distributionLevels,
+            'sgs' => $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1)  
+        ];
+        $this->set(compact('dropdownData'));
+        $this->render('add');
+    }
+
+    public function delete($id)
+    {
+        if (!$this->Collection->mayModify($this->Auth->user('id'), $id)) {
+            throw new MethodNotAllowedException(__('Invalid Collection or insuficient privileges'));
+        }
+        $this->CRUD->delete($id);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }   
+    }
+
+    public function view($id)
+    {
+        $this->set('mayModify', $this->Collection->mayModify($this->Auth->user('id'), $id));
+        if (!$this->Collection->mayView($this->Auth->user('id'), $id)) {
+            throw new MethodNotAllowedException(__('Invalid Collection or insuficient privileges'));
+        }
+        $this->set('menuData', array('menuList' => 'collections', 'menuItem' => 'view'));
+        $params = [
+            'contain' => [
+                'Orgc',
+                'Org',
+                'User',
+                'CollectionElement'
+            ],
+            'afterFind' => function (array $collection){
+                return $this->Collection->rearrangeCollection($collection);
+            }
+        ];
+        $this->CRUD->view($id, $params);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }
+        $this->set('id', $id);
+        $this->loadModel('Event');
+        $this->set('distributionLevels', $this->Event->distributionLevels);
+        $this->render('view');
+    }
+
+    public function index($filter = null)
+    {
+        $this->set('menuData', array('menuList' => 'collections', 'menuItem' => 'index'));
+        $params = [
+            'filters' => ['Collection.uuid', 'Collection.type', 'Collection.name'],
+            'quickFilters' => ['Collection.name'],
+            'contain' => ['Orgc'],
+            'afterFind' => function($collections) {
+                foreach ($collections as $k => $collection) {
+                    $collections[$k]['Collection']['element_count'] = $this->Collection->CollectionElement->find('count', [
+                        'recursive' => -1,
+                        'conditions' => ['CollectionElement.collection_id' => $collection['Collection']['id']]
+                    ]);
+                }
+                return $collections;
+            }
+        ];
+        if ($filter === 'my_collections') {
+            $params['conditions']['Collection.user_id'] = $this->Auth->user('id');
+        }
+        if ($filter === 'org_collections') {
+            $params['conditions']['Collection.orgc_id'] = $this->Auth->user('org_id');
+        }
+        $this->loadModel('Event');
+        $this->set('distributionLevels', $this->Event->distributionLevels);
+        $this->CRUD->index($params);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }
+    }
+}

app/Controller/Component/CRUDComponent.php

@@ -25,6 +25,7 @@ class CRUDComponent extends Component
             }
             $options['filters'][] = 'quickFilter';
         }
+        $this->Controller->{$this->Controller->modelClass}->includeAnalystData = true;
         $params = $this->Controller->IndexFilter->harvestParameters(empty($options['filters']) ? [] : $options['filters']);
         $query = [];
         $query = $this->setFilters($params, $query);
@@ -39,6 +40,7 @@ class CRUDComponent extends Component
             if (!empty($this->Controller->paginate['fields'])) {
                 $query['fields'] = $this->Controller->paginate['fields'];
             }
+            $query['includeAnalystData'] = true;
             $data = $this->Controller->{$this->Controller->modelClass}->find('all', $query);
             if (isset($options['afterFind'])) {
                 if (is_callable($options['afterFind'])) {
@@ -49,6 +51,7 @@ class CRUDComponent extends Component
             }
             $this->Controller->restResponsePayload = $this->Controller->RestResponse->viewData($data, 'json');
         } else {
+            $query['includeAnalystData'] = true;
             $this->Controller->paginate = $query;
             $data = $this->Controller->paginate();
             if (isset($options['afterFind'])) {
@@ -93,7 +96,7 @@ class CRUDComponent extends Component
             $savedData = $model->save($data);
             if ($savedData) {
                 if (isset($params['afterSave'])) {
-                    $params['afterSave']($data);
+                    $params['afterSave']($savedData);
                 }
                 $data = $model->find('first', [
                     'recursive' => -1,
@@ -200,7 +203,7 @@ class CRUDComponent extends Component
             if (isset($params['beforeSave'])) {
                 $data = $params['beforeSave']($data);
             }
-            if ($model->save($data)) {
+            if ($data = $model->save($data)) {
                 if (isset($params['afterSave'])) {
                     $params['afterSave']($data);
                 }
@@ -231,6 +234,8 @@ class CRUDComponent extends Component
         if (empty($id)) {
             throw new NotFoundException(__('Invalid %s.', $modelName));
         }
+        $this->Controller->{$modelName}->includeAnalystData = true;
+        $this->Controller->{$modelName}->includeAnalystDataRecursive = true;
         $query = [
             'recursive' => -1,
             'conditions' => [$modelName . '.id' => $id],
@@ -284,6 +289,9 @@ class CRUDComponent extends Component
                 }
             }
         }
+        if (isset($params['afterFind'])) {
+            $data = $params['afterFind']($data);
+        }
         if (isset($params['beforeDelete'])) {
             $data = $params['beforeDelete']($data);
             if (empty($data)) {
@@ -297,6 +305,9 @@ class CRUDComponent extends Component
                 $result = $this->Controller->{$modelName}->delete($id);
             }
             if ($result) {
+                if (isset($params['afterDelete']) && is_callable($params['afterDelete'])) {
+                    $params['afterDelete']($data);
+                }
                 $message = __('%s deleted.', $modelName);
                 if ($this->Controller->IndexFilter->isRest()) {
                     $this->Controller->restResponsePayload = $this->Controller->RestResponse->saveSuccessResponse($modelName, 'delete', $id, 'json', $message);
@@ -336,7 +347,9 @@ class CRUDComponent extends Component
                 if ($filter === 'quickFilter') {
                     continue;
                 }
-                if (strlen(trim($filterValue, '%')) === strlen($filterValue)) {
+                if (is_array($filterValue)) {
+                    $query['conditions']['AND'][] = [$filter => $filterValue];
+                } else if (strlen(trim($filterValue, '%')) === strlen($filterValue)) {
                     $query['conditions']['AND'][] = [$filter => $filterValue];
                 } else {
                     $query['conditions']['AND'][] = [$filter . ' LIKE' => $filterValue];

app/Controller/Component/CompressedRequestHandlerComponent.php

@@ -37,14 +37,17 @@ class CompressedRequestHandlerComponent extends Component
     private function decodeGzipEncodedContent(Controller $controller)
     {
         if (function_exists('gzdecode')) {
-            $decoded = gzdecode($controller->request->input());
+            $input = $controller->request->input();
+            if (empty($input)) {
+                throw new BadRequestException('Request data should be gzip encoded, but request is empty.');
+            }
+            $decoded = gzdecode($input);
             if ($decoded === false) {
                 throw new BadRequestException('Invalid compressed data.');
             }
             return $decoded;
-        } else {
-            throw new BadRequestException("This server doesn't support GZIP compressed requests.");
         }
+        throw new BadRequestException("This server doesn't support GZIP compressed requests.");
     }
 
     /**

app/Controller/Component/IndexFilterComponent.php

@@ -8,7 +8,9 @@ class IndexFilterComponent extends Component
 {
     /** @var Controller */
     public $Controller;
-    public $isRest = null;
+
+    /** @var bool|null  */
+    private $isRest = null;
 
     // Used for isApiFunction(), a check that returns true if the controller & action combo matches an action that is a non-xml and non-json automation method
     // This is used to allow authentication via headers for methods not covered by _isRest() - as that only checks for JSON and XML formats
@@ -93,6 +95,11 @@ class IndexFilterComponent extends Component
         }
     }
 
+    public function isXhr()
+    {
+        return $this->Controller->request->header('X-Requested-With') === 'XMLHttpRequest';
+    }
+
     public function isJson()
     {
         return $this->Controller->request->header('Accept') === 'application/json' || $this->Controller->RequestHandler->prefers() === 'json';
@@ -103,11 +110,6 @@ class IndexFilterComponent extends Component
         return $this->Controller->request->header('Accept') === 'text/csv' || $this->Controller->RequestHandler->prefers() === 'csv';
     }
 
-    public function isXml()
-    {
-
-    }
-
     /**
      * @param string $controller
      * @param string $action

app/Controller/Component/RateLimitComponent.php

@@ -12,58 +12,60 @@ class RateLimitComponent extends Component
         )
     );
 
-    public $components = array('RestResponse');
-
     /**
      * @param array $user
      * @param string $controller
      * @param string $action
-     * @param array $info
-     * @param string $responseType
-     * @return bool
+     * @return array|null
      * @throws RedisException
      */
-    public function check(array $user, $controller, $action, &$info = array(), $responseType)
+    public function check(array $user, $controller, $action)
     {
-        if (!empty($user['Role']['enforce_rate_limit']) && isset(self::LIMITED_FUNCTIONS[$controller][$action])) {
-            if ($user['Role']['rate_limit_count'] == 0) {
-                throw new MethodNotAllowedException(__('API searches are not allowed for this user role.'));
-            }
-            try {
-                $redis = RedisTool::init();
-            } catch (Exception $e) {
-                return true; // redis is not available, allow access
-            }
-            $uuid = Configure::read('MISP.uuid') ?: 'no-uuid';
-            $keyName = 'misp:' . $uuid . ':rate_limit:' . $user['id'];
-            $count = $redis->get($keyName);
-            if ($count !== false && $count >= $user['Role']['rate_limit_count']) {
-                $info = array(
-                    'limit' => $user['Role']['rate_limit_count'],
-                    'reset' => $redis->ttl($keyName),
-                    'remaining' => $user['Role']['rate_limit_count'] - $count,
-                );
-                return $this->RestResponse->throwException(
-                    429,
-                    __('Rate limit exceeded.'),
-                    '/' . $controller . '/' . $action,
-                    $responseType
-                );
-            } else {
-                if ($count === false) {
-                    $redis->setEx($keyName, 900, 1);
-                } else {
-                    $redis->setEx($keyName, $redis->ttl($keyName), intval($count) + 1);
-                }
-            }
-            $count += 1;
-            $info = array(
-                'limit' => $user['Role']['rate_limit_count'],
-                'reset' => $redis->ttl($keyName),
-                'remaining' => $user['Role']['rate_limit_count'] - $count
-            );
-
+        if (!isset(self::LIMITED_FUNCTIONS[$controller][$action])) {
+            return null; // no limit enforced for this controller action
         }
-        return true;
+
+        if (empty($user['Role']['enforce_rate_limit'])) {
+            return null; // no limit enforced for this role
+        }
+
+        $rateLimit = (int)$user['Role']['rate_limit_count'];
+        if ($rateLimit === 0) {
+            throw new MethodNotAllowedException(__('API searches are not allowed for this user role.'));
+        }
+
+        try {
+            $redis = RedisTool::init();
+        } catch (Exception $e) {
+            return null; // redis is not available, allow access
+        }
+
+        $uuid = Configure::read('MISP.uuid') ?: 'no-uuid';
+        $keyName = 'misp:' . $uuid . ':rate_limit:' . $user['id'];
+        $count = $redis->get($keyName);
+
+        if ($count !== false && $count >= $rateLimit) {
+            return [
+                'exceeded' => true,
+                'limit' => $rateLimit,
+                'reset' => $redis->ttl($keyName),
+                'remaining' => $rateLimit - $count,
+            ];
+        }
+
+        $newCount = $redis->incr($keyName);
+        if ($newCount === 1) {
+            $redis->expire($keyName, 900);
+            $reset = 900;
+        } else {
+            $reset = $redis->ttl($keyName);
+        }
+
+        return [
+            'exceeded' => false,
+            'limit' => $rateLimit,
+            'reset' => $reset,
+            'remaining' => $rateLimit - $newCount,
+        ];
     }
 }

app/Controller/Component/RestResponseComponent.php

@@ -213,6 +213,7 @@ class RestResponseComponent extends Component
                     'perm_tag_editor',
                     'default_role',
                     'perm_sighting',
+                    'perm_analyst_data',
                     'permission'
                 )
             ),
@@ -234,6 +235,7 @@ class RestResponseComponent extends Component
                     'perm_tag_editor',
                     'default_role',
                     'perm_sighting',
+                    'perm_analyst_data',
                     'permission'
                 )
             )
@@ -517,7 +519,7 @@ class RestResponseComponent extends Component
         if ($id) {
             $response['id'] = $id;
         }
-        return $this->__sendResponse($response, 403, $format);
+        return $this->prepareResponse($response, 403, $format);
     }
 
     /**
@@ -562,7 +564,7 @@ class RestResponseComponent extends Component
         if ($id) {
             $response['id'] = $id;
         }
-        return $this->__sendResponse($response, 200, $format);
+        return $this->prepareResponse($response, 200, $format);
     }
 
     /**
@@ -587,7 +589,7 @@ class RestResponseComponent extends Component
      * @return CakeResponse
      * @throws Exception
      */
-    private function __sendResponse($response, $code, $format = false, $raw = false, $download = false, $headers = array())
+    private function prepareResponse($response, $code, $format = false, $raw = false, $download = false, $headers = array())
     {
         App::uses('TmpFileTool', 'Tools');
         $format = !empty($format) ? strtolower($format) : 'json';
@@ -633,7 +635,7 @@ class RestResponseComponent extends Component
                 }
                 
                 // If response is big array, encode items separately to save memory
-                if (is_array($response) && count($response) > 10000) {
+                if (is_array($response) && count($response) > 10000 && JsonTool::arrayIsList($response)) {
                     $output = new TmpFileTool();
                     $output->write('[');
 
@@ -669,9 +671,10 @@ class RestResponseComponent extends Component
         }
 
         if ($response instanceof TmpFileTool) {
-            if (isset($_SERVER['HTTP_IF_NONE_MATCH'])) {
+            $requestEtag = $this->requestEtag();
+            if ($requestEtag !== null) {
                 $etag = '"' . $response->hash('sha1') . '"';
-                if ($_SERVER['HTTP_IF_NONE_MATCH'] === $etag) {
+                if ($requestEtag === $etag) {
                     return new CakeResponse(['status' => 304]);
                 }
                 $headers['ETag'] = $etag;
@@ -687,9 +690,10 @@ class RestResponseComponent extends Component
             }
         } else {
             // Check if resource was changed when `If-None-Match` header is send and return 304 Not Modified
-            if (isset($_SERVER['HTTP_IF_NONE_MATCH'])) {
+            $requestEtag = $this->requestEtag();
+            if ($requestEtag !== null) {
                 $etag = '"' . sha1($response) . '"';
-                if ($_SERVER['HTTP_IF_NONE_MATCH'] === $etag) {
+                if ($requestEtag === $etag) {
                     return new CakeResponse(['status' => 304]);
                 }
                 // Generate etag just when HTTP_IF_NONE_MATCH is set
@@ -722,6 +726,25 @@ class RestResponseComponent extends Component
         return $cakeResponse;
     }
 
+    /**
+     * Return etag from If-None-Match HTTP request header without compression marks added by Apache
+     * @return string|null
+     */
+    private function requestEtag()
+    {
+        if (isset($_SERVER['HTTP_IF_NONE_MATCH'])) {
+            // Remove compression marks that adds Apache for compressed content
+            $requestEtag = $_SERVER['HTTP_IF_NONE_MATCH'];
+            $etagWithoutQuotes = trim($requestEtag, '"');
+            $dashPos = strrpos($etagWithoutQuotes, '-');
+            if ($dashPos && in_array(substr($etagWithoutQuotes, $dashPos + 1), ['br', 'gzip'], true)) {
+                return '"' . substr($etagWithoutQuotes, 0, $dashPos) . '"';
+            }
+            return $requestEtag;
+        }
+        return null;
+    }
+
     /**
      * @param string $response
      * @return string Signature as base64 encoded string
@@ -775,7 +798,7 @@ class RestResponseComponent extends Component
         if (!empty($errors)) {
             $data['errors'] = $errors;
         }
-        return $this->__sendResponse($data, 200, $format, $raw, $download, $headers);
+        return $this->prepareResponse($data, 200, $format, $raw, $download, $headers);
     }
 
     /**
@@ -807,7 +830,7 @@ class RestResponseComponent extends Component
             'message' => $message,
             'url' => $url
         );
-        return $this->__sendResponse($message, $code, $format, $raw, false, $headers);
+        return $this->prepareResponse($message, $code, $format, $raw, false, $headers);
     }
 
     public function setHeader($header, $value)
@@ -834,7 +857,7 @@ class RestResponseComponent extends Component
             }
         }
         $response['url'] = $this->__generateURL($actionArray, $controller, $params);
-        return $this->__sendResponse($response, 200, $format);
+        return $this->prepareResponse($response, 200, $format);
     }
 
     private function __setup()
@@ -1052,7 +1075,7 @@ class RestResponseComponent extends Component
                 'input' => 'radio',
                 'type' => 'integer',
                 'values' => array(1 => 'True', 0 => 'False' ),
-                'help' => __('Include deleted elements')
+                'help' => __('Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned')
             ),
             'delta_merge' => array(
                 'input' => 'radio',
@@ -1557,6 +1580,11 @@ class RestResponseComponent extends Component
                 'type' => 'integer',
                 'values' => array(1 => 'True', 0 => 'False')
             ),
+            'perm_analyst_data' => array(
+                'input' => 'radio',
+                'type' => 'integer',
+                'values' => array(1 => 'True', 0 => 'False')
+            ),
             'permission' => array(
                 'input' => 'select',
                 'type' => 'string',

app/Controller/Component/RestSearchComponent.php

@@ -90,6 +90,7 @@ class RestSearchComponent extends Component
             'publish_timestamp',
             'timestamp',
             'event_timestamp', // redundant, but kept for backwards compatibility
+            'event_tags',
             'published',
             'enforceWarninglist',
             'sgReferenceOnly',
@@ -123,6 +124,7 @@ class RestSearchComponent extends Component
             'extended',
             'extensionList',
             'excludeGalaxy',
+            'includeAnalystData',
             'includeRelatedTags',
             'includeDecayScore',
             'includeScoresOnEvent',
@@ -142,7 +144,11 @@ class RestSearchComponent extends Component
             'retry',
             'expiry',
             'minimum_ttl',
-            'ttl'
+            'ttl',
+            'org.sector',
+            'org.local',
+            'org.nationality',
+            'galaxy.*',
         ],
         'Object' => [
             'returnFormat',
@@ -151,6 +157,8 @@ class RestSearchComponent extends Component
             'category',
             'org',
             'tags',
+            'first_seen',
+            'last_seen',
             'from',
             'to',
             'last',
@@ -177,12 +185,16 @@ class RestSearchComponent extends Component
             'attackGalaxy',
             'object_relation',
             'metadata',
-            'includeAllTags'
+            'includeAllTags',
+            'object_name',
+            'object_template_uuid',
+            'object_template_version'
         ],
         'Sighting' => [
             'context',
             'returnFormat',
             'id',
+            'uuid',
             'type',
             'from',
             'to',
@@ -190,7 +202,8 @@ class RestSearchComponent extends Component
             'org_id',
             'source',
             'includeAttribute',
-            'includeEvent'
+            'includeEvent',
+            'includeUuid',
         ],
         'GalaxyCluster' => [
             'page',
@@ -203,7 +216,7 @@ class RestSearchComponent extends Component
             'distribution',
             'org',
             'orgc',
-            'tag',
+            'tag_name',
             'custom',
             'sgReferenceOnly',
             'minimal',

app/Controller/EventReportsController.php

@@ -61,6 +61,8 @@ class EventReportsController extends AppController
 
     public function view($reportId, $ajax=false)
     {
+        $this->EventReport->includeAnalystData = true;
+        $this->EventReport->includeAnalystDataRecursive = true;
         $report = $this->EventReport->simpleFetchById($this->Auth->user(), $reportId);
         if ($this->_isRest()) {
             return $this->RestResponse->viewData($report, $this->response->type());
@@ -175,6 +177,7 @@ class EventReportsController extends AppController
         $filters = $this->IndexFilter->harvestParameters(['event_id', 'value', 'context', 'index_for_event', 'extended_event']);
         $filters['embedded_view']  = $this->request->is('ajax');
         $compiledConditions = $this->__generateIndexConditions($filters);
+        $this->EventReport->includeAnalystData = true;
         if ($this->_isRest()) {
             $reports = $this->EventReport->find('all', [
                 'recursive' => -1,
@@ -210,10 +213,13 @@ class EventReportsController extends AppController
 
     public function extractAllFromReport($reportId)
     {
-        if (!$this->request->is('ajax')) {
+        if (!$this->request->is('ajax') && !$this->_isRest()) {
             throw new MethodNotAllowedException(__('This function can only be reached via AJAX.'));
         }
         if ($this->request->is('post')) {
+            if (!isset($this->data['EventReport'])) {
+                $this->data = ['EventReport' => $this->data];
+            }
             $report = $this->EventReport->fetchIfAuthorized($this->Auth->user(), $reportId, 'edit', $throwErrors=true, $full=false);
             $results = $this->EventReport->getComplexTypeToolResultWithReplacements($this->Auth->user(), $report);
             $report['EventReport']['content'] = $results['replacementResult']['contentWithReplacements'];
@@ -296,13 +302,16 @@ class EventReportsController extends AppController
 
     public function importReportFromUrl($event_id)
     {
-        if (!$this->request->is('ajax')) {
-            throw new MethodNotAllowedException(__('This function can only be reached via AJAX.'));
+        if (!$this->request->is('ajax') && !$this->_isRest()) {
+            throw new MethodNotAllowedException(__('This function can only be reached via AJAX and via the API.'));
         }
         $fetcherModule = $this->EventReport->isFetchURLModuleEnabled();
         if ($this->request->is('post')) {
+            if (empty($this->data['EventReport'])) {
+                $this->data = ['EventReport' => $this->data];
+            }
             if (empty($this->data['EventReport']['url'])) {
-                throw new MethodNotAllowedException(__('An URL must be provided'));
+                throw new MethodNotAllowedException(__('A URL must be provided'));
             }
             $url = $this->data['EventReport']['url'];
             $format = 'html';
@@ -313,7 +322,6 @@ class EventReportsController extends AppController
                     $format = $parsed_format;
                 }
             }
-            
             $content = $this->EventReport->downloadMarkdownFromURL($event_id, $url, $format);
 
             $errors = [];
@@ -515,6 +523,7 @@ class EventReportsController extends AppController
     {
         $distributionLevels = $this->EventReport->Event->Attribute->distributionLevels;
         $this->set('distributionLevels', $distributionLevels);
+        $this->set('shortDist', $this->EventReport->Event->Attribute->shortDist);
         $this->set('initialDistribution', $this->EventReport->Event->Attribute->defaultDistribution());
     }
 
@@ -559,6 +568,12 @@ class EventReportsController extends AppController
                 $savedReport['EventReport'][$field] = $newReport['EventReport'][$field];
             }
         }
+        $this->loadModel('AnalystData');
+        foreach ($this->AnalystData::ANALYST_DATA_TYPES as $type) {
+            if (!empty($newReport['EventReport'][$type])) {
+                $savedReport['EventReport'][$type] = $newReport['EventReport'][$type];
+            }
+        }
         return $savedReport;
     }
 }

app/Controller/EventsController.php

@@ -14,16 +14,16 @@ class EventsController extends AppController
     );
 
     public $paginate = array(
-            'limit' => 60,
-            'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page.
-            'order' => array(
-                    'Event.timestamp' => 'DESC'
-            ),
-            'contain' => array(
-                    'Org' => array('fields' => array('id', 'name', 'uuid')),
-                    'Orgc' => array('fields' => array('id', 'name', 'uuid')),
-                    'SharingGroup' => array('fields' => array('id', 'name', 'uuid'))
-            )
+        'limit' => 60,
+        'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page.
+        'order' => array(
+            'Event.timestamp' => 'DESC'
+        ),
+        'contain' => array(
+            'Org' => array('fields' => array('id', 'name', 'uuid')),
+            'Orgc' => array('fields' => array('id', 'name', 'uuid')),
+            'SharingGroup' => array('fields' => array('id', 'name', 'uuid'))
+        )
     );
 
     // private
@@ -61,46 +61,14 @@ class EventsController extends AppController
         'publish_timestamp'
     ];
 
-    public $paginationFunctions = array('index', 'proposalEventIndex');
-
     public function beforeFilter()
     {
         parent::beforeFilter();
 
-        // what pages are allowed for non-logged-in users
-        $this->Auth->allow('xml');
-        $this->Auth->allow('csv');
-        $this->Auth->allow('nids');
-        $this->Auth->allow('hids_md5');
-        $this->Auth->allow('hids_sha1');
-        $this->Auth->allow('text');
-        $this->Auth->allow('restSearch');
-        $this->Auth->allow('stix');
-        $this->Auth->allow('stix2');
-
         $this->Security->unlockedActions[] = 'viewEventAttributes';
 
-        // TODO Audit, activate logable in a Controller
-        if (count($this->uses) && $this->{$this->modelClass}->Behaviors->attached('SysLogLogable')) {
-            $this->{$this->modelClass}->setUserData($this->activeUser);
-        }
-
-        // convert uuid to id if present in the url, and overwrite id field
-        if (isset($this->request->params->query['uuid'])) {
-            $params = array(
-                    'conditions' => array('Event.uuid' => $this->params->query['uuid']),
-                    'recursive' => 0,
-                    'fields' => 'Event.id'
-            );
-            $result = $this->Event->find('first', $params);
-            if (isset($result['Event']) && isset($result['Event']['id'])) {
-                $id = $result['Event']['id'];
-                $this->params->addParams(array('pass' => array($id))); // FIXME find better way to change id variable if uuid is found. params->url and params->here is not modified accordingly now
-            }
-        }
-
         // if not admin or own org, check private as well..
-        if (!$this->_isSiteAdmin() && in_array($this->request->action, $this->paginationFunctions, true)) {
+        if (!$this->_isSiteAdmin() && in_array($this->request->action, ['index', 'proposalEventIndex'], true)) {
             $conditions = $this->Event->createEventConditions($this->Auth->user());
             if ($this->userRole['perm_sync'] && $this->Auth->user('Server')['push_rules']) {
                 $conditions['AND'][] = $this->Event->filterRulesToConditions($this->Auth->user('Server')['push_rules']);
@@ -497,6 +465,11 @@ class EventsController extends AppController
                         continue 2;
                     }
                     $pieces = is_array($v) ? $v : explode('|', $v);
+                    $isANDed = false;
+                    if (count($pieces) == 1 && strpos($pieces[0], '&') !== false) {
+                        $pieces = explode('&', $pieces[0]);
+                        $isANDed = count($pieces) > 1;
+                    }
                     $filterString = "";
                     $expectOR = false;
                     $tagRules = [];
@@ -563,10 +536,19 @@ class EventsController extends AppController
                     }
 
                     if (!empty($tagRules['include'])) {
-                        $include = $this->Event->EventTag->find('column', array(
-                            'conditions' => array('EventTag.tag_id' => $tagRules['include']),
-                            'fields' => ['EventTag.event_id'],
-                        ));
+                        if ($isANDed) {
+                            $include = $this->Event->EventTag->find('column', array(
+                                'conditions' => ['EventTag.tag_id' => $tagRules['include']],
+                                'fields' => ['EventTag.event_id'],
+                                'group' => ['EventTag.event_id'],
+                                'having' => ['COUNT(*) =' => count($tagRules['include'])],
+                            ));
+                        } else {
+                            $include = $this->Event->EventTag->find('column', array(
+                                'conditions' => array('EventTag.tag_id' => $tagRules['include']),
+                                'fields' => ['EventTag.event_id'],
+                            ));
+                        }
                         if (!empty($include)) {
                             $this->paginate['conditions']['AND'][] = 'Event.id IN (' . implode(",", $include) . ')';
                         } else {
@@ -755,7 +737,8 @@ class EventsController extends AppController
         if ($nothing) {
             $this->paginate['conditions']['AND'][] = ['Event.id' => -1]; // do not fetch any event
         }
-
+        $this->Event->includeAnalystData = isset($passedArgs['includeAnalystData']) ? $passedArgs['includeAnalystData'] : false;
+        $this->paginate['includeAnalystData'] = isset($passedArgs['includeAnalystData']) ? $passedArgs['includeAnalystData'] : false;
         $events = $this->paginate();
 
         if (count($events) === 1 && isset($this->passedArgs['searchall'])) {
@@ -811,6 +794,7 @@ class EventsController extends AppController
             $rules = [
                 'contain' => ['EventTag'],
                 'fields' => array_keys($fieldNames),
+                'includeAnalystData' => isset($passedArgs['includeAnalystData']) ? $passedArgs['includeAnalystData'] : false,
             ];
         }
         if (isset($passedArgs['sort']) && isset($fieldNames[$passedArgs['sort']])) {
@@ -834,33 +818,29 @@ class EventsController extends AppController
         }
 
         if (empty($rules['limit'])) {
-            $events = array();
+            $events = [];
             $i = 1;
             $rules['limit'] = 20000;
             while (true) {
-                $rules['page'] = $i;
+                $rules['page'] = $i++;
                 $temp = $this->Event->find('all', $rules);
                 $resultCount = count($temp);
                 if ($resultCount !== 0) {
-                    // this is faster and memory efficient than array_merge
-                    foreach ($temp as $tempEvent) {
-                        $events[] = $tempEvent;
-                    }
+                    array_push($events, ...$temp);
                 }
                 if ($resultCount < $rules['limit']) {
                     break;
                 }
-                $i++;
             }
             unset($temp);
-            $absolute_total = count($events);
+            $absoluteTotal = count($events);
         } else {
             $counting_rules = $rules;
             unset($counting_rules['limit']);
             unset($counting_rules['page']);
-            $absolute_total = $this->Event->find('count', $counting_rules);
+            $absoluteTotal = $this->Event->find('count', $counting_rules);
 
-            $events = $absolute_total === 0 ? [] : $this->Event->find('all', $rules);
+            $events = $absoluteTotal === 0 ? [] : $this->Event->find('all', $rules);
         }
 
         $isCsvResponse = $this->response->type() === 'text/csv';
@@ -979,7 +959,7 @@ class EventsController extends AppController
             $events = $export->eventIndex($events);
         }
 
-        return $this->RestResponse->viewData($events, $this->response->type(), false, false, false, ['X-Result-Count' => $absolute_total]);
+        return $this->RestResponse->viewData($events, $this->response->type(), false, false, false, ['X-Result-Count' => $absoluteTotal]);
     }
 
     private function __indexColumns()
@@ -1231,6 +1211,9 @@ class EventsController extends AppController
             }
         }
 
+        $this->Event->Attribute->includeAnalystData = true;
+        $this->Event->Attribute->includeAnalystDataRecursive = true;
+
         if (isset($filters['focus'])) {
             $this->set('focus', $filters['focus']);
         }
@@ -1415,7 +1398,8 @@ class EventsController extends AppController
         $exception = false;
         $warningTagConflicts = array();
         $filters = $this->_harvestParameters($filterData, $exception);
-
+        $analystData = $this->Event->attachAnalystData($event['Event']);
+        $event['Event'] = array_merge($event['Event'], $analystData); 
         $emptyEvent = (empty($event['Object']) && empty($event['Attribute']));
         $this->set('emptyEvent', $emptyEvent);
 
@@ -1489,7 +1473,6 @@ class EventsController extends AppController
                 $containsProposals = true;
             }
         }
-
         foreach ($event['Object'] as $k => $object) {
             $modDate = date("Y-m-d", $object['timestamp']);
             $modificationMap[$modDate] = !isset($modificationMap[$modDate])? 1 : $modificationMap[$modDate] + 1;
@@ -1521,7 +1504,6 @@ class EventsController extends AppController
                 }
             }
         }
-
         if ($containsProposals && $this->__canPublishEvent($event, $user)) {
             $mess = $this->Session->read('Message');
             if (empty($mess)) {
@@ -1695,8 +1677,8 @@ class EventsController extends AppController
         }
 
         $namedParams = $this->request->params['named'];
-
         if ($this->_isRest()) {
+            $conditions['includeAnalystData'] = true;
             $conditions['includeAttachments'] = isset($namedParams['includeAttachments']) ? $namedParams['includeAttachments'] : true;
         } else {
             $conditions['includeAllTags'] = true;
@@ -1790,7 +1772,6 @@ class EventsController extends AppController
         } else {
             $user = $this->Auth->user();
         }
-
         $results = $this->Event->fetchEvent($user, $conditions);
         if (empty($results)) {
             throw new NotFoundException(__('Invalid event'));
@@ -2369,7 +2350,13 @@ class EventsController extends AppController
                     }
 
                     $isXml = $ext === 'xml';
-                    $data = FileAccessTool::readFromFile($file['tmp_name'], $file['size']);
+                    $matches = null;
+                    $tmp_name = $file['tmp_name'];
+                    if (preg_match_all('/[\w\/\-\.]*/', $tmp_name, $matches) && file_exists($file['tmp_name'])) {
+                        $data = FileAccessTool::readFromFile($matches[0][0], $file['size']);
+                    } else {
+                        throw new NotFoundException(__('Invalid file.'));    
+                    }
                 } else {
                     throw new MethodNotAllowedException(__('No file uploaded.'));
                 }
@@ -2378,12 +2365,11 @@ class EventsController extends AppController
                     && (isset($this->request->data['Event']['takeownership']) && $this->request->data['Event']['takeownership'] == 1);
 
                 $publish = $this->request->data['Event']['publish'] ?? false;
-
                 try {
                     $results = $this->Event->addMISPExportFile($this->Auth->user(), $data, $isXml, $takeOwnership, $publish);
                 } catch (Exception $e) {
                     $this->log("Exception during processing MISP file import: {$e->getMessage()}");
-                    $this->Flash->error(__('Could not process MISP export file. %s.', $e->getMessage()));
+                    $this->Flash->error(__('Could not process MISP export file. %s', $e->getMessage()));
                     $this->redirect(['controller' => 'events', 'action' => 'add_misp_export']);
                 }
             }
@@ -2408,24 +2394,41 @@ class EventsController extends AppController
                 }
                 if (isset($this->params['named']['distribution'])) {
                     $distribution = intval($this->params['named']['distribution']);
-                    if (array_key_exists($distribution, $distributionLevels)) {
-                        $initialDistribution = $distribution;
-                    } else {
-                        throw new MethodNotAllowedException(__('Wrong distribution level'));
+                    if (!array_key_exists($distribution, $distributionLevels)) {
+                        throw new BadRequestException(__('Wrong distribution level'));
                     }
+                } else {
+                    $distribution = $initialDistribution;
                 }
                 $sharingGroupId = null;
-                if ($initialDistribution == 4) {
+                if ($distribution == 4) {
                     if (!isset($this->params['named']['sharing_group_id'])) {
-                        throw new MethodNotAllowedException(__('The sharing group id is needed when the distribution is set to 4 ("Sharing group").'));
+                        throw new BadRequestException(__('The sharing group id is needed when the distribution is set to 4 ("Sharing group").'));
                     }
                     $sharingGroupId = intval($this->params['named']['sharing_group_id']);
                     if (!array_key_exists($sharingGroupId, $sgs)) {
-                        throw new MethodNotAllowedException(__('Please select a valid sharing group id.'));
+                        throw new BadRequestException(__('Please select a valid sharing group id.'));
                     }
                 }
+                $clusterDistribution = $initialDistribution;
+                $clusterSharingGroupId = null;
                 if (isset($this->params['named']['galaxies_as_tags'])) {
                     $galaxies_as_tags = $this->params['named']['galaxies_as_tags'];
+                    if (isset($this->params['name']['cluster_distribution'])) {
+                        $clusterDistribution = intval($this->params['named']['cluster_distribution']);
+                        if (!array_key_exists($clusterDistribution, $distributionLevels)) {
+                            throw new BadRequestException(__('Wrong cluster distribution level'));
+                        }
+                        if ($clusterDistribution == 4) {
+                            if (!isset($this->params['named']['cluster_sharing_group_id'])) {
+                                throw new BadRequestException(__('The cluster sharing group id is needed when the cluster distribution is set to 4 ("Sharing group").'));
+                            }
+                            $clusterSharingGroupId = intval($this->params['named']['cluster_sharing_group_id']);
+                            if (!array_key_exists($clusterSharingGroupId, $sgs)) {
+                                throw new BadRequestException(__('Please select a valid cluster sharing group id.'));
+                            }
+                        }
+                    }
                 }
                 if (isset($this->params['named']['debugging'])) {
                     $debug = $this->params['named']['debugging'];
@@ -2437,9 +2440,11 @@ class EventsController extends AppController
                     $stix_version,
                     'uploaded_stix_file.' . ($stix_version == '1' ? 'xml' : 'json'),
                     $publish,
-                    $initialDistribution,
+                    $distribution,
                     $sharingGroupId,
                     $galaxies_as_tags,
+                    $clusterDistribution,
+                    $clusterSharingGroupId,
                     $debug
                 );
                 if (is_numeric($result)) {
@@ -2452,8 +2457,8 @@ class EventsController extends AppController
                 } else {
                     return $this->RestResponse->saveFailResponse('Events', 'upload_stix', false, $result, $this->response->type());
                 }
-            } else {
-                $original_file = !empty($this->data['Event']['original_file']) ? $this->data['Event']['stix']['name'] : '';
+            } else { // not REST request
+                $originalFile = !empty($this->data['Event']['original_file']) ? $this->data['Event']['stix']['name'] : '';
                 if (isset($this->data['Event']['stix']) && $this->data['Event']['stix']['size'] > 0 && is_uploaded_file($this->data['Event']['stix']['tmp_name'])) {
                     $filePath = FileAccessTool::createTempFile();
                     if (!move_uploaded_file($this->data['Event']['stix']['tmp_name'], $filePath)) {
@@ -2466,11 +2471,13 @@ class EventsController extends AppController
                         $this->Auth->user(),
                         $filePath,
                         $stix_version,
-                        $original_file,
+                        $originalFile,
                         $this->data['Event']['publish'],
                         $this->data['Event']['distribution'],
-                        $this->data['Event']['sharing_group_id'],
-                        $this->data['Event']['galaxies_handling'],
+                        $this->data['Event']['sharing_group_id'] ?? null,
+                        $this->data['Event']['galaxies_handling'] ?? false,
+                        $this->data['Event']['cluster_distribution'] ?? 0,
+                        $this->data['Event']['cluster_sharing_group_id'] ?? null,
                         $debug
                     );
                     if (is_numeric($result)) {
@@ -2501,15 +2508,31 @@ class EventsController extends AppController
         foreach ($distributionLevels as $key => $value) {
             $fieldDesc['distribution'][$key] = $this->Event->distributionDescriptions[$key]['formdesc'];
         }
-        $debugOptions = $this->Event->debugOptions;
+
+        $debugOptions = [
+            0 => __('Standard debugging'),
+            1 => __('Advanced debugging'),
+        ];
+        $debugDescriptions = [
+            0 => __('The critical errors are logged in the usual log file.'),
+            1 => __('All the errors and warnings are logged in the usual log file.'),
+        ];
+        $galaxiesOptions = [
+            0 => __('As MISP standard format'),
+            1 => __('As tag names'),
+        ];
+        $galaxiesOptionsDescriptions = [
+            0 => __('Galaxies and Clusters are passed as MISP standard format. New generic Galaxies and Clusters are created when there is no match with existing ones.'),
+            1 => __('Galaxies are passed as tags and there is only a simple search with existing galaxy tag names.'),
+        ];
+
         $this->set('debugOptions', $debugOptions);
         foreach ($debugOptions as $key => $value) {
-            $fieldDesc['debug'][$key] = $this->Event->debugDescriptions[$key];
+            $fieldDesc['debug'][$key] = $debugDescriptions[$key];
         }
-        $galaxiesOptions = $this->Event->galaxiesOptions;
         $this->set('galaxiesOptions', $galaxiesOptions);
         foreach ($galaxiesOptions as $key => $value) {
-            $fieldDesc['galaxies_handling'][$key] = $this->Event->galaxiesOptionsDescriptions[$key];
+            $fieldDesc['galaxies_handling'][$key] = $galaxiesOptionsDescriptions[$key];
         }
         $this->set('sharingGroups', $sgs);
         $this->set('fieldDesc', $fieldDesc);
@@ -2680,7 +2703,7 @@ class EventsController extends AppController
                 $this->request->data = $this->request->data['Event'];
             }
             $eventToSave = $event;
-            $capturedObjects = ['Attribute', 'Object', 'Tag', 'Galaxy', 'EventReport'];
+            $capturedObjects = ['Attribute', 'Object', 'Tag', 'Galaxy', 'EventReport', 'Note', 'Opinion', 'Relationship',];
             foreach ($capturedObjects as $objectType) {
                 if (!empty($this->request->data[$objectType])) {
                     if (!empty($regenerateUUIDs)) {
@@ -3070,9 +3093,9 @@ class EventsController extends AppController
                         $errors['Module'] = 'Module failure.';
                     }
                 } else {
+                    $errors['failed_servers'] = $result;
                     $lastResult = array_pop($result);
                     $resultString = (count($result) > 0) ? implode(', ', $result) . ' and ' . $lastResult : $lastResult;
-                    $errors['failed_servers'] = $result;
                     $message = __('Event published but not pushed to %s, re-try later. If the issue persists, make sure that the correct sync user credentials are used for the server link and that the sync user on the remote server has authentication privileges.', $resultString);
                 }
             } else {
@@ -3187,7 +3210,7 @@ class EventsController extends AppController
         $event = $this->Event->find('first', [
             'conditions' => Validation::uuid($id) ? ['Event.uuid' => $id] : ['Event.id' => $id],
             'recursive' => -1,
-            'fields' => ['id', 'info', 'publish_timestamp', 'orgc_id'],
+            'fields' => ['id', 'info', 'publish_timestamp', 'orgc_id', 'user_id'],
         ]);
         if (empty($event)) {
             throw new NotFoundException(__('Invalid event.'));
@@ -3206,6 +3229,16 @@ class EventsController extends AppController
                 }
             }
         }
+        if (
+            Configure::read('MISP.block_publishing_for_same_creator', false) &&
+            $this->Auth->user()['id'] == $event['Event']['user_id']
+        ) {
+            $message = __('Could not publish the event, the publishing user cannot be the same as the event creator as per this instance\'s configuration.');
+            if (!$this->_isRest()) {
+                $this->Flash->error($message);
+            }
+            throw new MethodNotAllowedException($message);
+        }
 
         return $event;
     }
@@ -3309,7 +3342,7 @@ class EventsController extends AppController
             $this->Flash->info(__('Warning, you are logged in as a site admin, any export that you generate will contain the FULL UNRESTRICTED data-set. If you would like to generate an export for your own organisation, please log in with a different user.'));
         }
         // Check if the background jobs are enabled - if not, fall back to old export page.
-        if (Configure::read('MISP.background_jobs') && !Configure::read('MISP.disable_cached_exports')) {
+        if (Configure::read('MISP.background_jobs') && !Configure::read('MISP.disable_cached_exports', true)) {
             $now = time();
 
             // as a site admin we'll use the ADMIN identifier, not to overwrite the cached files of our own org with a file that includes too much data.
@@ -3396,7 +3429,7 @@ class EventsController extends AppController
 
     public function downloadExport($type, $extra = null)
     {
-        if (Configure::read('MISP.disable_cached_exports')) {
+        if (Configure::read('MISP.disable_cached_exports', true)) {
             throw new MethodNotAllowedException(__('This feature is currently disabled'));
         }
         if ($this->_isSiteAdmin()) {
@@ -3792,11 +3825,21 @@ class EventsController extends AppController
         if ($id === false) {
             $id = $this->request->data['event'];
         }
-        $this->Event->recursive = -1;
-        $event = $this->Event->read(array(), $id);
+        $conditions = ['Event.id' => $id];
+        if (Validation::uuid($id)) {
+            $conditions = ['Event.uuid' => $id];
+        }
+        $event = $this->Event->find(
+            'first',
+            [
+                'recursive' => -1,
+                'conditions' => $conditions
+            ]
+        );
         if (empty($event)) {
             return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Invalid event.')), 'status'=>200, 'type' => 'json'));
         }
+        $id = $event['Event']['id'];
         $local = !empty($this->params['named']['local']);
         if (!$this->request->is('post')) {
             $this->set('local', $local);
@@ -4099,7 +4142,13 @@ class EventsController extends AppController
                 }
             }
             $this->Event->Attribute->fetchRelated($this->Auth->user(), $resultArray);
-            $typeCategoryMapping = array();
+            $typeCategoryMapping = [
+                'ip-src/ip-dst' => [
+                    'Network activity' => 'Network activity',
+                    'Payload delivery' => 'Payload delivery',
+                    'External analysis' => 'External analysis'
+                ],
+            ];
             foreach ($this->Event->Attribute->categoryDefinitions as $k => $cat) {
                 foreach ($cat['types'] as $type) {
                     $typeCategoryMapping[$type][$k] = $k;
@@ -4341,12 +4390,12 @@ class EventsController extends AppController
         $id = $event['Event']['id'];
         $exports = array(
             'json' => array(
-                'url' => $this->baseurl . '/events/restSearch/json/eventid:' . $id . '.json',
+                'url' => $this->baseurl . '/events/restSearch/json/includeAnalystData:1/eventid:' . $id . '.json',
                 'text' => __('MISP JSON (metadata + all attributes)'),
                 'requiresPublished' => false,
                 'checkbox' => true,
                 'checkbox_text' => __('Encode Attachments'),
-                'checkbox_set' => $this->baseurl . '/events/restSearch/json/withAttachments:1/eventid:' . $id . '.json',
+                'checkbox_set' => $this->baseurl . '/events/restSearch/json/withAttachments:1/includeAnalystData:1/eventid:' . $id . '.json',
                 'checkbox_default' => true,
             ),
             'xml' => array(
@@ -4800,16 +4849,18 @@ class EventsController extends AppController
     public function updateGraph($id, $type = 'event')
     {
         $user = $this->_closeSession();
+
         $validTools = array('event', 'galaxy', 'tag');
         if (!in_array($type, $validTools, true)) {
             throw new MethodNotAllowedException(__('Invalid type.'));
         }
+
         $this->loadModel('Taxonomy');
         $this->loadModel('GalaxyCluster');
         App::uses('CorrelationGraphTool', 'Tools');
-        $grapher = new CorrelationGraphTool();
+
         $data = $this->request->is('post') ? $this->request->data : array();
-        $grapher->construct($this->Event, $this->Taxonomy, $this->GalaxyCluster, $user, $data);
+        $grapher = new CorrelationGraphTool($this->Event, $this->Taxonomy, $this->GalaxyCluster, $user, $data);
         $json = $grapher->buildGraphJson($id, $type);
         array_walk_recursive($json, function (&$item, $key) {
             if (!mb_detect_encoding($item, 'utf-8', true)) {

app/Controller/FeedsController.php

@@ -74,6 +74,8 @@ class FeedsController extends AppController
                 );
             }
         }
+        $loggedUser = $this->Auth->user();
+        $this->loadModel('TagCollection');
 
         $this->CRUD->index([
             'filters' => [
@@ -92,7 +94,7 @@ class FeedsController extends AppController
                 'source_format'
             ],
             'conditions' => $conditions,
-            'afterFind' => function (array $feeds) {
+            'afterFind' => function (array $feeds) use ($loggedUser) {
                 if ($this->_isSiteAdmin()) {
                     $feeds = $this->Feed->attachFeedCacheTimestamps($feeds);
                 }
@@ -106,6 +108,19 @@ class FeedsController extends AppController
                     }
                 }
 
+                foreach ($feeds as &$feed) {
+                    if (!empty($feed['Feed']['tag_collection_id'])) {
+                        $tagCollection = $this->TagCollection->fetchTagCollection($loggedUser, [
+                            'conditions' => [
+                                'TagCollection.id' => $feed['Feed']['tag_collection_id'],
+                            ]
+                        ]);
+                        if (!empty($tagCollection)) {
+                            $feed['TagCollection'] = $tagCollection;
+                        }
+                    }
+                }
+
                 return $feeds;
             }
         ]);
@@ -294,6 +309,10 @@ class FeedsController extends AppController
         }
         $tags = $this->Event->EventTag->Tag->find('list', array('fields' => array('Tag.name'), 'order' => array('lower(Tag.name) asc')));
         $tags[0] = 'None';
+        $this->loadModel('TagCollection');
+        $tagCollections = $this->TagCollection->fetchTagCollection($this->Auth->user());
+        $tagCollections = Hash::combine($tagCollections, '{n}.TagCollection.id', '{n}.TagCollection.name');
+        $tagCollections[0] = 'None';
 
         $this->loadModel('Server');
         $allTypes = $this->Server->getAllTypes();
@@ -304,6 +323,7 @@ class FeedsController extends AppController
                 'order' => 'LOWER(name)'
             )),
             'tags' => $tags,
+            'tag_collections' => $tagCollections,
             'feedTypes' => $this->Feed->getFeedTypesOptions(),
             'sharingGroups' => $sharingGroups,
             'distributionLevels' => $distributionLevels,
@@ -340,6 +360,7 @@ class FeedsController extends AppController
                 'distribution',
                 'sharing_group_id',
                 'tag_id',
+                'tag_collection_id',
                 'event_id',
                 'publish',
                 'delta_merge',
@@ -442,8 +463,17 @@ class FeedsController extends AppController
         if (empty(Configure::read('Security.disable_local_feed_access'))) {
             $inputSources['local'] = 'Local';
         }
+        $tags = $this->Event->EventTag->Tag->find('all', [
+            'recursive' => -1,
+            'fields' => ['Tag.name', 'Tag.id'],
+            'order' => ['lower(Tag.name) asc']
+        ]);
         $tags = $this->Event->EventTag->Tag->find('list', array('fields' => array('Tag.name'), 'order' => array('lower(Tag.name) asc')));
         $tags[0] = 'None';
+        $this->loadModel('TagCollection');
+        $tagCollections = $this->TagCollection->fetchTagCollection($this->Auth->user());
+        $tagCollections = Hash::combine($tagCollections, '{n}.TagCollection.id', '{n}.TagCollection.name');
+        $tagCollections[0] = 'None';
 
         $this->loadModel('Server');
         $allTypes = $this->Server->getAllTypes();
@@ -457,6 +487,7 @@ class FeedsController extends AppController
                 'order' => 'LOWER(name)'
             )),
             'tags' => $tags,
+            'tag_collections' => $tagCollections,
             'feedTypes' => $this->Feed->getFeedTypesOptions(),
             'sharingGroups' => $sharingGroups,
             'distributionLevels' => $distributionLevels,

app/Controller/GalaxiesController.php

@@ -202,7 +202,7 @@ class GalaxiesController extends AppController
         }
         $result = $this->Galaxy->save($galaxy);
         if ($result) {
-            $message = __('Galaxy enabled');
+            $message = __('Galaxy %s', $enabled ? __('enabled') : __('disabled'));
             if ($this->_isRest()) {
                 return $this->RestResponse->saveSuccessResponse('Galaxy', 'toggle', false, $this->response->type(), $message);
             } else {

app/Controller/GalaxyClustersController.php

@@ -59,6 +59,8 @@ class GalaxyClustersController extends AppController
             $contextConditions['GalaxyCluster.default'] = true;
         } elseif ($filters['context'] == 'custom') {
             $contextConditions['GalaxyCluster.default'] = false;
+        } elseif ($filters['context'] == 'orgc') {
+            $contextConditions['GalaxyCluster.orgc_id'] = $this->Auth->user('org_id');
         } elseif ($filters['context'] == 'org') {
             $contextConditions['GalaxyCluster.org_id'] = $this->Auth->user('org_id');
         } elseif ($filters['context'] == 'deleted') {
@@ -173,6 +175,8 @@ class GalaxyClustersController extends AppController
      */
     public function view($id)
     {
+        $this->GalaxyCluster->includeAnalystData = true;
+        $this->GalaxyCluster->includeAnalystDataRecursive = true;
         $cluster = $this->GalaxyCluster->fetchIfAuthorized($this->Auth->user(), $id, 'view', $throwErrors=true, $full=true);
         $tag = $this->GalaxyCluster->Tag->find('first', array(
             'conditions' => array(
@@ -208,6 +212,7 @@ class GalaxyClustersController extends AppController
         $this->loadModel('Attribute');
         $distributionLevels = $this->Attribute->distributionLevels;
         $this->set('distributionLevels', $distributionLevels);
+        $this->set('shortDist', $this->Attribute->shortDist);
         if (!$cluster['GalaxyCluster']['default'] && !$cluster['GalaxyCluster']['published'] && $cluster['GalaxyCluster']['orgc_id'] == $this->Auth->user()['org_id']) {
             $this->Flash->warning(__('This cluster is not published. Users will not be able to use it'));
         }

app/Controller/JobsController.php

@@ -151,9 +151,12 @@ class JobsController extends AppController
 
     public function cache($type)
     {
-        if (Configure::read('MISP.disable_cached_exports')) {
+        if (Configure::read('MISP.disable_cached_exports', true)) {
             throw new MethodNotAllowedException('This feature is currently disabled');
         }
+        if (!$this->request->is('post')) {
+            throw new MethodNotAllowedException('This endpoint only accept POST.');
+        }
         if ($this->_isSiteAdmin()) {
             $target = 'All events.';
         } else {

app/Controller/LogsController.php

@@ -117,9 +117,15 @@ class LogsController extends AppController
             $this->paginate['conditions']['Log.action'] = $validFilters[$this->params['named']['filter']]['values'];
         }
         foreach ($filters as $key => $value) {
+            if ($key == 'page' || $key == 'limit') { // These should not be part of the condition parameter
+                continue;
+            }
             if ($key === 'created') {
                 $key = 'created >=';
             }
+            if ($key == 'page' || $key == 'limit') {
+                continue;
+            }
             $this->paginate['conditions']["Log.$key"] = $value;
         }
         $this->set('validFilters', $validFilters);

app/Controller/ObjectsController.php

@@ -1410,4 +1410,34 @@ class ObjectsController extends AppController
         }
         return $conditions;
     }
+
+    public function viewAnalystData($id, $seed = null)
+    {
+        $this->MispObject->includeAnalystDataRecursive = true;
+        $object = $this->MispObject->fetchObjects(
+            $this->Auth->user(),
+            [
+                'conditions' => $this->__objectIdToConditions($id)
+            ]
+        );
+        if(empty($object)) {
+            throw new NotFoundException(__('Invalid Object.'));
+        } else {
+            $object[0]['Object'] = array_merge_recursive($object[0]['Object'], $this->MispObject->attachAnalystData($object[0]['Object']));
+        }
+        if ($this->_isRest()) {
+            $validFields = ['Note', 'Opinion', 'Relationship'];
+            $results = [];
+            foreach ($validFields as $field) {
+                if (!empty($object[0]['Object'][$field])) {
+                    $results[$field] = $object[0]['Object'][$field];
+                }
+            }
+            return $this->RestResponse->viewData($results, $this->response->type());
+        }
+        $this->layout = null;
+        $this->set('shortDist', $this->MispObject->Attribute->shortDist);
+        $this->set('object', $object[0]['Object']);
+        $this->set('seed', $seed);
+    }
 }

app/Controller/OrganisationsController.php

@@ -481,13 +481,40 @@ class OrganisationsController extends AppController
             $extension = pathinfo($logo['name'], PATHINFO_EXTENSION);
             $filename = $orgId . '.' . ($extension === 'svg' ? 'svg' : 'png');
 
+            if ($logo['size'] > 250 * 1024) {
+                $this->Flash->error(__('This organisation logo is too large, maximum file size allowed is 250 kB.'));
+                return false;
+            }
+
+            if ($extension !== 'svg' && $extension !== 'png') {
+                $this->Flash->error(__('Invalid file extension, Only PNG and SVG images are allowed.'));
+                return false;
+            }
+            $matches = null;
+            $tmp_name = $logo['tmp_name'];
+            if (preg_match_all('/[\w\/\-\.]*/', $tmp_name, $matches) && file_exists($logo['tmp_name'])) {
+                $tmp_name = $matches[0][0];
+                $imgMime = mime_content_type($tmp_name);
+            } else {
+                throw new NotFoundException(__('Invalid file.'));    
+            }
+            if ($extension === 'png' && (function_exists('exif_imagetype') && !exif_imagetype($logo['tmp_name']))) {
+                $this->Flash->error(__('This is not a valid PNG image.'));
+                return false;
+            }
+
+            if ($extension === 'svg' && !($imgMime === 'image/svg+xml' || $imgMime === 'image/svg')) {
+                $this->Flash->error(__('This is not a valid SVG image.'));
+                return false;
+            }
+
             if ($extension === 'svg' && !Configure::read('Security.enable_svg_logos')) {
                 $this->Flash->error(__('Invalid file extension, SVG images are not allowed.'));
                 return false;
             }
 
-            if (!empty($logo['tmp_name']) && is_uploaded_file($logo['tmp_name'])) {
-                return move_uploaded_file($logo['tmp_name'], APP . 'webroot/img/orgs/' . $filename);
+            if (!empty($tmp_name) && is_uploaded_file($tmp_name)) {
+                return move_uploaded_file($tmp_name, APP . 'files/img/orgs/' . $filename);
             }
         }
 

app/Controller/ServersController.php

@@ -529,7 +529,7 @@ class ServersController extends AppController
             
             if (!$fail) {
                 // say what fields are to be updated
-                $fieldList = array('id', 'url', 'push', 'pull', 'push_sightings', 'push_galaxy_clusters', 'pull_galaxy_clusters', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'remove_missing_tags', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
+                $fieldList = array('id', 'url', 'push', 'pull', 'push_sightings', 'push_galaxy_clusters', 'pull_galaxy_clusters', 'push_analyst_data', 'pull_analyst_data', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'remove_missing_tags', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
                 $this->request->data['Server']['id'] = $id;
                 if (isset($this->request->data['Server']['authkey']) && "" != $this->request->data['Server']['authkey']) {
                     $fieldList[] = 'authkey';
@@ -776,7 +776,7 @@ class ServersController extends AppController
             if (!Configure::read('MISP.background_jobs')) {
                 $result = $this->Server->pull($this->Auth->user(), $technique, $s);
                 if (is_array($result)) {
-                    $success = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4]);
+                    $success = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled, %s analyst data pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4], $result[5]);
                 } else {
                     $error = $result;
                 }
@@ -784,6 +784,7 @@ class ServersController extends AppController
                 $this->set('fails', $result[1]);
                 $this->set('pulledProposals', $result[2]);
                 $this->set('pulledSightings', $result[3]);
+                $this->set('pulledAnalystData', $result[5]);
             } else {
                 $this->loadModel('Job');
                 $jobId = $this->Job->createJob(
@@ -1073,7 +1074,7 @@ class ServersController extends AppController
         );
         $dumpResults = array();
         $tempArray = array();
-        foreach ($finalSettings as $k => $result) {
+        foreach ($finalSettings as $result) {
             if ($result['level'] == 3) {
                 $issues['deprecated']++;
             }
@@ -1105,18 +1106,19 @@ class ServersController extends AppController
         $diagnostic_errors = 0;
         App::uses('File', 'Utility');
         App::uses('Folder', 'Utility');
+
         if ($tab === 'correlations') {
             $this->loadModel('Correlation');
             $correlation_metrics = $this->Correlation->collectMetrics();
             $this->set('correlation_metrics', $correlation_metrics);
-        }
-        if ($tab === 'files') {
+        } else if ($tab === 'files') {
             if (!empty(Configure::read('Security.disable_instance_file_uploads'))) {
                 throw new MethodNotAllowedException(__('This functionality is disabled.'));
             }
             $files = $this->Server->grabFiles();
             $this->set('files', $files);
         }
+
         // Only run this check on the diagnostics tab
         if ($tab === 'diagnostics' || $tab === 'download' || $this->_isRest()) {
             $php_ini = php_ini_loaded_file();
@@ -1279,12 +1281,10 @@ class ServersController extends AppController
         $this->set('workerIssueCount', $workerIssueCount);
         $priorityErrorColours = array(0 => 'red', 1 => 'yellow', 2 => 'green');
         $this->set('priorityErrorColours', $priorityErrorColours);
-        $this->set('phpversion', phpversion());
+        $this->set('phpversion', PHP_VERSION);
         $this->set('phpmin', $this->phpmin);
         $this->set('phprec', $this->phprec);
         $this->set('phptoonew', $this->phptoonew);
-        $this->set('pythonmin', $this->pythonmin);
-        $this->set('pythonrec', $this->pythonrec);
         $this->set('title_for_layout', __('Diagnostics'));
     }
 
@@ -1770,6 +1770,7 @@ class ServersController extends AppController
                 $perm_sighting = isset($result['info']['perm_sighting']) ? $result['info']['perm_sighting'] : false;
                 $local_version = $this->Server->checkMISPVersion();
                 $version = explode('.', $result['info']['version']);
+                $uuid = isset($result['info']['uuid']) ? $result['info']['uuid'] : '?';
                 $mismatch = false;
                 $newer = false;
                 $parts = array('major', 'minor', 'hotfix');
@@ -1805,6 +1806,7 @@ class ServersController extends AppController
                     'response_encoding' => isset($result['post']['content-encoding']) ? $result['post']['content-encoding'] : null,
                     'request_encoding' => isset($result['info']['request_encoding']) ? $result['info']['request_encoding'] : null,
                     'client_certificate' => $result['client_certificate'],
+                    'uuid' => $uuid,
                 ], 'json');
             } else {
                 $result['status'] = 3;
@@ -1863,7 +1865,7 @@ class ServersController extends AppController
         }
 
         if (Configure::read('SimpleBackgroundJobs.enabled')) {
-            $this->Server->getBackgroundJobsTool()->purgeQueue($worker);
+            $this->Server->getBackgroundJobsTool()->clearQueue($worker);
         } else {
             // CakeResque
             $worker_array = array('cache', 'default', 'email', 'prio');
@@ -1888,6 +1890,8 @@ class ServersController extends AppController
             'perm_sync' => (bool) $user['Role']['perm_sync'],
             'perm_sighting' => (bool) $user['Role']['perm_sighting'],
             'perm_galaxy_editor' => (bool) $user['Role']['perm_galaxy_editor'],
+            'perm_analyst_data' => (bool) $user['Role']['perm_analyst_data'],
+            'uuid' => $user['Role']['perm_sync'] ? Configure::read('MISP.uuid') : '-',
             'request_encoding' => $this->CompressedRequestHandler->supportedEncodings(),
             'filter_sightings' => true, // check if Sightings::filterSightingUuidsForPush method is supported
         ];
@@ -2080,7 +2084,10 @@ class ServersController extends AppController
 
     public function updateJSON()
     {
-        $results = $this->Server->updateJSON();
+        $results = [];
+        foreach ($this->Server->updateJSON() as $type => $result) {
+            $results[$type] = $results['success'];
+        }
         return $this->RestResponse->viewData($results, $this->response->type());
     }
 
@@ -2183,7 +2190,7 @@ class ServersController extends AppController
                 if ($this->_isRest()) {
                     return $this->RestResponse->saveFailResponse('Servers', 'addFromJson', false, $this->Server->validationErrors, $this->response->type());
                 } else {
-                    $this->Flash->error(__('Could not save the server. Error: %s', json_encode($this->Server->validationErrors, true)));
+                    $this->Flash->error(__('Could not save the server. Error: %s', json_encode($this->Server->validationErrors)));
                     $this->redirect(array('action' => 'index'));
                 }
             }

app/Controller/SharingGroupBlueprintsController.php

@@ -193,4 +193,91 @@ class SharingGroupBlueprintsController extends AppController
             $this->render('/genericTemplates/confirm');
         }
     }
+
+    public function generateUuidList($id)
+    {
+        $orgs = $this->__getUuidList($id);
+        return $this->RestResponse->viewData($orgs, 'json');
+    }
+
+    private function __getUuidList($id)
+    {
+        $conditions = [];
+        if (empty($id)) {
+            throw new MethodNotAllowedException(__('No ID specified.'));
+        }
+        $conditions['SharingGroupBlueprint.id'] = $id;
+        if (!$this->Auth->user('Role')['perm_admin']) {
+            $conditions['SharingGroupBlueprint.org_id'] = $this->Auth->user('org_id');
+        }
+        $sharingGroupBlueprint = $this->SharingGroupBlueprint->find('first', ['conditions' => $conditions, 'recursive' => -1]);
+        if (empty($sharingGroupBlueprint)) {
+            throw new NotFoundException(__('Invalid Sharing Group Blueprint'));
+        }
+        // we create a fake user to restrict the visible sharing groups to the creator of the SharingGroupBlueprint, in case an admin wants to update it
+        $fake_user = [
+            'Role' => [
+                'perm_site_admin' => false
+            ],
+            'org_id' => $sharingGroupBlueprint['SharingGroupBlueprint']['org_id'],
+            'id' => 1
+        ];
+        $temp = $this->SharingGroupBlueprint->evaluateSharingGroupBlueprint($sharingGroupBlueprint, $fake_user);
+        $orgs = $this->SharingGroupBlueprint->SharingGroup->Organisation->find('list', [
+            'recursive' => -1,
+            'fields' => ['uuid'],
+            'conditions' => ['id' => $temp['orgs']]
+        ]);
+        return array_values($orgs);
+    }
+
+    public function encodeSyncRule($id)
+    {
+        $org_uuids = $this->__getUuidList($id);
+        $this->loadModel('Server');
+        if ($this->request->is('post')) {
+            if (!isset($this->request->data['SharingGroupBlueprint'])) {
+                $this->request->data = ['SharingGroupBlueprint' => $this->request->data];
+            }
+            $server = $this->Server->find('first', [
+                'conditions' => ['Server.id' => $this->request->data['SharingGroupBlueprint']['server_id']],
+                'recursive' => -1
+            ]);
+            if (empty($server)) {
+                throw new NotFoundException(__('Invalid server.'));
+            }
+            $server['Server']['pull_rules'] = json_decode($server['Server']['pull_rules'], true);
+            $server['Server']['push_rules'] = json_decode($server['Server']['push_rules'], true);
+            $rules = [];
+            $type_to_update = empty($this->request->data['SharingGroupBlueprint']['type']) ? 'pull' : $this->request->data['SharingGroupBlueprint']['type'];
+            $rule_to_update = empty($this->request->data['SharingGroupBlueprint']['rule']) ? 'OR' : $this->request->data['SharingGroupBlueprint']['rule'];
+            $rules[$type_to_update][$rule_to_update] = $org_uuids;
+            $server['Server'][$type_to_update . '_rules']['orgs'][$rule_to_update] = $rules[$type_to_update][$rule_to_update];
+            $server['Server']['pull_rules'] = json_encode($server['Server']['pull_rules']);
+            $server['Server']['push_rules'] = json_encode($server['Server']['push_rules']);
+            if (!$this->Server->save($server)) {
+                throw new InvalidArgumentException(__('Could not update the server - something went wrong.'));
+            } else {
+                if ($this->_isRest()) {
+                    $server = $this->Server->find('first', [
+                        'recursive' => -1,
+                        'conditions' => ['Server.id' => $this->request->data['SharingGroupBlueprint']['server_id']]
+                    ]);
+                    return $this->RestResponse->viewData($server, 'json');
+                } else {
+                    $this->Flash->success(__('Server %s\'s %s rules\' %s branch updated with the blueprint\'s rules.', $server['Server']['id'], $type_to_update, $rule_to_update));
+                    $this->redirect('/servers/index');
+                }
+            }
+        }
+        $servers = $this->Server->find('all', ['recursive' => -1]);
+        if (empty($servers)) {
+            throw new NotFoundException(__('No valid servers found.'));
+        }
+        $server_data = [];
+        foreach ($servers as $s) {
+            $server_data[$s['Server']['id']] = $s['Server']['name'];
+        }
+        $this->set('servers', $server_data);
+    }
 }

app/Controller/SightingBlocklistsController.php

@@ -0,0 +1,47 @@
+<?php
+App::uses('AppController', 'Controller');
+
+class SightingBlocklistsController extends AppController
+{
+    public $components = array('Session', 'RequestHandler', 'BlockList');
+
+    public function beforeFilter()
+    {
+        parent::beforeFilter();
+        if (!$this->_isSiteAdmin()) {
+            $this->redirect('/');
+        }
+        if (Configure::check('MISP.enableSightingBlocklisting') && !Configure::read('MISP.enableSightingBlocklisting') !== false) {
+            $this->Flash->info(__('Sighting BlockListing is not currently enabled on this instance.'));
+            $this->redirect('/');
+        }
+    }
+
+    public $paginate = array(
+            'limit' => 60,
+            'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page.
+            'order' => array(
+                    'SightingBlocklist.created' => 'DESC'
+            ),
+    );
+
+    public function index()
+    {
+        return $this->BlockList->index($this->_isRest());
+    }
+
+    public function add()
+    {
+        return $this->BlockList->add($this->_isRest());
+    }
+
+    public function edit($id)
+    {
+        return $this->BlockList->edit($this->_isRest(), $id);
+    }
+
+    public function delete($id)
+    {
+        return $this->BlockList->delete($this->_isRest(), $id);
+    }
+}

app/Controller/TagCollectionsController.php

@@ -357,7 +357,7 @@ class TagCollectionsController extends AppController
             if (!$tagCollection) {
                 throw new NotFoundException(__('Invalid tag collection.'));
             }
-            if ($this->ACL->canModifyTagCollection($this->Auth->user(), $tagCollection)) {
+            if (!$this->ACL->canModifyTagCollection($this->Auth->user(), $tagCollection)) {
                 throw new ForbiddenException(__('You dont have a permission to do that'));
             }
             $tagCollectionTag = $this->TagCollection->TagCollectionTag->find('first', [

app/Controller/TagsController.php

@@ -820,6 +820,15 @@ class TagsController extends AppController
                     }
                     $message = 'Global tag ' . $existingTag['Tag']['name'] . '(' . $existingTag['Tag']['id'] . ') successfully attached to ' . $objectType . '(' . $object[$objectType]['id'] . ').';
                 }
+                $this->loadModel('Log');
+                $this->Log->createLogEntry(
+                    $this->Auth->user(),
+                    'attachTagToObject',
+                    $objectType,
+                    $object[$objectType]['id'],
+                    $message,
+                    null
+                );
                 $successes++;
             } else {
                 $fails[] = __('Failed to attach tag to object.');
@@ -896,6 +905,17 @@ class TagsController extends AppController
         $result = $this->$objectType->$connectorObject->delete($existingAssociation[$connectorObject]['id']);
         if ($result) {
             $message = __('%s tag %s (%s) successfully removed from %s(%s).', $local ? __('Local') : __('Global'), $existingTag['Tag']['name'], $existingTag['Tag']['id'], $objectType, $object[$objectType]['id']);
+            $this->loadModel('Log');
+            $this->Log->createLogEntry(
+                $this->Auth->user(),
+                'removeTagFromObject',
+                $objectType,
+                $object[$objectType]['id'],
+                $message,
+                __(
+                    '',
+                )
+            );
             if (!$local) {
                 if ($objectType === 'Attribute') {
                     $this->Attribute->touch($object['Attribute']['id']);

app/Controller/UserLoginProfilesController.php

@@ -79,7 +79,7 @@ class UserLoginProfilesController extends AppController
                 'fields' => ['UserLoginProfile.*']
             ));
             if (empty($profile)) {
-                throw new NotFoundException(__('Invalid UserLoginProfile'));
+                throw new NotFoundException(__('Invalid user login profile'));
             }
             if ($this->UserLoginProfile->delete($id)) {
                 $this->loadModel('Log');
@@ -87,13 +87,13 @@ class UserLoginProfilesController extends AppController
                 $this->Log->createLogEntry($this->Auth->user(), 'delete', 'UserLoginProfile', $id, $fieldsDescrStr, json_encode($profile));
                 
                 if ($this->_isRest()) {
-                    return $this->RestResponse->saveSuccessResponse('UserLoginProfile', 'admin_delete', $id, $this->response->type(), 'UserLoginProfile deleted.');
+                    return $this->RestResponse->saveSuccessResponse('UserLoginProfile', 'admin_delete', $id, $this->response->type(), 'User login profile deleted.');
                 } else {
                     $this->Flash->success(__('UserLoginProfile deleted'));
                     $this->redirect(array('admin'=> false, 'controller' => 'userLoginProfiles', 'action' => 'index', $profile['UserLoginProfile']['user_id']));
                 }
             }
-            $this->Flash->error(__('UserLoginProfile was not deleted'));
+            $this->Flash->error(__('User login profile was not deleted'));
             $this->redirect(array('admin'=> false, 'controller' => 'userLoginProfiles', 'action' => 'index', $profile['UserLoginProfile']['user_id']));
         }
     }
@@ -110,7 +110,7 @@ class UserLoginProfilesController extends AppController
     {
         if ($this->request->is('post')) {
             $userLoginProfile = $this->__setTrust($logId, 'malicious');
-            $this->Flash->info(__('You marked a login suspicious. You must change your password NOW !'));
+            $this->Flash->info(__('You marked a login suspicious. You must change your password NOW!'));
             $this->loadModel('Log');
             $details = 'User reported suspicious login for log ID: '. $logId;
             // raise an alert (the SIEM component should ensure (org)admins are informed)
@@ -123,9 +123,9 @@ class UserLoginProfilesController extends AppController
                 'recursive' => -1
             ));
             unset($user['User']['password']);
-            $this->UserLoginProfile->email_report_malicious($user, $userLoginProfile);
+            $this->UserLoginProfile->emailReportMalicious($user, $userLoginProfile);
             // change account info to force password change, redirect to new password page.
-            $this->User->id =  $this->Auth->user('id');
+            $this->User->id = $this->Auth->user('id');
             $this->User->saveField('change_pw', 1);
             $this->redirect(array('controller' => 'users', 'action' => 'change_pw'));
             return;
@@ -153,14 +153,13 @@ class UserLoginProfilesController extends AppController
         $data['hash'] = $this->UserLoginProfile->hash($data);
 
         // add the userLoginProfile trust status if it not already there, based on the hash
-        $result = $this->UserLoginProfile->find('count', array(
-            'conditions' => array('UserLoginProfile.hash' => $data['hash'])
-        ));
-        if ($result == 0) {
+        $exists = $this->UserLoginProfile->hasAny([
+            'UserLoginProfile.hash' => $data['hash']
+        ]);
+        if (!$exists) {
             // no row yet, save it. 
             $this->UserLoginProfile->save($data);
         }
         return $data;
     }
-
 }

app/Controller/UsersController.php

@@ -1,5 +1,5 @@
 <?php
-App::uses('AppController', 'Controller', 'OTPHP\TOTP');
+App::uses('AppController', 'Controller');
 
 /**
  * @property User $User
@@ -1214,11 +1214,13 @@ class UsersController extends AppController
                     $this->Auth->constructAuthenticate();
                 }
                 // user has TOTP token, check creds and redirect to TOTP validation
-                if (!empty($unauth_user['User']['totp']) && !$unauth_user['User']['disabled'] && class_exists('\OTPHP\TOTP')) {
-                    $user = $this->Auth->identify($this->request, $this->response);
-                    if ($user && !$user['disabled']) {
-                        $this->Session->write('otp_user', $user);
-                        return $this->redirect('otp');
+                if (!Configure::read('Security.otp_disabled')) {
+                    if (!empty($unauth_user['User']['totp']) && !$unauth_user['User']['disabled'] && class_exists('\OTPHP\TOTP')) {
+                        $user = $this->Auth->identify($this->request, $this->response);
+                        if ($user && !$user['disabled']) {
+                            $this->Session->write('otp_user', $user);
+                            return $this->redirect('otp');
+                        }
                     }
                 }
             }
@@ -1244,8 +1246,6 @@ class UsersController extends AppController
             // login was successful, do everything that is needed such as logging and more:
             $this->_postlogin();
         } else {
-            $dataSourceConfig = ConnectionManager::getDataSource('default')->config;
-            $dataSource = $dataSourceConfig['datasource'];
             // don't display authError before first login attempt
             if (str_replace("//", "/", $this->webroot . $this->Session->read('Auth.redirect')) == $this->webroot && $this->Session->read('Message.auth.message') == $this->Auth->authError) {
                 $this->Session->delete('Message.auth');
@@ -1260,106 +1260,43 @@ class UsersController extends AppController
                 }
             }
 
-            //
-            // Actions needed for the first access, when the database is not populated yet.
-            // 
-
-            // populate the DB with the first role (site admin) if it's empty
-            if (!$this->User->Role->hasAny()) {
-                $siteAdmin = array('Role' => array(
-                    'id' => 1,
-                    'name' => 'Site Admin',
-                    'permission' => 3,
-                    'perm_add' => 1,
-                    'perm_modify' => 1,
-                    'perm_modify_org' => 1,
-                    'perm_publish' => 1,
-                    'perm_sync' => 1,
-                    'perm_admin' => 1,
-                    'perm_audit' => 1,
-                    'perm_auth' => 1,
-                    'perm_site_admin' => 1,
-                    'perm_regexp_access' => 1,
-                    'perm_sharing_group' => 1,
-                    'perm_template' => 1,
-                    'perm_tagger' => 1,
-                ));
-                $this->User->Role->save($siteAdmin);
-                // PostgreSQL: update value of auto incremented serial primary key after setting the column by force
-                if ($dataSource === 'Database/Postgres') {
-                    $sql = "SELECT setval('roles_id_seq', (SELECT MAX(id) FROM roles));";
-                    $this->User->Role->query($sql);
-                }
-            }
-            if (!$this->User->Organisation->hasAny(array('Organisation.local' => true))) {
-                $this->User->runUpdates();
-                $date = date('Y-m-d H:i:s');
-                $org = array('Organisation' => array(
-                        'id' => 1,
-                        'name' => !empty(Configure::read('MISP.org')) ? Configure::read('MISP.org') : 'ADMIN',
-                        'description' => 'Automatically generated admin organisation',
-                        'type' => 'ADMIN',
-                        'uuid' => CakeText::uuid(),
-                        'local' => 1,
-                        'date_created' => $date,
-                        'sector' => '',
-                        'nationality' => ''
-                ));
-                $this->User->Organisation->save($org);
-                // PostgreSQL: update value of auto incremented serial primary key after setting the column by force
-                if ($dataSource === 'Database/Postgres') {
-                    $sql = "SELECT setval('organisations_id_seq', (SELECT MAX(id) FROM organisations));";
-                    $this->User->Organisation->query($sql);
-                }
-                $org_id = $this->User->Organisation->id;
-            }
-            // populate the DB with the first user if it's empty
-            if (!$this->User->hasAny()) {
-                if (!isset($org_id)) {
-                    $hostOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.name' => Configure::read('MISP.org'), 'Organisation.local' => true), 'recursive' => -1));
-                    if (!empty($hostOrg)) {
-                        $org_id = $hostOrg['Organisation']['id'];
-                    } else {
-                        $firstOrg = $this->User->Organisation->find('first', array('conditions' => array('Organisation.local' => true), 'order' => 'Organisation.id ASC'));
-                        $org_id = $firstOrg['Organisation']['id'];
-                    }
-                }
-                $this->User->runUpdates();
-                $this->User->createInitialUser($org_id);
-            }
+            $this->User->init();
         }
     }
 
     private function _postlogin()
     {
-        $this->User->extralog($this->Auth->user(), "login");
+        $authUser = $this->Auth->user();
+        $this->User->extralog($authUser, "login");
+
         $this->User->Behaviors->disable('SysLogLogable.SysLogLogable');
-        $this->User->id = $this->Auth->user('id');
         $user = $this->User->find('first', array(
             'conditions' => array(
-                'User.id' => $this->Auth->user('id')
+                'User.id' => $authUser['id'],
             ),
+            'fields' => ['User.id', 'User.current_login', 'User.last_login'],
             'recursive' => -1
         ));
-        unset($user['User']['password']);
         // update login timestamp and welcome user
         $this->User->updateLoginTimes($user['User']);
-        $lastUserLogin = $user['User']['last_login'];
         $this->User->Behaviors->enable('SysLogLogable.SysLogLogable');
+
+        $lastUserLogin = $user['User']['last_login'];
         if ($lastUserLogin) {
             $readableDatetime = (new DateTime())->setTimestamp($lastUserLogin)->format('D, d M y H:i:s O'); // RFC822
             $this->Flash->info(__('Welcome! Last login was on %s', $readableDatetime));
         }
+
         if (Configure::read('Security.alert_on_suspicious_logins')) {
             try {
                 // there are reasons to believe there is evil happening, suspicious. Inform user and (org)admins.
-                $suspiciousness_reason = $this->User->UserLoginProfile->_isSuspicious();
-                if ($suspiciousness_reason) {
+                $suspiciousnessReason = $this->User->UserLoginProfile->_isSuspicious();
+                if ($suspiciousnessReason) {
                     // raise an alert (the SIEM component should ensure (org)admins are informed)
                     $this->loadModel('Log');
-                    $this->Log->createLogEntry($this->Auth->user(), 'auth_alert', 'User', $this->Auth->user('id'), 'Suspicious login.', $suspiciousness_reason);
+                    $this->Log->createLogEntry($authUser, 'auth_alert', 'User', $authUser['id'], 'Suspicious login.', $suspiciousnessReason);
                     // Line below commented out to NOT inform user/org admin of the suspicious login.
-                    // The reason is that we want to prevent other user actions cause trouble. 
+                    // The reason is that we want to prevent other user actions cause trouble.
                     // However this also means we're sitting on data that could be used to detect new evil logins.
                     // As we're generating alerts, the sysadmin should be keeping an eye on these
                     // $this->User->UserLoginProfile->email_suspicious($user, $suspiciousness_reason);
@@ -1367,11 +1304,12 @@ class UsersController extends AppController
                 // verify UserLoginProfile trust status and perform informative actions
                 if (!$this->User->UserLoginProfile->_isTrusted()) {
                     // send email to inform the user
-                    $this->User->UserLoginProfile->email_newlogin($user);
+                    $this->User->UserLoginProfile->emailNewLogin($authUser);
                 }
             } catch (Exception $e) {
                 // At first login after code update and before DB schema update we might end up with problems.
                 // Just catch it cleanly here to prevent problems.
+                $this->log($e->getMessage(), LOG_WARNING);
             }
         }
 
@@ -1875,6 +1813,7 @@ class UsersController extends AppController
             $this->Flash->error(__("The required PHP libraries to support TOTP are not installed. Please contact your administrator to address this."));
             $this->redirect($this->referer());
         }
+
         // only allow the users themselves to generate a TOTP secret.
         // If TOTP is enforced they will be invited to generate it at first login
         $user = $this->User->find('first', array(
@@ -1944,8 +1883,9 @@ class UsersController extends AppController
         $this->set('secret', $secret);
     }
 
-    public function totp_delete($id) {
-        if ($this->request->is('post') || $this->request->is('delete')) {
+    public function totp_delete($id)
+    {
+        if ($this->request->is(['post', 'delete'])) {
             $user = $this->User->find('first', array(
                 'conditions' => $this->__adminFetchConditions($id),
                 'recursive' => -1,
@@ -2052,8 +1992,7 @@ class UsersController extends AppController
     // shows some statistics about the instance
     public function statistics($page = 'data')
     {
-        $user = $this->Auth->user();
-        @session_write_close(); // loading this page can take long time, so we can close session
+        $user = $this->_closeSession(true); // loading this page can take long time, so we can close session
 
         if (!$this->_isRest()) {
             $pages = [
@@ -2135,7 +2074,7 @@ class UsersController extends AppController
 
         $stats['attribute_count'] = $this->User->Event->Attribute->find('count', array('conditions' => array('Attribute.deleted' => 0), 'recursive' => -1));
         $stats['attribute_count_month'] = $this->User->Event->Attribute->find('count', array('conditions' => array('Attribute.timestamp >' => $this_month, 'Attribute.deleted' => 0), 'recursive' => -1));
-        $stats['attributes_per_event'] = round($stats['attribute_count'] / $stats['event_count']);
+        $stats['attributes_per_event'] = $stats['event_count'] != 0 ? round($stats['attribute_count'] / $stats['event_count']) : 0;
 
         $stats['correlation_count'] = $this->User->Event->Attribute->Correlation->find('count', array('recursive' => -1));
 
@@ -2146,7 +2085,7 @@ class UsersController extends AppController
         $stats['org_count'] = count($orgs);
         $stats['local_org_count'] = $local_orgs_count;
         $stats['contributing_org_count'] = $this->User->Event->find('count', array('recursive' => -1, 'group' => array('Event.orgc_id')));
-        $stats['average_user_per_org'] = round($stats['user_count'] / $stats['local_org_count'], 1);
+        $stats['average_user_per_org'] = $stats['local_org_count'] != 0 ?  round($stats['user_count'] / $stats['local_org_count'], 1) : 0;
 
         $this->loadModel('Thread');
         $stats['thread_count'] = $this->Thread->find('count', array('conditions' => array('Thread.post_count >' => 0), 'recursive' => -1));
@@ -2667,6 +2606,7 @@ class UsersController extends AppController
                 'org_name',
                 'org_uuid',
                 'message',
+                'pgp',
                 'custom_perms',
                 'perm_sync',
                 'perm_publish',
@@ -3065,7 +3005,7 @@ class UsersController extends AppController
      * @return array
      * @throws NotFoundException
      */
-    private function __adminFetchConditions($id, $edit = True)
+    private function __adminFetchConditions($id, $edit = true)
     {
         if (empty($id)) {
             throw new NotFoundException(__('Invalid user'));
@@ -3076,7 +3016,7 @@ class UsersController extends AppController
         if (!$user['Role']['perm_site_admin']) {
             $conditions['User.org_id'] = $user['org_id']; // org admin
             if ($edit) {
-                $conditions['Role.perm_site_admin'] = False;
+                $conditions['Role.perm_site_admin'] = false;
             }
         }
         return $conditions;
@@ -3096,106 +3036,102 @@ class UsersController extends AppController
             }
         }
         if (!empty($conditions)) {
-            $user_ids = $this->User->find('list', [
+            $userIds = $this->User->find('list', [
                 'recursive' => -1,
                 'fields' => ['email', 'id'],
                 'conditions' => $conditions
             ]);
         } else {
-            $user_ids = [__('Every user') => 'all'];
+            $userIds = [__('Every user') => 'all'];
         }
         if ($this->request->is('post')) {
             $redis = RedisTool::init();
-            $kill_before = time();
-            foreach (array_values($user_ids) as $user_id) {
-                $redis->set('misp:session_destroy:' . $user_id, $kill_before);
+            $killBefore = time();
+            foreach ($userIds as $userId) {
+                $redis->set('misp:session_destroy:' . $userId, $killBefore);
             }
             $message = __(
                 'Session destruction cutoff set to the current timestamp for the given selection (%s). Session(s) will be destroyed on the next user interaction.',
-                implode(', ', array_keys($user_ids))
+                implode(', ', array_keys($userIds))
             );
             if ($this->_isRest()) {
-                return $this->RestResponse->saveSuccessResponse('User', 'admin_destroy', false, $this->response->type(), $message);
+                return $this->RestResponse->successResponse(null, $message);
             }
             $this->Flash->success($message);
             $this->redirect($this->referer());
-        } else {
-            $this->set(
-                'question',
-                __(
-                    'Do you really wish to destroy the session for: %s ? The session destruction will occur when the users try to interact with MISP the next time.',
-                    implode(', ', array_keys($user_ids))
-                )
-            );
-            $this->set('title', __('Destroy sessions'));
-            $this->set('actionName', 'Destroy');
-            $this->render('/genericTemplates/confirm');
         }
+
+        $this->set(
+            'question',
+            __(
+                'Do you really wish to destroy the session for: %s? The session destruction will occur when the users try to interact with MISP the next time.',
+                implode(', ', array_keys($userIds))
+            )
+        );
+        $this->set('title', __('Destroy sessions'));
+        $this->set('actionName', 'Destroy');
+        $this->render('/genericTemplates/confirm');
     }
 
-    public function view_login_history($user_id = null) { 
-        if ($user_id && $this->_isAdmin()) {   // org and site admins
-            $user = $this->User->find('first', array(
-                'recursive' => -1,
-                'conditions' => $this->__adminFetchConditions($user_id),
-                'contain' => [
-                    'UserSetting',
-                    'Role',
-                    'Organisation'
-                ]
-            ));
-            if (empty($user)) {
+    public function view_login_history($userId = null)
+    {
+        if ($userId && $this->_isAdmin()) {   // org and site admins
+            $userExists = $this->User->hasAny($this->__adminFetchConditions($userId));
+            if (!$userExists) {
                 throw new NotFoundException(__('Invalid user'));
             }
         } else {
-            $user_id = $this->Auth->user('id');
+            $userId = $this->Auth->user('id');
         }
-        $this->loadModel('UserLoginProfile');
+
         $this->loadModel('Log');
         $logs = $this->Log->find('all', array(
             'conditions' => array(
-                'Log.user_id' => $user_id,
-                'OR' => array ('Log.action' => array('login', 'login_fail', 'auth', 'auth_fail'))
+                'Log.user_id' => $userId,
+                'OR' => array('Log.action' => array('login', 'login_fail', 'auth', 'auth_fail'))
             ),
             'fields' => array('Log.action', 'Log.created', 'Log.ip', 'Log.change', 'Log.id'),
-            'order' => array('Log.created DESC'),
+            'order' => array('Log.id DESC'),
             'limit' => 100          // relatively high limit, as we'll be grouping data afterwards.
         ));
-        $lst = array();
+
+        $profiles = [];
         $prevProfile = null;
         $prevCreatedLast = null;
         $prevCreatedFirst = null;
         $prevLogEntry = null;
         $prevActions = array();
 
-        $actions_translator = [
+        $actionsTranslator = [
             'auth_fail' => 'API:failed',
             'auth' => 'API:login',
             'login' => 'web:login',
             'login_fail' => 'web:failed'
         ];
         
-        $max_rows = 6;  // limit to a few rows, to prevent cluttering the interface. 
+        $maxRows = 6;  // limit to a few rows, to prevent cluttering the interface.
                         // We didn't filter the data at SQL query too much, nor by age, as we want to show "enough" data, even if old
         $rows = 0;
         // group authentications by type of loginprofile, to make the list shorter
-        foreach($logs as $logEntry) {
-            $loginProfile = $this->UserLoginProfile->_fromLog($logEntry['Log']);
-            if (!$loginProfile) continue; // skip if empty log
+        foreach ($logs as $logEntry) {
+            $loginProfile = $this->User->UserLoginProfile->_fromLog($logEntry['Log']);
+            if (!$loginProfile) {
+                continue; // skip if empty log
+            }
             $loginProfile['ip'] = $logEntry['Log']['ip'] ?? null; // transitional workaround
-            if ($this->UserLoginProfile->_isSimilar($loginProfile, $prevProfile)) {
+            if ($this->User->UserLoginProfile->_isSimilar($loginProfile, $prevProfile)) {
                 // continue find as same type of login
                 $prevCreatedFirst = $logEntry['Log']['created'];
-                $prevActions[] = $actions_translator[$logEntry['Log']['action']] ?? $logEntry['Log']['action'];
+                $prevActions[] = $actionsTranslator[$logEntry['Log']['action']];
             } else {
                 // add as new entry
-                if (null != $prevProfile) {
+                if (null !== $prevProfile) {
                     $actionsString = '';  // count actions
-                    foreach(array_count_values($prevActions) as $action => $cnt) {
+                    foreach (array_count_values($prevActions) as $action => $cnt) {
                         $actionsString .=  $action . ' (' . $cnt . "x) ";
                     }
-                    $lst[] = array(
-                        'status' => $this->UserLoginProfile->_getTrustStatus($prevProfile, $user_id),
+                    $profiles[] = [
+                        'status' => $this->User->UserLoginProfile->_getTrustStatus($prevProfile, $userId),
                         'platform' => $prevProfile['ua_platform'],
                         'browser' => $prevProfile['ua_browser'],
                         'region' => $prevProfile['geoip'],
@@ -3204,40 +3140,47 @@ class UsersController extends AppController
                         'last_seen' => $prevCreatedLast,
                         'first_seen' => $prevCreatedFirst,
                         'actions' => $actionsString,
-                        'actions_button' => ('unknown' == $this->UserLoginProfile->_getTrustStatus($prevProfile, $user_id)) ? true : false,
-                        'id' => $prevLogEntry);
+                        'actions_button' => ('unknown' == $this->User->UserLoginProfile->_getTrustStatus($prevProfile, $userId)) ? true : false,
+                        'id' => $prevLogEntry
+                    ];
                 }
                 // build new entry
                 $prevProfile = $loginProfile;
                 $prevCreatedFirst = $prevCreatedLast = $logEntry['Log']['created'];
-                $prevActions[] = $actions_translator[$logEntry['Log']['action']] ?? $logEntry['Log']['action'];
+                $prevActions[] = $actionsTranslator[$logEntry['Log']['action']];
                 $prevLogEntry = $logEntry['Log']['id'];
-                $rows += 1;
-                if ($rows == $max_rows) break;
+                $rows++;
+                if ($rows === $maxRows) {
+                    break;
+                }
             }
         }
         // add last entry
-        $actionsString = '';  // count actions
-        foreach(array_count_values($prevActions) as $action => $cnt) {
-            $actionsString .=  $action . ' (' . $cnt . "x) ";
+        if (null !== $prevProfile) {
+            $actionsString = '';  // count actions
+            foreach (array_count_values($prevActions) as $action => $cnt) {
+                $actionsString .= $action . ' (' . $cnt . "x) ";
+            }
+            $profiles[] = array(
+                'status' => $this->User->UserLoginProfile->_getTrustStatus($prevProfile, $userId),
+                'platform' => $prevProfile['ua_platform'],
+                'browser' => $prevProfile['ua_browser'],
+                'region' => $prevProfile['geoip'],
+                'ip' => $prevProfile['ip'],
+                'accept_lang' => $prevProfile['accept_lang'],
+                'last_seen' => $prevCreatedLast,
+                'first_seen' => $prevCreatedFirst,
+                'actions' => $actionsString,
+                'actions_button' => ('unknown' == $this->User->UserLoginProfile->_getTrustStatus($prevProfile, $userId)) ? true : false,
+                'id' => $prevLogEntry
+            );
         }
-        $lst[] = array(
-            'status' => $this->UserLoginProfile->_getTrustStatus($prevProfile, $user_id),
-            'platform' => $prevProfile['ua_platform'],
-            'browser' => $prevProfile['ua_browser'],
-            'region' => $prevProfile['geoip'],
-            'ip' =>  $prevProfile['ip'],
-            'accept_lang' => $prevProfile['accept_lang'],
-            'last_seen' => $prevCreatedLast,
-            'first_seen' => $prevCreatedFirst,
-            'actions' => $actionsString,
-            'actions_button' => ('unknown' == $this->UserLoginProfile->_getTrustStatus($prevProfile, $user_id)) ? true : false,
-            'id' => $prevLogEntry);
-        $this->set('data', $lst);
-        $this->set('user_id', $user_id);
+        $this->set('data', $profiles);
+        $this->set('user_id', $userId);
     }
 
-    public function logout401() {
+    public function logout401()
+    {
         # You should read the documentation in docs/CONFIG.ApacheSecureAuth.md
         # before using this endpoint. It is not useful without webserver config
         # changes.
@@ -3247,10 +3190,6 @@ class UsersController extends AppController
 
     public function forgot()
     {
-        if (empty(Configure::read('Security.allow_password_forgotten'))) {
-            $this->Flash->error(__('This feature is disabled.'));
-            $this->redirect('/');
-        }
         if (!empty($this->Auth->user()) && !$this->_isRest()) {
             $this->Flash->info(__('You are already logged in, no need to ask for a password reset. Log out first.'));
             $this->redirect('/');
@@ -3262,16 +3201,9 @@ class UsersController extends AppController
             if (empty($this->request->data['User']['email'])) {
                 throw new MethodNotAllowedException(__('No email provided, cannot generate password reset message.'));
             }
-            $user = [
-                'id' => 0,
-                'email' => 'SYSTEM',
-                'Organisation' => [
-                    'name' => 'SYSTEM'
-                ]
-            ];
             $this->loadModel('Log');
-            $this->Log->createLogEntry($user, 'forgot', 'User', 0, 'Password reset requested for: ' . $this->request->data['User']['email']);
-            $this->User->forgotRouter($this->request->data['User']['email'], $this->_remoteIp());
+            $this->Log->createLogEntry('SYSTEM', 'forgot', 'User', 0, 'Password reset requested for: ' . $this->request->data['User']['email']);
+            $this->User->forgotRouter($this->request->data['User']['email'], $this->User->_remoteIp());
             $message = __('Password reset request submitted. If a valid user is found, you should receive an e-mail with a temporary reset link momentarily. Please be advised that this link is only valid for 10 minutes.');
             if ($this->_isRest()) {
                 return $this->RestResponse->saveSuccessResponse('User', 'forgot', false, $this->response->type(), $message);
@@ -3283,10 +3215,6 @@ class UsersController extends AppController
 
     public function password_reset($token)
     {
-        if (empty(Configure::read('Security.allow_password_forgotten'))) {
-            $this->Flash->error(__('This feature is disabled.'));
-            $this->redirect('/');
-        }
         $this->loadModel('Server');
         $this->set('complexity', !empty(Configure::read('Security.password_policy_complexity')) ? Configure::read('Security.password_policy_complexity') : $this->Server->serverSettings['Security']['password_policy_complexity']['value']);
         $this->set('length', !empty(Configure::read('Security.password_policy_length')) ? Configure::read('Security.password_policy_length') : $this->Server->serverSettings['Security']['password_policy_length']['value']);
@@ -3303,10 +3231,9 @@ class UsersController extends AppController
                 $this->redirect('/');
             }
         }
-        if ($this->request->is('post') || $this->request->is('put')) {
+        if ($this->request->is(['post', 'put'])) {
             $abortPost = false;
             return $this->__pw_change(['User' => $user], 'password_reset', $abortPost, $token, true);
         }
     }
-
 }

app/Error/AppExceptionRenderer.php

@@ -0,0 +1,77 @@
+<?php
+
+App::uses('ExceptionRenderer', 'Error');
+
+class AppExceptionRenderer extends ExceptionRenderer {
+
+    public function __construct($exception) {
+        $this->controller = $this->_getController($exception);
+
+        if (method_exists($this->controller, 'appError')) {
+            $this->controller->appError($exception);
+            return;
+        }
+        $method = $template = Inflector::variable(str_replace('Exception', '', get_class($exception)));
+        $code = $exception->getCode();
+
+        $methodExists = method_exists($this, $method);
+        if ($exception instanceof CakeException && !$methodExists) {
+            $this->_customErrorLogging($exception);
+            $method = '_cakeError';
+            if (empty($template) || $template === 'internalError') {
+                $template = 'error500';
+            }
+        } elseif ($exception instanceof PDOException) {
+            $method = 'pdoError';
+            $template = 'pdo_error';
+            $code = 500;
+        } elseif (!$methodExists) {
+            $method = 'error500';
+            if ($code >= 400 && $code < 500) {
+                $method = 'error400';
+            }
+        }
+
+        $isNotDebug = !Configure::read('debug');
+        if ($isNotDebug && $method === '_cakeError') {
+            $method = 'error400';
+        }
+        if ($isNotDebug && $code == 500) {
+            $method = 'error500';
+        }
+        $this->template = $template;
+        $this->method = $method;
+        $this->error = $exception;
+    }
+
+    protected function _customErrorLogging($exception): bool
+    {
+        $message = $exception->getMessage();
+        $errorDetection = [
+            'Maximum execution time of' => 'timeout',
+            'Allowed memory size of' => 'out of memory'
+        ];
+        $user = $this->controller->Auth->user();
+        $ua = env('HTTP_USER_AGENT');
+        $source = $this->controller->IndexFilter->isRest() ? 'API' : 'web UI';
+        if (strpos($ua, 'MISP ') !== false) {
+            $source = 'MISP sync';
+        }
+        foreach ($errorDetection as $search => $errorName) {
+            if (strpos($message, $search) !== false) {
+                $logMessage = sprintf(
+                    '%s %s error triggered by User %s (%s) via the %s on %s.',
+                    date('Y-m-d H:i:s'),
+                    $errorName,
+                    $user['id'],
+                    $user['email'],
+                    $source,
+                    $this->controller->request->here()
+                );
+                file_put_contents(LOGS . 'fatal_error.log', $logMessage . PHP_EOL, FILE_APPEND);
+                return true;
+            }
+        }
+        return true;
+    }
+}

app/Lib/Dashboard/AchievementsWidget.php

@@ -191,4 +191,3 @@ class AchievementsWidget
         return $result;
     }
 }
-?>

app/Lib/Dashboard/AttackWidget.php

@@ -35,4 +35,3 @@ class AttackWidget
         return $data;
     }
 }
-?>

app/Lib/Dashboard/BenchmarkTopListWidget.php

@@ -0,0 +1,115 @@
+<?php
+
+
+class BenchmarkTopListWidget
+{
+    public $title = 'Benchmark top list';
+    public $render = 'MultiLineChart';
+    public $width = 3;
+    public $height = 3;
+    public $description = 'A graph showing the top list for a given scope and field in the captured metrics.';
+    public $cacheLifetime = false;
+    public $autoRefreshDelay = 30;
+    public $params = array(
+        'days' => 'Number of days to consider for the graph. There will be a data entry for each day (assuming the benchmarking has been enabled). Defaults to returning all data.',
+        'weeks' => 'Number of weeks to consider for the graph. There will be a data entry for each day (assuming the benchmarking has been enabled). Defaults to returning all data.',
+        'months' => 'Number of months to consider for the graph. There will be a data entry for each day (assuming the benchmarking has been enabled). Defaults to returning all data.',
+        'scope' => 'The scope of the benchmarking refers to what was being tracked. The following scopes are valid: user, endpoint, user_agent',
+        'field' => 'The individual metric to be queried from the benchmark results. Valid values are: time, sql_time, sql_queries, memory, endpoint',
+        'average' => 'If you wish to view the averages per scope/field, set this variable to true. It will divide the result by the number of executions recorded for the scope/field combination for the given day.'
+    );
+    public $Benchmark;
+    public $User;
+
+    public $placeholder =
+        '{
+    "days": "30",
+    "scope": "endpoints",
+    "field": "sql_time"
+}';
+
+    public function handler($user, $options = array())
+    {
+        $this->User = ClassRegistry::init('User');
+        $currentTime = strtotime("now");
+        $endOfDay = strtotime("tomorrow", $currentTime) - 1;
+        if (!empty($options['days'])) {
+            $limit = (int)($options['days']);
+            $delta = 'day';
+        } else if (!empty($options['weeks'])) {
+            $limit = (int)($options['weeks']);
+            $delta = 'week';
+        } else if (!empty($options['months'])) {
+            $limit = (int)($options['months']);
+            $delta = 'month';
+        } else {
+            $limit = 30;
+            $delta = 'day';
+        }
+        $axis_info = [
+            'time' => 'Total time taken (ms)',
+            'sql_time' => 'SQL time taken (ms)',
+            'sql_queries' => 'Queries (#)',
+            'memory' => 'Memory (MB)',
+            'endpoint' => 'Queries to endpoint (#)'
+        ];
+        $y_axis = $axis_info[isset($options['field']) ? $options['field'] : 'time'];
+        $data = ['y-axis' => $y_axis];
+        $data['data'] = array();
+        // Add total users data for all timestamps
+        
+        for ($i = 0; $i < $limit; $i++) {
+            $itemTime = strtotime('- ' . $i . $delta, $endOfDay);
+            $item = array();
+            $date = strftime('%Y-%m-%d', $itemTime);
+            $item = $this->getData($date, $options);
+            if (!empty($item)) {
+                $item['date'] = $date;
+                $data['data'][] = $item;
+            }
+        }
+        $keys = [];
+        foreach ($data['data'] as $day_data) {
+            foreach ($day_data as $key => $temp) {
+                $keys[$key] = 1;
+            }
+        }
+        $keys = array_keys($keys);
+        foreach ($data['data'] as $k => $day_data) {
+            foreach ($keys as $key) {
+                if (!isset($day_data[$key])) {
+                    $data['data'][$k][$key] = 0;
+                }
+            }
+            foreach ($day_data as $key => $temp) {
+                $keys[$key] = 1;
+            }
+        }
+        return $data;
+    }
+
+    private function getData($time, $options)
+    {
+        $dates = [$time];
+        $this->Benchmark = new BenchmarkTool($this->User);
+        $result = $this->Benchmark->getTopList(
+            isset($options['scope']) ? $options['scope'] : 'endpoint',
+            isset($options['field']) ? $options['field'] : 'memory',
+            $dates,
+            isset($options['limit']) ? $options['limit'] : 5,
+            isset($options['average']) ? $options['average'] : false,
+        );
+        if (!empty($result)) {
+            return $result[$time];
+        }
+        return false;
+    }
+
+    public function checkPermissions($user)
+    {
+        if (empty($user['Role']['perm_site_admin'])) {
+            return false;
+        }
+        return true;
+    }
+}

app/Lib/Dashboard/EventEvolutionLineWidget.php

@@ -58,9 +58,11 @@ class EventEvolutionLineWidget
             'recursive' => -1
         ];
         $eparams = [];
+        $filteringOnOrg = false;
         if (!empty($options['filter']) && is_array($options['filter'])) {
             foreach ($this->validFilterKeys as $filterKey) {
                 if (!empty($options['filter'][$filterKey])) {
+                    $filteringOnOrg = true;
                     if (!is_array($options['filter'][$filterKey])) {
                         $options['filter'][$filterKey] = [$options['filter'][$filterKey]];
                     }
@@ -87,6 +89,9 @@ class EventEvolutionLineWidget
             'conditions' => $oparams['conditions'],
             'fields' => ['id']
         ]);
+        if ($filteringOnOrg) {
+            $eparams['conditions']['AND']['Event.orgc_id IN'] = !empty($org_ids) ? $org_ids : [-1];
+        }
         $this->Event->virtualFields = [
             'published_date' => null
         ];

app/Lib/Dashboard/MispAdminSyncTestWidget.php

@@ -37,6 +37,10 @@ class MispAdminSyncTestWidget
                     $colour = 'orange';
                     $message .= ' ' . __('No sighting access.');
                 }
+                if (empty($result['info']['perm_analyst_data'])) {
+                    $colour = 'orange';
+                    $message .= ' ' . __('No analyst data sync access.');
+                }
             } else {
                 $colour = 'red';
                 $message = $syncTestErrorCodes[$result['status']];

app/Lib/Dashboard/OrgContributionToplistWidget.php

@@ -137,4 +137,3 @@ class OrgContributionToplistWidget
         return ['data' => $results];
     }
 }
-?>

app/Lib/Dashboard/OrganisationListWidget.php

@@ -78,4 +78,3 @@ class OrganisationListWidget
         return ['data' => $results];
     }
 }
-?>

app/Lib/Dashboard/OrganisationMapWidget.php

@@ -85,4 +85,3 @@ class OrganisationMapWidget
         return $results;
     }
 }
-?>

app/Lib/Dashboard/OrgsContributorLastMonthWidget.php

@@ -19,4 +19,3 @@ class OrgsContributorLastMonthWidget extends OrgsContributorsGeneric
         return count($results) > 0;
     }
 }
- ?>

app/Lib/Dashboard/OrgsContributorsGeneric.php

@@ -45,4 +45,3 @@ class OrgsContributorsGeneric
         return $result;
     }
 }
-?>

app/Lib/Dashboard/OrgsUsingMitreWidget.php

@@ -32,4 +32,3 @@ class OrgsUsingMitreWidget extends OrgsContributorsGeneric
         return count($events) > 0;
     }
 }
-?>

app/Lib/Dashboard/OrgsUsingObjectsWidget.php

@@ -25,4 +25,3 @@ class OrgsUsingObjectsWidget extends OrgsContributorsGeneric
         return count($eventsIds) > 0;
     }
 }
-?>

app/Lib/Dashboard/UsageDataWidget.php

@@ -243,7 +243,8 @@ class UsageDataWidget
             $count = $this->Event->Attribute->find('count', [
                 'conditions' => $conditions,
                 'contain' => ['Event'],
-                'recursive' => -1
+                'recursive' => -1,
+                'ignoreIndexHint' => 'deleted'
             ]);
             $this->redis->setEx('misp:dashboard:attribute_count:' . $hash, 3600, $count);
         }

app/Lib/Dashboard/UserContributionToplistWidget.php

@@ -145,4 +145,3 @@ class UserContributionToplistWidget
         return true;
     }
 }
-?>

app/Lib/Export/NidsExport.php

@@ -6,10 +6,7 @@ abstract class NidsExport
 
     public $classtype = 'trojan-activity';
 
-    public $format = "";   // suricata (default), snort
-    
-
-    public $checkWhitelist = true;
+    protected $format;   // suricata (default), snort
 
     public $additional_params = array(
         'contain' => array(
@@ -17,36 +14,31 @@ abstract class NidsExport
                 'fields' => array('threat_level_id')
             )
         ),
-
     );
 
     public function handler($data, $options = array())
     {
-        $continue = empty($format);
-        $this->checkWhitelist = false;
         if ($options['scope'] === 'Attribute') {
             $this->export(
                 array($data),
-                $options['user']['nids_sid'],
-                $options['returnFormat'],
-                $continue
+                $options['user']['nids_sid']
             );
         } else if ($options['scope'] === 'Event') {
             if (!empty($data['EventTag'])) {
                 $data['Event']['EventTag'] = $data['EventTag'];
             }
             if (!empty($data['Attribute'])) {
-                $this->__convertFromEventFormat($data['Attribute'], $data, $options, $continue);
+                $this->convertFromEventFormat($data['Attribute'], $data, $options);
             }
             if (!empty($data['Object'])) {
-                $this->__convertFromEventFormatObject($data['Object'], $data, $options, $continue);
+                $this->convertFromEventFormatObject($data['Object'], $data, $options);
             }
         }
         return '';
     }
 
-    private function __convertFromEventFormat($attributes, $event, $options = array(), $continue = false) {
-
+    private function convertFromEventFormat($attributes, $event, $options = array())
+    {
         $rearranged = array();
         foreach ($attributes as $attribute) {
             $attributeTag = array();
@@ -62,15 +54,12 @@ abstract class NidsExport
         }
         $this->export(
             $rearranged,
-            $options['user']['nids_sid'],
-            $options['returnFormat'],
-            $continue
+            $options['user']['nids_sid']
         );
         return true;
-
     }
 
-    private function __convertFromEventFormatObject($objects, $event, $options = array(), $continue = false)
+    private function convertFromEventFormatObject($objects, $event, $options = array())
     {
         $rearranged = array();
         foreach ($objects as $object) {
@@ -93,20 +82,18 @@ abstract class NidsExport
                     'Event' => $event['Event']
                 );
             } else { // In case no custom export exists for the object, the approach falls back to the attribute case
-                $this->__convertFromEventFormat($object['Attribute'], $event, $options, $continue);
+                $this->convertFromEventFormat($object['Attribute'], $event, $options);
             }
         }
 
         $this->export(
             $rearranged,
-            $options['user']['nids_sid'],
-            $options['returnFormat'],
-            $continue
+            $options['user']['nids_sid']
         );
         return true;
     }
 
-    public function header($options = array())
+    public function header()
     {
         $this->explain();
         return '';
@@ -122,7 +109,7 @@ abstract class NidsExport
         return '';
     }
 
-    public function explain()
+    protected function explain()
     {
         $this->rules[] = '# MISP export of IDS rules - optimized for '.$this->format;
         $this->rules[] = '#';
@@ -136,21 +123,8 @@ abstract class NidsExport
         $this->rules[] = '# ';
     }
 
-    private $whitelist = null;
-
-
-    public function export($items, $startSid, $format="suricata", $continue = false)
+    protected function export($items, $startSid)
     {
-        $this->format = $format;
-        if ($this->checkWhitelist && !isset($this->Whitelist)) {
-            $this->Whitelist = ClassRegistry::init('Whitelist');
-            $this->whitelist = $this->Whitelist->getBlockedValues();
-        }
-
-        // output a short explanation
-        if (!$continue) {
-            $this->explain();
-        }
         // generate the rules
         foreach ($items as $item) {
             // retrieve all tags for this item to add them to the msg
@@ -180,7 +154,6 @@ abstract class NidsExport
             $sid++;
 
             if (!empty($item['Attribute']['type'])) { // item is an 'Attribute'
-            
                 switch ($item['Attribute']['type']) {
                     // LATER nids - test all the snort attributes
                     // LATER nids - add the tag keyword in the rules to capture network traffic
@@ -195,7 +168,7 @@ abstract class NidsExport
                         break;
                     case 'email':
                         $this->emailSrcRule($ruleFormat, $item['Attribute'], $sid);
-			$sid++;
+			            $sid++;
                         $this->emailDstRule($ruleFormat, $item['Attribute'], $sid);
                         break;
                     case 'email-src':
@@ -228,17 +201,17 @@ abstract class NidsExport
                     case 'ja3-fingerprint-md5':
                         $this->ja3Rule($ruleFormat, $item['Attribute'], $sid);
                         break;
-                    case 'ja3s-fingerprint-md5': // Atribute type doesn't exists yet (2020-12-10) but ready when created.
+                    case 'ja3s-fingerprint-md5': // Attribute type doesn't exists yet (2020-12-10) but ready when created.
                         $this->ja3sRule($ruleFormat, $item['Attribute'], $sid);
                         break;
                     case 'snort':
-                        $this->snortRule($ruleFormat, $item['Attribute'], $sid, $ruleFormatMsg, $ruleFormatReference);
+                        $this->snortRule($item['Attribute'], $sid, $ruleFormatMsg, $ruleFormatReference);
                         // no break
                     default:
                         break;
                 }
 
-            } else if(!empty($item['Attribute']['name'])) { // Item is an 'Object'
+            } else if (!empty($item['Attribute']['name'])) { // Item is an 'Object'
 
                 switch ($item['Attribute']['name']) {
                     case 'network-connection':
@@ -252,34 +225,30 @@ abstract class NidsExport
                 }
 
             }
-
         }
-		
-        return $this->rules;
     }
 
-    public function networkConnectionRule($ruleFormat, $object, &$sid)
+    protected function networkConnectionRule($ruleFormat, $object, &$sid)
     {
-    
         $attributes = NidsExport::getObjectAttributes($object);
         
-        if(!array_key_exists('layer4-protocol', $attributes)){
+        if (!array_key_exists('layer4-protocol', $attributes)) {
             $attributes['layer4-protocol'] = 'ip';  // If layer-4 protocol is unknown, we roll-back to layer-3 ('ip')
         }
-        if(!array_key_exists('ip-src', $attributes)){
+        if (!array_key_exists('ip-src', $attributes)) {
             $attributes['ip-src'] = '$HOME_NET';    // If ip-src is unknown, we roll-back to $HOME_NET
         }
-        if(!array_key_exists('ip-dst', $attributes)){
+        if (!array_key_exists('ip-dst', $attributes)) {
             $attributes['ip-dst'] = '$HOME_NET';    // If ip-dst is unknown, we roll-back to $HOME_NET
         }
-        if(!array_key_exists('src-port', $attributes)){
+        if (!array_key_exists('src-port', $attributes)) {
             $attributes['src-port'] = 'any';    // If src-port is unknown, we roll-back to 'any'
         }
-        if(!array_key_exists('dst-port', $attributes)){
+        if (!array_key_exists('dst-port', $attributes)) {
             $attributes['dst-port'] = 'any';    // If dst-port is unknown, we roll-back to 'any'
         }
 
-    $this->rules[] = sprintf(
+        $this->rules[] = sprintf(
             $ruleFormat,
             false,
             $attributes['layer4-protocol'],                     // proto
@@ -294,12 +263,10 @@ abstract class NidsExport
             $sid,                                               // sid
             1                                                   // rev
             );
-        
     }
-    
-    public function ddosRule($ruleFormat, $object, &$sid)
+
+    protected function ddosRule($ruleFormat, $object, &$sid)
     {
-    
         $attributes = NidsExport::getObjectAttributes($object);
         
         if(!array_key_exists('protocol', $attributes)){
@@ -318,7 +285,7 @@ abstract class NidsExport
             $attributes['dst-port'] = 'any';    // If dst-port is unknown, we roll-back to 'any'
         }
 
-    $this->rules[] = sprintf(
+        $this->rules[] = sprintf(
             $ruleFormat,
             false,
             $attributes['protocol'],                            // proto
@@ -333,12 +300,10 @@ abstract class NidsExport
             $sid,                                               // sid
             1                                                   // rev
             );
-        
     }
-    
-    public static function getObjectAttributes($object)
+
+    protected static function getObjectAttributes($object)
     {
-        
         $attributes = array();
         
         foreach ($object['Attribute'] as $attribute) {
@@ -348,7 +313,7 @@ abstract class NidsExport
         return $attributes;
     }
 
-    public function domainIpRule($ruleFormat, $attribute, &$sid)
+    protected function domainIpRule($ruleFormat, $attribute, &$sid)
     {
         $values = explode('|', $attribute['value']);
         $attributeCopy = $attribute;
@@ -361,7 +326,7 @@ abstract class NidsExport
         $this->ipSrcRule($ruleFormat, $attributeCopy, $sid);
     }
 
-    public function ipDstRule($ruleFormat, $attribute, &$sid)
+    protected function ipDstRule($ruleFormat, $attribute, &$sid)
     {
         $overruled = $this->checkWhitelist($attribute['value']);
         $ipport = NidsExport::getIpPort($attribute);
@@ -382,7 +347,7 @@ abstract class NidsExport
                 );
     }
 
-    public function ipSrcRule($ruleFormat, $attribute, &$sid)
+    protected function ipSrcRule($ruleFormat, $attribute, &$sid)
     {
         $overruled = $this->checkWhitelist($attribute['value']);
         $ipport = NidsExport::getIpPort($attribute);
@@ -403,7 +368,7 @@ abstract class NidsExport
                 );
     }
 
-    public function emailSrcRule($ruleFormat, $attribute, &$sid)
+    protected function emailSrcRule($ruleFormat, $attribute, &$sid)
     {
         $overruled = $this->checkWhitelist($attribute['value']);
         $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']);  // substitute chars not allowed in rule
@@ -425,7 +390,7 @@ abstract class NidsExport
                 );
     }
 
-    public function emailDstRule($ruleFormat, $attribute, &$sid)
+    protected function emailDstRule($ruleFormat, $attribute, &$sid)
     {
         $overruled = $this->checkWhitelist($attribute['value']);
         $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']);  // substitute chars not allowed in rule
@@ -447,7 +412,7 @@ abstract class NidsExport
                 );
     }
 
-    public function emailSubjectRule($ruleFormat, $attribute, &$sid)
+    protected function emailSubjectRule($ruleFormat, $attribute, &$sid)
     {
         // LATER nids - email-subject rule might not match because of line-wrapping
         $overruled = $this->checkWhitelist($attribute['value']);
@@ -470,7 +435,7 @@ abstract class NidsExport
                 );
     }
 
-    public function emailAttachmentRule($ruleFormat, $attribute, &$sid)
+    protected function emailAttachmentRule($ruleFormat, $attribute, &$sid)
     {
         // LATER nids - email-attachment rule might not match because of line-wrapping
         $overruled = $this->checkWhitelist($attribute['value']);
@@ -493,7 +458,7 @@ abstract class NidsExport
                 );
     }
 
-    public function hostnameRule($ruleFormat, $attribute, &$sid)
+    protected function hostnameRule($ruleFormat, $attribute, &$sid)
     {
         $overruled = $this->checkWhitelist($attribute['value']);
         $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']);  // substitute chars not allowed in rule
@@ -549,7 +514,7 @@ abstract class NidsExport
         );
     }
 
-    public function domainRule($ruleFormat, $attribute, &$sid)
+    protected function domainRule($ruleFormat, $attribute, &$sid)
     {
         $overruled = $this->checkWhitelist($attribute['value']);
         $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']);  // substitute chars not allowed in rule
@@ -605,7 +570,7 @@ abstract class NidsExport
         );
     }
 
-    public function urlRule($ruleFormat, $attribute, &$sid)
+    protected function urlRule($ruleFormat, $attribute, &$sid)
     {
         // TODO in hindsight, an url should not be excluded given a host or domain name.
         //$hostpart = parse_url($attribute['value'], PHP_URL_HOST);
@@ -630,7 +595,7 @@ abstract class NidsExport
                 );
     }
 
-    public function userAgentRule($ruleFormat, $attribute, &$sid)
+    protected function userAgentRule($ruleFormat, $attribute, &$sid)
     {
         $overruled = $this->checkWhitelist($attribute['value']);
         $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']);  // substitute chars not allowed in rule
@@ -652,17 +617,17 @@ abstract class NidsExport
         );
     }
 
-    public function ja3Rule($ruleFormat, $attribute, &$sid)
+    protected function ja3Rule($ruleFormat, $attribute, &$sid)
     {
         //Empty because Snort doesn't support JA3 Rules
     }
 
-    public function ja3sRule($ruleFormat, $attribute, &$sid)
+    protected function ja3sRule($ruleFormat, $attribute, &$sid)
     {
         //Empty because Snort doesn't support JA3S Rules
     }
 
-    public function snortRule($ruleFormat, $attribute, &$sid, $ruleFormatMsg, $ruleFormatReference)
+    protected function snortRule($attribute, &$sid, $ruleFormatMsg, $ruleFormatReference)
     {
         // LATER nids - test using lots of snort rules, some rules don't contain all the necessary to be a valid rule.
 
@@ -678,46 +643,46 @@ abstract class NidsExport
         //   tag       - '/tag\s*:\s*.+?;/'
         $replaceCount = array();
         $tmpRule = preg_replace('/sid\s*:\s*[0-9]+\s*;/', 'sid:' . $sid . ';', $tmpRule, -1, $replaceCount['sid']);
-        if (null == $tmpRule) {
+        if (null === $tmpRule) {
             return false;
         }   // don't output the rule on error with the regex
         $tmpRule = preg_replace('/rev\s*:\s*[0-9]+\s*;/', 'rev:1;', $tmpRule, -1, $replaceCount['rev']);
-        if (null == $tmpRule) {
+        if (null === $tmpRule) {
             return false;
         }   // don't output the rule on error with the regex
         $tmpRule = preg_replace('/classtype:[a-zA-Z_-]+;/', 'classtype:' . $this->classtype . ';', $tmpRule, -1, $replaceCount['classtype']);
-        if (null == $tmpRule) {
+        if (null === $tmpRule) {
             return false;
         }   // don't output the rule on error with the regex
         $tmpRule = preg_replace('/msg\s*:\s*"(.*?)"\s*;/', sprintf($ruleFormatMsg, 'snort-rule | $1') . ';', $tmpRule, -1, $replaceCount['msg']);
-        if (null == $tmpRule) {
+        if (null === $tmpRule) {
             return false;
         }   // don't output the rule on error with the regex
         $tmpRule = preg_replace('/reference\s*:\s*.+?;/', $ruleFormatReference . ';', $tmpRule, -1, $replaceCount['reference']);
-        if (null == $tmpRule) {
+        if (null === $tmpRule) {
             return false;
         }   // don't output the rule on error with the regex
         $tmpRule = preg_replace('/reference\s*:\s*.+?;/', $ruleFormatReference . ';', $tmpRule, -1, $replaceCount['reference']);
-        if (null == $tmpRule) {
+        if (null === $tmpRule) {
             return false;
         }   // don't output the rule on error with the regex
         // FIXME nids -  implement priority overwriting
 
         // some values were not replaced, so we need to add them ourselves, and insert them in the rule
         $extraForRule = "";
-        if (0 == $replaceCount['sid']) {
+        if (0 === $replaceCount['sid']) {
             $extraForRule .= 'sid:' . $sid . ';';
         }
-        if (0 == $replaceCount['rev']) {
+        if (0 === $replaceCount['rev']) {
             $extraForRule .= 'rev:1;';
         }
-        if (0 == $replaceCount['classtype']) {
+        if (0 === $replaceCount['classtype']) {
             $extraForRule .= 'classtype:' . $this->classtype . ';';
         }
-        if (0 == $replaceCount['msg']) {
-            $extraForRule .= $tmpMessage . ';';
+        if (0 === $replaceCount['msg']) {
+            $extraForRule .= $ruleFormatMsg . ';';
         }
-        if (0 == $replaceCount['reference']) {
+        if (0 === $replaceCount['reference']) {
             $extraForRule .= $ruleFormatReference . ';';
         }
         $tmpRule = preg_replace('/;\s*\)/', '; ' . $extraForRule . ')', $tmpRule);
@@ -734,7 +699,7 @@ abstract class NidsExport
      * @param string $type the type of dns name - domain (default) or hostname
      * @return string raw snort compatible format of the dns name
      */
-    public static function dnsNameToRawFormat($name, $type='domain')
+    protected static function dnsNameToRawFormat($name, $type='domain')
     {
         $rawName = "";
         if ('hostname' == $type) {
@@ -747,7 +712,7 @@ abstract class NidsExport
             // count the length of the part, and add |length| before
             $length = strlen($explodedName);
             if ($length > 255) {
-                log('WARNING: DNS name is too long for RFC: '.$name);
+                CakeLog::notice('WARNING: DNS name is too long for RFC: '.$name);
             }
             $hexLength = dechex($length);
             if (1 == strlen($hexLength)) {
@@ -768,7 +733,7 @@ abstract class NidsExport
      * @param string $name dns name to be converted
      * @return string raw snort compatible format of the dns name
      */
-    public static function dnsNameToMSDNSLogFormat($name)
+    protected static function dnsNameToMSDNSLogFormat($name)
     {
         $rawName = "";
         // in MS DNS log format we can't use (0) to distinguish between hostname and domain (including subdomains)
@@ -779,7 +744,7 @@ abstract class NidsExport
             // count the length of the part, and add |length| before
             $length = strlen($explodedName);
             if ($length > 255) {
-                log('WARNING: DNS name is too long for RFC: '.$name);
+                CakeLog::notice('WARNING: DNS name is too long for RFC: '.$name);
             }
             $hexLength = dechex($length);
             $rawName .= '(' . $hexLength . ')' . $explodedName;
@@ -793,34 +758,32 @@ abstract class NidsExport
     /**
      * Replaces characters that are not allowed in a signature.
      *   example: " is converted to |22|
-     * @param unknown_type $value
+     * @param string $value
      */
-    public static function replaceIllegalChars($value)
+    protected static function replaceIllegalChars($value)
     {
         $replace_pairs = array(
-                '|' => '|7c|', // Needs to stay on top !
-                '"' => '|22|',
-                ';' => '|3b|',
-                ':' => '|3a|',
-                '\\' => '|5c|',
-                '0x' => '|30 78|'
-                );
+            '|' => '|7c|', // Needs to stay on top !
+            '"' => '|22|',
+            ';' => '|3b|',
+            ':' => '|3a|',
+            '\\' => '|5c|',
+            '0x' => '|30 78|'
+        );
         return strtr($value, $replace_pairs);
     }
 
-    public function checkWhitelist($value)
+    /**
+     * @deprecated
+     * @param $value
+     * @return false
+     */
+    protected function checkWhitelist($value)
     {
-        if ($this->checkWhitelist && is_array($this->whitelist)) {
-            foreach ($this->whitelist as $wlitem) {
-                if (preg_match($wlitem, $value)) {
-                    return true;
-                }
-            }
-        }
         return false;
     }
 
-    public static function getProtocolPort($protocol, $customPort)
+    protected static function getProtocolPort($protocol, $customPort)
     {
         if ($customPort == null) {
             switch ($protocol) {
@@ -840,7 +803,7 @@ abstract class NidsExport
         }
     }
 
-    public static function getCustomIP($customIP)
+    protected static function getCustomIP($customIP)
     {
         if (filter_var($customIP, FILTER_VALIDATE_IP)) {
             return $customIP;
@@ -853,7 +816,7 @@ abstract class NidsExport
      * @param array $attribute
      * @return array|string[]
      */
-    public static function getIpPort($attribute)
+    protected static function getIpPort($attribute)
     {
         if (strpos($attribute['type'], 'port') !== false) {
             return explode('|', $attribute['value']);

app/Lib/Export/NidsSnortExport.php

@@ -4,11 +4,5 @@ App::uses('NidsExport', 'Export');
 
 class NidsSnortExport extends NidsExport
 {
-    public function export($items, $startSid, $format = "suricata", $continue = false)
-    {
-        // set the specific format
-        $this->format = 'snort';
-        // call the generic function
-        return parent::export($items, $startSid, $format, $continue);
-    }
+    protected $format = 'snort';
 }

app/Lib/Export/NidsSuricataExport.php

@@ -3,16 +3,10 @@ App::uses('NidsExport', 'Export');
 
 class NidsSuricataExport extends NidsExport
 {
-    public function export($items, $startSid, $format = "suricata", $continue = false)
-    {
-        // set the specific format
-        $this->format = "suricata";
-        // call the generic function
-        return parent::export($items, $startSid, $format, $continue);
-    }
+    protected $format = "suricata";
 
     // below overwrite functions from NidsExport
-    public function hostnameRule($ruleFormat, $attribute, &$sid)
+    protected function hostnameRule($ruleFormat, $attribute, &$sid)
     {
         $overruled = $this->checkWhitelist($attribute['value']);
         $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']);  // substitute chars not allowed in rule
@@ -53,7 +47,7 @@ class NidsSuricataExport extends NidsExport
         );
     }
 
-    public function domainRule($ruleFormat, $attribute, &$sid)
+    protected function domainRule($ruleFormat, $attribute, &$sid)
     {
         $overruled = $this->checkWhitelist($attribute['value']);
         $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']);  // substitute chars not allowed in rule
@@ -94,7 +88,7 @@ class NidsSuricataExport extends NidsExport
         );
     }
 
-    public function urlRule($ruleFormat, $attribute, &$sid)
+    protected function urlRule($ruleFormat, $attribute, &$sid)
     {
         $createRule = true;
         $overruled = $this->checkWhitelist($attribute['value']);
@@ -207,7 +201,7 @@ class NidsSuricataExport extends NidsExport
         }
     }
 
-    public function userAgentRule($ruleFormat, $attribute, &$sid)
+    protected function userAgentRule($ruleFormat, $attribute, &$sid)
     {
         $overruled = $this->checkWhitelist($attribute['value']);
         $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']);  // substitute chars not allowed in rule
@@ -230,7 +224,7 @@ class NidsSuricataExport extends NidsExport
         );
     }
 
-    public function ja3Rule($ruleFormat, $attribute, &$sid)
+    protected function ja3Rule($ruleFormat, $attribute, &$sid)
     {
         $overruled = $this->checkWhitelist($attribute['value']);
         $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']);  // substitute chars not allowed in rule
@@ -253,7 +247,7 @@ class NidsSuricataExport extends NidsExport
     }
 
     // For Future use once JA3S Hash Attribute type is created
-    public function ja3sRule($ruleFormat, $attribute, &$sid)
+    protected function ja3sRule($ruleFormat, $attribute, &$sid)
     {
         $overruled = $this->checkWhitelist($attribute['value']);
         $attribute['value'] = NidsExport::replaceIllegalChars($attribute['value']);  // substitute chars not allowed in rule

app/Lib/Export/RPZExport.php

@@ -2,107 +2,104 @@
 
 class RPZExport
 {
-    private $__policies = array(
-            'Local-Data' => array(
-                    'explanation' => 'returns the defined alternate location.',
-                    'action' => '$walled_garden',
-                    'setting_id' => 3,
-            ),
-            'NXDOMAIN' => array(
-                    'explanation' => 'return NXDOMAIN (name does not exist) irrespective of actual result received.',
-                    'action' => '.',
-                    'setting_id' => 1,
-            ),
-            'NODATA' => array(
-                    'explanation' => 'returns NODATA (name exists but no answers returned) irrespective of actual result received.',
-                    'action' => '*.',
-                    'setting_id' => 2,
-            ),
-            'DROP' => array(
-                    'explanation' => 'timeout.',
-                    'action' => 'rpz-drop.',
-                    'setting_id' => 0,
-            ),
-            'PASSTHRU' => array(
-                    'explanation' => 'lets queries through, but allows for logging the hits (useful for testing).',
-                    'action' => 'rpz-passthru.',
-                    'setting_id' => 4,
-            ),
-            'TCP-only' => array(
-                    'explanation' => 'force the client to use TCP.',
-                    'action' => 'rpz-tcp-only.',
-                    'setting_id' => 5,
-            ),
+    const POLICIES = array(
+        'Local-Data' => array(
+            'explanation' => 'returns the defined alternate location.',
+            'action' => '$walled_garden',
+            'setting_id' => 3,
+        ),
+        'NXDOMAIN' => array(
+            'explanation' => 'return NXDOMAIN (name does not exist) irrespective of actual result received.',
+            'action' => '.',
+            'setting_id' => 1,
+        ),
+        'NODATA' => array(
+            'explanation' => 'returns NODATA (name exists but no answers returned) irrespective of actual result received.',
+            'action' => '*.',
+            'setting_id' => 2,
+        ),
+        'DROP' => array(
+            'explanation' => 'timeout.',
+            'action' => 'rpz-drop.',
+            'setting_id' => 0,
+        ),
+        'PASSTHRU' => array(
+            'explanation' => 'lets queries through, but allows for logging the hits (useful for testing).',
+            'action' => 'rpz-passthru.',
+            'setting_id' => 4,
+        ),
+        'TCP-only' => array(
+            'explanation' => 'force the client to use TCP.',
+            'action' => 'rpz-tcp-only.',
+            'setting_id' => 5,
+        ),
     );
 
-	private $__items = array();
+	private $items = array();
 
 	public $additional_params = array(
 		'flatten' => 1
 	);
 
-	private $__rpzSettings = array();
-
-	private $__valid_policies = array('NXDOMAIN', 'NODATA', 'DROP', 'Local-Data', 'PASSTHRU', 'TCP-only');
+	private $rpzSettings = array();
 
 	private $__server = null;
 
-	public $validTypes = array(
+	const VALID_TYPES = array(
 		'ip-src' => array(
-				'value' => 'ip'
+            'value' => 'ip'
 		),
 		'ip-dst' => array(
-				'value' => 'ip'
+            'value' => 'ip'
 		),
 		'domain' => array(
-				'value' => 'domain'
+            'value' => 'domain'
 		),
 		'domain|ip' => array(
-				'value1' => 'domain',
-				'value2' => 'ip'
+            'value1' => 'domain',
+            'value2' => 'ip'
 		),
 		'hostname' => array(
-				'value' => 'hostname'
+            'value' => 'hostname'
 		)
 	);
 
 	public function handler($data, $options = array())
 	{
 		if ($options['scope'] === 'Attribute') {
-			return $this->__attributeHandler($data, $options);
+			$this->attributeHandler($data);
 		} else {
-			return $this->__eventHandler($data, $options);
+			$this->eventHandler($data);
 		}
+        return '';
 	}
 
-	private function __eventHandler($event, $options = array()) {
+	private function eventHandler($event)
+    {
 		foreach ($event['Attribute'] as $attribute) {
-			if (isset($this->validTypes[$attribute['type']])) {
-				if ($attribute['type'] == 'domain|ip') {
+			if (isset(self::VALID_TYPES[$attribute['type']])) {
+				if ($attribute['type'] === 'domain|ip') {
 					$temp = explode('|', $attribute['value']);
 					$attribute['value1'] = $temp[0];
 					$attribute['value2'] = $temp[1];
 				}
-				$this->__attributeHandler(array('Attribute' => $attribute, $options));
+				$this->attributeHandler(array('Attribute' => $attribute));
 			}
 		}
-		return '';
 	}
 
-	private function __attributeHandler($attribute, $options = array())
+	private function attributeHandler($attribute)
 	{
 		if (isset($attribute['Attribute'])) {
 			$attribute = $attribute['Attribute'];
 		}
-		if (isset($this->validTypes[$attribute['type']])) {
-			foreach ($this->validTypes[$attribute['type']] as $field => $mapping) {
-				// get rid of the in_array check
-				if (empty($this->__items[$mapping]) || !isset($this->__items[$mapping][$attribute[$field]])) {
-					$this->__items[$mapping][$attribute[$field]] = true;
+		if (isset(self::VALID_TYPES[$attribute['type']])) {
+			foreach (self::VALID_TYPES[$attribute['type']] as $field => $mapping) {
+				if (!isset($this->items[$mapping][$attribute[$field]])) {
+					$this->items[$mapping][$attribute[$field]] = true;
 				}
 			}
 		}
-		return '';
 	}
 
 	public function header($options = array())
@@ -117,16 +114,16 @@ class RPZExport
 				}
 			}
 			if (isset($options['filters'][$v])) {
-				$this->__rpzSettings[$v] = $options['filters'][$v];
+				$this->rpzSettings[$v] = $options['filters'][$v];
 			} else {
 				$tempSetting = Configure::read('Plugin.RPZ_' . $v);
 				if (isset($tempSetting)) {
-					$this->__rpzSettings[$v] = Configure::read('Plugin.RPZ_' . $v);
+					$this->rpzSettings[$v] = $tempSetting;
 				} else {
 					if (empty($this->__server)) {
 						$this->__server = ClassRegistry::init('Server');
 					}
-					$this->__rpzSettings[$v] = $this->__server->serverSettings['Plugin']['RPZ_' . $v]['value'];
+					$this->rpzSettings[$v] = $this->__server->serverSettings['Plugin']['RPZ_' . $v]['value'];
 				}
 			}
 		}
@@ -135,10 +132,7 @@ class RPZExport
 
 	public function footer($options = array())
 	{
-		foreach ($this->__items as $k => $v) {
-			$this->__items[$k] = array_keys($this->__items[$k]);
-		}
-		return $this->export($this->__items, $this->__rpzSettings);
+		return $this->export($this->items, $this->rpzSettings);
 	}
 
 	public function separator()
@@ -146,39 +140,32 @@ class RPZExport
 		return '';
 	}
 
-    public function getPolicyById($id)
+    private function getPolicyById($id)
     {
-        foreach ($this->__policies as $k => $v) {
-            if ($id == $v['setting_id']) {
+        foreach (self::POLICIES as $k => $v) {
+            if ($id === $v['setting_id']) {
                 return $k;
             }
         }
+        return null;
     }
 
-    public function getIdByPolicy($policy)
+    private function getIdByPolicy($policy)
     {
-        return $this->__policies[$policy]['setting_id'];
+        return self::POLICIES[$policy]['setting_id'];
     }
 
-    public function explain($type, $policy)
+    private function explain($type, $policy)
     {
         $explanations = array(
             'ip' => '; The following list of IP addresses will ',
             'domain' => '; The following domain names and all of their sub-domains will ',
             'hostname' => '; The following hostnames will '
         );
-        $policy_explanations = array(
-            'Local-Data' => 'returns the defined alternate location.',
-            'NXDOMAIN' => 'return NXDOMAIN (name does not exist) irrespective of actual result received.',
-            'NODATA' => 'returns NODATA (name exists but no answers returned) irrespective of actual result received.',
-            'DROP' => 'timeout.',
-            'PASSTHRU' => 'lets queries through, but allows for logging the hits (useful for testing).',
-            'TCP-only' => 'force the client to use TCP.',
-        );
-        return $explanations[$type] . $this->__policies[$policy]['explanation'] . PHP_EOL;
+        return $explanations[$type] . self::POLICIES[$policy]['explanation'] . PHP_EOL;
     }
 
-    public function buildHeader($rpzSettings)
+    private function buildHeader(array $rpzSettings)
     {
         $rpzSettings['serial'] = str_replace('$date', date('Ymd'), $rpzSettings['serial']);
         $rpzSettings['serial'] = str_replace('$time', time(), $rpzSettings['serial']);
@@ -196,55 +183,55 @@ class RPZExport
         return $header;
     }
 
-    public function export($items, $rpzSettings)
+    private function export(array $items, array $rpzSettings)
     {
         $result = $this->buildHeader($rpzSettings);
         $policy = $this->getPolicyById($rpzSettings['policy']);
-        $action = $this->__policies[$policy]['action'];
-        if ($policy == 'Local-Data') {
+        $action = self::POLICIES[$policy]['action'];
+        if ($policy === 'Local-Data') {
             $action = str_replace('$walled_garden', $rpzSettings['walled_garden'], $action);
         }
 
         if (isset($items['ip'])) {
             $result .= $this->explain('ip', $policy);
-            foreach ($items['ip'] as $item) {
-                $result .= $this->__convertIP($item, $action);
+            foreach ($items['ip'] as $item => $foo) {
+                $result .= $this->convertIp($item, $action);
             }
             $result .= PHP_EOL;
         }
 
         if (isset($items['domain'])) {
             $result .= $this->explain('domain', $policy);
-            foreach ($items['domain'] as $item) {
-                $result .= $this->__convertdomain($item, $action);
+            foreach ($items['domain'] as $item => $foo) {
+                $result .= $this->convertDomain($item, $action);
             }
             $result .= PHP_EOL;
         }
 
         if (isset($items['hostname'])) {
             $result .= $this->explain('hostname', $policy);
-            foreach ($items['hostname'] as $item) {
-                $result .= $this->__converthostname($item, $action);
+            foreach ($items['hostname'] as $item => $foo) {
+                $result .= $this->convertHostname($item, $action);
             }
             $result .= PHP_EOL;
         }
         return $result;
     }
 
-    private function __convertdomain($input, $action)
+    private function convertDomain($input, $action)
     {
         return $input . ' CNAME ' . $action . PHP_EOL . '*.' . $input . ' CNAME ' . $action . PHP_EOL;
     }
 
-    private function __converthostname($input, $action)
+    private function convertHostname($input, $action)
     {
         return $input . ' CNAME ' . $action . PHP_EOL;
     }
 
-    private function __convertIP($input, $action)
+    private function convertIp($input, $action)
     {
-        $type = filter_var($input, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6) ? 'ipv6' : 'ipv4';
-        if ($type == 'ipv6') {
+        $isIpv6 = filter_var($input, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6);
+        if ($isIpv6) {
             $prefix = '128';
         } else {
             $prefix = '32';
@@ -252,7 +239,8 @@ class RPZExport
         if (strpos($input, '/')) {
             list($input, $prefix) = explode('/', $input);
         }
-        return $prefix . '.' . $this->{'__' . $type}($input) . '.rpz-ip CNAME ' . $action . PHP_EOL;
+        $converted = $isIpv6 ? $this->__ipv6($input) : $this->__ipv4($input);
+        return $prefix . '.' . $converted . '.rpz-ip CNAME ' . $action . PHP_EOL;
     }
 
     private function __ipv6($input)

app/Lib/Export/Stix1Export.php

@@ -11,7 +11,7 @@ class Stix1Export extends StixExport
     {
         return [
             ProcessTool::pythonBin(),
-            $this->__framing_script,
+            self::FRAMING_SCRIPT,
             'stix1',
             '-s', $this->__scope,
             '-v', $this->__version,
@@ -25,7 +25,7 @@ class Stix1Export extends StixExport
     {
         $command = [
             ProcessTool::pythonBin(),
-            $this->__scripts_dir . 'misp2stix.py',
+            self::SCRIPTS_DIR . 'misp2stix.py',
             '-s', $this->__scope,
             '-v', $this->__version,
             '-f', $this->__return_format,
@@ -33,6 +33,10 @@ class Stix1Export extends StixExport
             '-i',
         ];
         $command = array_merge($command, $this->__filenames);
-        return ProcessTool::execute($command, null, true);
+        try {
+            return ProcessTool::execute($command, null, true);
+        } catch (ProcessException $e) {
+            return $e->stdout();
+        }
     }
 }

app/Lib/Export/Stix2Export.php

@@ -11,16 +11,20 @@ class Stix2Export extends StixExport
     {
         return [
             ProcessTool::pythonBin(),
-            $this->__framing_script,
+            self::FRAMING_SCRIPT,
             'stix2',
             '-v', $this->__version,
             '--uuid', CakeText::uuid(),
         ];
     }
 
+    /**
+     * @return string
+     * @throws Exception
+     */
     protected function __parse_misp_data()
     {
-        $scriptFile = $this->__scripts_dir . 'stix2/misp2stix2.py';
+        $scriptFile = self::SCRIPTS_DIR . 'stix2/misp2stix2.py';
         $command = [
             ProcessTool::pythonBin(),
             $scriptFile,
@@ -28,7 +32,11 @@ class Stix2Export extends StixExport
             '-i',
         ];
         $command = array_merge($command, $this->__filenames);
-        $result = ProcessTool::execute($command, null, true);
+        try {
+            $result = ProcessTool::execute($command, null, true);
+        } catch (ProcessException $e) {
+            $result = $e->stdout();
+        }
         $result = preg_split("/\r\n|\n|\r/", trim($result));
         return end($result);
     }

app/Lib/Export/StixExport.php

@@ -6,13 +6,14 @@ App::uses('ProcessTool', 'Tools');
 
 abstract class StixExport
 {
+    const SCRIPTS_DIR = APP . 'files/scripts/',
+        FRAMING_SCRIPT = APP . 'files/scripts/misp_framing.py';
+
     public $additional_params = array(
         'includeEventTags' => 1,
         'includeGalaxy' => 1
     );
     protected $__return_format = 'json';
-    protected $__scripts_dir = APP . 'files/scripts/';
-    protected $__framing_script = APP . 'files/scripts/misp_framing.py';
     protected $__return_type = null;
 
     /** @var array Full paths to files to convert */

app/Lib/Tools/ApcuCacheTool.php

@@ -0,0 +1,198 @@
+<?php
+declare(strict_types=1);
+
+class ApcuCacheTool implements \Psr\SimpleCache\CacheInterface
+{
+    /** @var string */
+    private $prefix;
+
+    /**
+     * @param string $prefix
+     */
+    public function __construct(string $prefix)
+    {
+        $this->prefix = $prefix;
+    }
+
+    /**
+     * Fetches a value from the cache.
+     *
+     * @param string $key     The unique key of this item in the cache.
+     * @param mixed  $default Default value to return if the key does not exist.
+     *
+     * @return mixed The value of the item from the cache, or $default in case of cache miss.
+     *
+     * @throws \Psr\SimpleCache\InvalidArgumentException
+     *   MUST be thrown if the $key string is not a legal value.
+     */
+    public function get($key, $default = null)
+    {
+        $value = \apcu_fetch("$this->prefix:$key", $success);
+        if ($success) {
+            return $value;
+        }
+        return $default;
+    }
+
+    /**
+     * Persists data in the cache, uniquely referenced by a key with an optional expiration TTL time.
+     *
+     * @param string                 $key   The key of the item to store.
+     * @param mixed                  $value The value of the item to store, must be serializable.
+     * @param null|int|\DateInterval $ttl   Optional. The TTL value of this item. If no value is sent and
+     *                                      the driver supports TTL then the library may set a default value
+     *                                      for it or let the driver take care of that.
+     *
+     * @return bool True on success and false on failure.
+     *
+     * @throws \Psr\SimpleCache\InvalidArgumentException
+     *   MUST be thrown if the $key string is not a legal value.
+     */
+    public function set($key, $value, $ttl = null)
+    {
+        return \apcu_store("$this->prefix:$key", $value, $this->tllToInt($ttl));
+    }
+
+    /**
+     * Delete an item from the cache by its unique key.
+     *
+     * @param string $key The unique cache key of the item to delete.
+     *
+     * @return bool True if the item was successfully removed. False if there was an error.
+     *
+     * @throws \Psr\SimpleCache\InvalidArgumentException
+     *   MUST be thrown if the $key string is not a legal value.
+     */
+    public function delete($key)
+    {
+        return \apcu_delete("$this->prefix:$key");
+    }
+
+    /**
+     * Wipes clean the entire cache's keys.
+     *
+     * @return bool True on success and false on failure.
+     */
+    public function clear()
+    {
+        $iterator = new APCUIterator(
+            '/^' . preg_quote($this->prefix . ':', '/') . '/',
+            APC_ITER_NONE
+        );
+        return \apcu_delete($iterator);
+    }
+
+    /**
+     * Obtains multiple cache items by their unique keys.
+     *
+     * @param iterable $keys    A list of keys that can obtained in a single operation.
+     * @param mixed    $default Default value to return for keys that do not exist.
+     *
+     * @return iterable A list of key => value pairs. Cache keys that do not exist or are stale will have $default as value.
+     *
+     * @throws \Psr\SimpleCache\InvalidArgumentException
+     *   MUST be thrown if $keys is neither an array nor a Traversable,
+     *   or if any of the $keys are not a legal value.
+     */
+    public function getMultiple($keys, $default = null)
+    {
+        $keysToFetch = $this->keysToFetch($keys);
+        $values = \apcu_fetch($keysToFetch);
+        foreach ($keysToFetch as $keyToFetch) {
+            if (!isset($values[$keyToFetch])) {
+                $values[$keyToFetch] = $default;
+            }
+        }
+        return $values;
+    }
+
+    /**
+     * Persists a set of key => value pairs in the cache, with an optional TTL.
+     *
+     * @param iterable               $values A list of key => value pairs for a multiple-set operation.
+     * @param null|int|\DateInterval $ttl    Optional. The TTL value of this item. If no value is sent and
+     *                                       the driver supports TTL then the library may set a default value
+     *                                       for it or let the driver take care of that.
+     *
+     * @return bool True on success and false on failure.
+     *
+     * @throws \Psr\SimpleCache\InvalidArgumentException
+     *   MUST be thrown if $values is neither an array nor a Traversable,
+     *   or if any of the $values are not a legal value.
+     */
+    public function setMultiple($values, $ttl = null)
+    {
+        $dataToSave = [];
+        foreach ($values as $key => $value) {
+            $dataToSave["$this->prefix:$key"] = $value;
+        }
+        return \apcu_store($dataToSave, null, $this->tllToInt($ttl));
+    }
+
+    /**
+     * Deletes multiple cache items in a single operation.
+     *
+     * @param iterable $keys A list of string-based keys to be deleted.
+     *
+     * @return bool True if the items were successfully removed. False if there was an error.
+     *
+     * @throws \Psr\SimpleCache\InvalidArgumentException
+     *   MUST be thrown if $keys is neither an array nor a Traversable,
+     *   or if any of the $keys are not a legal value.
+     */
+    public function deleteMultiple($keys)
+    {
+        $keysToDelete = $this->keysToFetch($keys);
+        return \apcu_delete($keysToDelete);
+    }
+
+    /**
+     * Determines whether an item is present in the cache.
+     *
+     * NOTE: It is recommended that has() is only to be used for cache warming type purposes
+     * and not to be used within your live applications operations for get/set, as this method
+     * is subject to a race condition where your has() will return true and immediately after,
+     * another script can remove it making the state of your app out of date.
+     *
+     * @param string $key The cache item key.
+     *
+     * @return bool
+     *
+     * @throws \Psr\SimpleCache\InvalidArgumentException
+     *   MUST be thrown if the $key string is not a legal value.
+     */
+    public function has($key)
+    {
+        return \apcu_exists("$this->prefix:$key");
+    }
+
+    /**
+     * @param iterable $keys
+     * @return array
+     */
+    private function keysToFetch(iterable $keys): array
+    {
+        $keysToFetch = [];
+        foreach ($keys as $key) {
+            $keysToFetch[] = "$this->prefix:$key";
+        }
+        return $keysToFetch;
+    }
+
+    /**
+     * @param null|int|\DateInterval $ttl
+     * @return int
+     */
+    private function tllToInt($ttl = null): int
+    {
+        if ($ttl === null) {
+            return 0;
+        } elseif (is_int($ttl)) {
+            return $ttl;
+        } elseif ($ttl instanceof \DateInterval) {
+            return $ttl->days * 86400 + $ttl->h * 3600 + $ttl->i * 60 + $ttl->s;
+        } else {
+            throw new \Psr\SimpleCache\InvalidArgumentException("Invalid ttl value '$ttl' provided.");
+        }
+    }
+}

app/Lib/Tools/AttributeValidationTool.php

@@ -41,7 +41,7 @@ class AttributeValidationTool
         switch ($type) {
             case 'ip-src':
             case 'ip-dst':
-                return self::compressIpv6($value);
+                return self::normalizeIp($value);
             case 'md5':
             case 'sha1':
             case 'sha224':
@@ -98,7 +98,7 @@ class AttributeValidationTool
                         $parts[0] = $punyCode;
                     }
                 }
-                $parts[1] = self::compressIpv6($parts[1]);
+                $parts[1] = self::normalizeIp($parts[1]);
                 return "$parts[0]|$parts[1]";
             case 'filename|md5':
             case 'filename|sha1':
@@ -175,7 +175,7 @@ class AttributeValidationTool
                 } else {
                     return $value;
                 }
-                return self::compressIpv6($parts[0]) . '|' . $parts[1];
+                return self::normalizeIp($parts[0]) . '|' . $parts[1];
             case 'mac-address':
             case 'mac-eui-64':
                 $value = str_replace(array('.', ':', '-', ' '), '', strtolower($value));
@@ -556,7 +556,12 @@ class AttributeValidationTool
                   if (!is_numeric($value) || $value < 0 || $value > 10) {
                       return __('The value has to be a number between 0 and 10.');
                   }
-                  return true;*/
+                return true;*/
+            case 'integer':
+                if (is_int($value)) {
+                    return true;
+                }
+                return __('The value has to be an integer value.');
             case 'iban':
             case 'bic':
             case 'btc':
@@ -700,11 +705,30 @@ class AttributeValidationTool
      * @param string $value
      * @return string
      */
-    private static function compressIpv6($value)
+    private static function normalizeIp($value)
     {
+        // If IP is a CIDR
+        if (strpos($value, '/')) {
+            list($ip, $range) = explode('/', $value, 2);
+
+            // Compress IPv6
+            if (strpos($ip, ':') && $converted = inet_pton($ip)) {
+                $ip = inet_ntop($converted);
+            }
+
+            // If IP is in CIDR format, but the network is 32 for IPv4 or 128 for IPv6, normalize to non CIDR type
+            if (($range === '32' && strpos($value, '.')) || ($range === '128' && strpos($value, ':'))) {
+                return $ip;
+            }
+
+            return "$ip/$range";
+        }
+
+        // Compress IPv6
         if (strpos($value, ':') && $converted = inet_pton($value)) {
             return inet_ntop($converted);
         }
+
         return $value;
     }
     

app/Lib/Tools/BackgroundJobs/BackgroundJob.php

@@ -66,8 +66,9 @@ class BackgroundJob implements JsonSerializable
 
     /**
      * Run the job command
+     * @param callable|null $runningCallback
      */
-    public function run(): void
+    public function run(callable $runningCallback = null): void
     {
         $descriptorSpec = [
             1 => ["pipe", "w"], // stdout
@@ -88,7 +89,7 @@ class BackgroundJob implements JsonSerializable
             ['BACKGROUND_JOB_ID' => $this->id]
         );
 
-        $this->pool($process, $pipes);
+        $this->pool($process, $pipes, $runningCallback);
 
         if ($this->returnCode === 0 && empty($stderr)) {
             $this->setStatus(BackgroundJob::STATUS_COMPLETED);
@@ -98,7 +99,13 @@ class BackgroundJob implements JsonSerializable
         }
     }
 
-    private function pool($process, array $pipes)
+    /**
+     * @param resource $process
+     * @param array $pipes
+     * @param callable|null $runningCallback
+     * @return void
+     */
+    private function pool($process, array $pipes, callable $runningCallback = null)
     {
         stream_set_blocking($pipes[1], false);
         stream_set_blocking($pipes[2], false);
@@ -106,6 +113,14 @@ class BackgroundJob implements JsonSerializable
         $this->output = '';
         $this->error = '';
 
+        if ($runningCallback) {
+            $status = proc_get_status($process);
+            if ($status === false) {
+                throw new RuntimeException("Could not get process status");
+            }
+            $runningCallback($status);
+        }
+
         while (true) {
             $read = [$pipes[1], $pipes[2]];
             $write = null;
@@ -118,6 +133,12 @@ class BackgroundJob implements JsonSerializable
                 $this->error .= stream_get_contents($pipes[2]);
             }
             $status = proc_get_status($process);
+            if ($status === false) {
+                throw new RuntimeException("Could not get process status");
+            }
+            if ($runningCallback) {
+                $runningCallback($status);
+            }
             if (!$status['running']) {
                 // Just in case read rest data from stream
                 $this->output .= stream_get_contents($pipes[1]);
@@ -153,6 +174,9 @@ class BackgroundJob implements JsonSerializable
         return ['id', 'command', 'args', 'createdAt', 'updatedAt', 'status', 'output', 'error', 'metadata'];
     }
 
+    /**
+     * @return string Background job ID in UUID format
+     */
     public function id(): string
     {
         return $this->id;

app/Lib/Tools/BackgroundJobs/Worker.php

@@ -65,7 +65,7 @@ class Worker implements JsonSerializable
         ];
     }
 
-    public function pid(): ?int
+    public function pid(): int
     {
         return $this->pid;
     }

app/Lib/Tools/BackgroundJobsTool.php

@@ -91,7 +91,8 @@ class BackgroundJobsTool
     ];
 
     const JOB_STATUS_PREFIX = 'job_status',
-        DATA_CONTENT_PREFIX = 'data_content';
+        DATA_CONTENT_PREFIX = 'data_content',
+        RUNNING_JOB_PREFIX = 'running';
 
     /** @var array */
     private $settings;
@@ -277,6 +278,54 @@ class BackgroundJobsTool
         return null;
     }
 
+    /**
+     * @param Worker $worker
+     * @param BackgroundJob $job
+     * @param int|null $pid
+     * @return void
+     * @throws RedisException
+     */
+    public function markAsRunning(Worker $worker, BackgroundJob $job, $pid = null)
+    {
+        $key = self::RUNNING_JOB_PREFIX . ':' . $worker->queue() . ':' . $job->id();
+        $this->RedisConnection->setex($key, 60, [
+            'worker_pid' => $worker->pid(),
+            'process_pid' => $pid,
+        ]);
+    }
+
+    /**
+     * @param Worker $worker
+     * @param BackgroundJob $job
+     * @return void
+     * @throws RedisException
+     */
+    public function removeFromRunning(Worker $worker, BackgroundJob $job)
+    {
+        $key = self::RUNNING_JOB_PREFIX . ':' . $worker->queue() . ':' . $job->id();
+        $this->RedisConnection->del($key);
+    }
+
+    /**
+     * Return current running jobs
+     * @return array
+     * @throws RedisException
+     */
+    public function runningJobs(): array
+    {
+        $pattern = $this->RedisConnection->_prefix(self::RUNNING_JOB_PREFIX . ':*');
+        $keys = RedisTool::keysByPattern($this->RedisConnection, $pattern);
+
+        $jobIds = [];
+        foreach ($keys as $key) {
+            $parts = explode(':', $key);
+            $queue = $parts[2];
+            $jobId = $parts[3];
+            $jobIds[$queue][$jobId] = $this->RedisConnection->get(self::RUNNING_JOB_PREFIX . ":$queue:$jobId");
+        }
+        return $jobIds;
+    }
+
     /**
      * Get the job status.
      *
@@ -500,19 +549,6 @@ class BackgroundJobsTool
         $this->getSupervisor()->startProcessGroup(self::MISP_WORKERS_PROCESS_GROUP, $waitForRestart);
     }
 
-    /**
-     * Purge queue
-     *
-     * @param string $queue
-     * @return void
-     */
-    public function purgeQueue(string $queue)
-    {
-        $this->validateQueue($queue);
-
-        $this->RedisConnection->del($queue);
-    }
-
     /**
      * Return Background Jobs status
      *
@@ -728,8 +764,7 @@ class BackgroundJobsTool
      *
      * @param integer $pid
      * @return \Supervisor\Process
-     *
-     * @throws NotFoundException
+     * @throws NotFoundException|Exception
      */
     private function getProcessByPid(int $pid): \Supervisor\Process
     {

app/Lib/Tools/BenchmarkTool.php

@@ -0,0 +1,178 @@
+<?php
+
+/**
+ * Get filter parameters from index searches
+ */
+
+class BenchmarkTool
+{
+    /** @var Model */
+    public $Model;
+    
+    /** @var redis */
+    public $redis;
+
+    /** @var retention */
+    private $retention = 0;
+
+    const BENCHMARK_SCOPES = ['user', 'endpoint', 'user_agent'];
+    const BENCHMARK_FIELDS = ['time', 'sql_time', 'sql_queries', 'memory'];
+    const BENCHMARK_UNITS = [
+        'time' => 's',
+        'sql_time' => 'ms',
+        'sql_queries' => '',
+        'memory' => 'MB'
+    ];
+
+    public $namespace = 'misp:benchmark:';
+
+    function __construct(Model $model) {
+        $this->Model = $model;
+    }
+
+    public function getSettings()
+    {
+        return [
+            'scope' => self::BENCHMARK_SCOPES,
+            'field' => self::BENCHMARK_FIELDS,
+            'average' => [0, 1],
+            'aggregate' => [0, 1]
+        ];
+    }
+
+    public function getUnits()
+    {
+        return self::BENCHMARK_UNITS;
+    }
+
+    public function startBenchmark()
+    {
+        $start_time = microtime(true);
+        $this->redis = $this->Model->setupRedis();
+        $this->retention = Configure::check('Plugin.benchmark_retention') ? Configure::read('Plugin.benchmark_retention') : 0;
+        return $start_time;
+    }
+
+    public function stopBenchmark(array $options)
+    {
+        $start_time = $options['start_time'];
+        if (!empty($options['user'])) {
+            $sql = $this->Model->getDataSource()->getLog(false, false);
+            $benchmarkData = [
+                'user' => $options['user'],
+                'endpoint' => $options['controller'] . '/' . $options['action'],
+                'user_agent' => $_SERVER['HTTP_USER_AGENT'],
+                'sql_queries' => $sql['count'],
+                'sql_time' => $sql['time'],
+                'time' => (microtime(true) - $start_time),
+                'memory' => (int)(memory_get_peak_usage(true) / 1024 / 1024),
+                //'date' => date('Y-m-d', strtotime("-3 days"))
+                'date' => date('Y-m-d')
+            ];
+            $this->pushBenchmarkDataToRedis($benchmarkData);
+        } else {
+            $sql = $this->Model->getDataSource()->getLog(false, false);
+            $benchmarkData = [
+                'user' => 'SYSTEM',
+                'endpoint' => $options['controller'] . '/' . $options['action'],
+                'user_agent' => 'CLI',
+                'sql_queries' => $sql['count'],
+                'sql_time' => $sql['time'],
+                'time' => (microtime(true) - $start_time),
+                'memory' => (int)(memory_get_peak_usage(true) / 1024 / 1024),
+                //'date' => date('Y-m-d', strtotime("-3 days"))
+                'date' => date('Y-m-d')
+            ];
+            $this->pushBenchmarkDataToRedis($benchmarkData);
+        }
+    }
+
+    private function pushBenchmarkDataToRedis($benchmarkData)
+    {
+        $this->redis = $this->Model->setupRedis();
+        $this->redis->pipeline();
+        $this->redis->sAdd(
+            $this->namespace . 'days',
+            $benchmarkData['date']
+        );
+        foreach (self::BENCHMARK_SCOPES as $scope) {
+            $this->redis->sAdd(
+                $this->namespace . $scope . ':list',
+                $benchmarkData[$scope]
+            );
+            $this->redis->zIncrBy(
+                $this->namespace . $scope . ':count:' . $benchmarkData['date'],
+                1,
+                $benchmarkData[$scope]
+            );
+            foreach (self::BENCHMARK_FIELDS as $field) {
+                $this->redis->zIncrBy(
+                    $this->namespace . $scope . ':' . $field . ':' . $benchmarkData['date'],
+                    $benchmarkData[$field],
+                    $benchmarkData[$scope]
+                );
+            }
+            $this->redis->zIncrBy(
+                $this->namespace . $scope . ':endpoint:' . $benchmarkData['date'] . ':' . $benchmarkData['user'],
+                1,
+                $benchmarkData['endpoint']
+            );
+        }
+        $this->redis->exec();
+    }
+
+    public function getTopList(string $scope, string $field, array $days = [], $limit = 10, $average = false, $aggregate = false)
+    {
+        if (empty($this->redis)) {
+            $this->redis = $this->Model->setupRedis();
+        }
+        $results = [];
+        if (is_string($days)) {
+            $days = [$days];
+        }
+        foreach ($days as $day) {
+            $temp = $this->redis->zrevrange($this->namespace . $scope . ':' . $field . ':' . $day, 0, $limit, true);
+            foreach ($temp as $k => $v) {
+                if ($average) {
+                    $divisor = $this->redis->zscore($this->namespace . $scope . ':count:' . $day, $k);
+                    if ($aggregate) {
+                        $results['aggregate'][$k] = empty($results['aggregate'][$k]) ? ($v / $divisor) : ($results['aggregate'][$k] + ($v / $divisor));
+                    } else {
+                        $results[$day][$k] = (int)($v / $divisor);
+                    }
+                } else {
+                    if ($aggregate) {
+                        $results['aggregate'][$k] = empty($results['aggregate'][$k]) ? $v : ($results['aggregate'][$k] + $v);
+                    } else {
+                        $results[$day][$k] = $v;
+                    }
+                }
+            }
+        }
+        if ($aggregate && $average) {
+            $count_days = count($days);
+            foreach ($results['aggregate'] as $k => $result) {
+                $results['aggregate'][$k] = (int)($result / $count_days);
+            }
+        }
+        return $results;
+    }
+
+    public function getAllTopLists(array $days = null, $limit = 10, $average = false, $aggregate = false, $scope_filter = [])
+    {
+        if (empty($this->redis)) {
+            $this->redis = $this->Model->setupRedis();
+        }
+        if ($days === null) {
+            $days = $this->redis->smembers($this->namespace . 'days');
+        }
+        foreach (self::BENCHMARK_SCOPES as $scope) {
+            if (empty($scope_filter) || in_array($scope, $scope_filter)) {
+                foreach (self::BENCHMARK_FIELDS as $field) {
+                    $results[$scope][$field] = $this->getTopList($scope, $field, $days, $limit, $average, $aggregate);
+                }
+            }
+        }
+        return $results;
+    }
+}

app/Lib/Tools/BetterCakeEventManager.php

@@ -52,10 +52,10 @@ class BetterCakeEventManager extends CakeEventManager
         $result = [];
         foreach ($priorities as $priority) {
             if (isset($globalListeners[$priority])) {
-                $result = array_merge($result, $globalListeners[$priority]);
+                array_push($result, ...$globalListeners[$priority]);
             }
             if (isset($localListeners[$priority])) {
-                $result = array_merge($result, $localListeners[$priority]);
+                array_push($result, ...$localListeners[$priority]);
             }
         }
         return $result;

app/Lib/Tools/BetterSecurity.php

@@ -7,8 +7,8 @@ class BetterSecurity
 
     /**
      * @param string $plain
-     * @param string $key
-     * @return string
+     * @param string $key Encryption key
+     * @return string Cipher text with IV and tag
      * @throws Exception
      */
     public static function encrypt($plain, $key)
@@ -33,17 +33,17 @@ class BetterSecurity
     }
 
     /**
-     * @param string $cipher
-     * @param string $key
+     * @param string $cipherText Cipher text with IV and tag
+     * @param string $key Decryption key
      * @return string
      * @throws Exception
      */
-    public static function decrypt($cipher, $key)
+    public static function decrypt($cipherText, $key)
     {
         if (strlen($key) < 32) {
             throw new Exception('Invalid key, key must be at least 256 bits (32 bytes) long.');
         }
-        if (empty($cipher)) {
+        if (empty($cipherText)) {
             throw new Exception('The data to decrypt cannot be empty.');
         }
 
@@ -52,12 +52,18 @@ class BetterSecurity
 
         $ivSize = openssl_cipher_iv_length(self::METHOD);
 
-        // Split out hmac for comparison
-        $iv = substr($cipher, 0, $ivSize);
-        $tag = substr($cipher, $ivSize, self::TAG_SIZE);
-        $cipher = substr($cipher, $ivSize + self::TAG_SIZE);
+        if (strlen($cipherText) < $ivSize + self::TAG_SIZE) {
+            $length = strlen($cipherText);
+            $minLength = $ivSize + self::TAG_SIZE;
+            throw new Exception("Provided cipher text is too short, $length bytes provided, expected at least $minLength bytes.");
+        }
 
-        $decrypted = openssl_decrypt($cipher, self::METHOD, $key, true, $iv, $tag);
+        // Split out hmac for comparison
+        $iv = substr($cipherText, 0, $ivSize);
+        $tag = substr($cipherText, $ivSize, self::TAG_SIZE);
+        $cipherText = substr($cipherText, $ivSize + self::TAG_SIZE);
+
+        $decrypted = openssl_decrypt($cipherText, self::METHOD, $key, OPENSSL_RAW_DATA, $iv, $tag);
         if ($decrypted === false) {
             throw new Exception('Could not decrypt. Maybe invalid encryption key?');
         }

app/Lib/Tools/ComplexTypeTool.php

@@ -308,14 +308,13 @@ class ComplexTypeTool
      */
     private function parseFreetext($input)
     {
-        $input = str_replace("\xc2\xa0", ' ', $input); // non breaking space to normal space
-        $input = preg_replace('/\p{C}+/u', ' ', $input);
-        $iocArray = preg_split("/\r\n|\n|\r|\s|\s+|,|\<|\>|;/", $input);
+        // convert non breaking space to normal space and all unicode chars from "other" category
+        $input = preg_replace("/\p{C}+|\xc2\xa0/u", ' ', $input);
+        $iocArray = preg_split("/\r\n|\n|\r|\s|\s+|,|<|>|;/", $input);
 
         preg_match_all('/\"([^\"]+)\"/', $input, $matches);
-        foreach ($matches[1] as $match) {
-            $iocArray[] = $match;
-        }
+        array_push($iocArray, ...$matches[1]);
+
         return $iocArray;
     }
 

app/Lib/Tools/CorrelationGraphTool.php

@@ -1,4 +1,6 @@
 <?php
+App::uses('OrgImgHelper', 'View/Helper');
+
   class CorrelationGraphTool
   {
       private $__lookupTables = array();
@@ -9,20 +11,24 @@
       /** @var Taxonomy */
       private $__taxonomyModel;
       private $__galaxyClusterModel = false;
-      private $__user = false;
-      private $__json = array();
+      /** @var User */
+      private $__user;
+      /** @var array */
+      private $data;
+      private $orgImgHelper;
 
-      public function construct(Event $eventModel, $taxonomyModel, $galaxyClusterModel, $user, $json)
+      public function __construct(Event $eventModel, $taxonomyModel, $galaxyClusterModel, array $user, array $data)
       {
           $this->__eventModel = $eventModel;
           $this->__taxonomyModel = $taxonomyModel;
           $this->__galaxyClusterModel = $galaxyClusterModel;
           $this->__user = $user;
-          $this->__json = $json;
+          $this->data = $data;
           $this->__lookupTables = array(
-        'analysisLevels' => $this->__eventModel->analysisLevels,
-        'distributionLevels' => $this->__eventModel->Attribute->distributionLevels
-      );
+            'analysisLevels' => $eventModel->analysisLevels,
+            'distributionLevels' => $eventModel->Attribute->distributionLevels
+          );
+          $this->orgImgHelper = new OrgImgHelper(new View());
           return true;
       }
 
@@ -38,7 +44,7 @@
               'sgReferenceOnly' => true,
           ));
           if (empty($event)) {
-              return $this->__json;
+              return $this->data;
           }
           $this->cleanLinks();
           $event[0]['Event']['Orgc'] = $event[0]['Orgc'];
@@ -75,7 +81,7 @@
       public function buildGraphJson($id, $type = 'event', $action = 'create')
       {
           if ($action == 'delete') {
-              return $this->__json;
+              return $this->data;
           }
           switch ($type) {
         case 'event':
@@ -88,13 +94,13 @@
           $this->__expandTag($id);
           break;
       }
-          return $this->__json;
+          return $this->data;
       }
 
       private function __deleteObject($id)
       {
           $this->cleanLinks();
-          return $this->__json;
+          return $this->data;
       }
 
       private function __handleObjects($objects, $anchor_id, $full = false)
@@ -193,8 +199,8 @@
 
       private function __expandGalaxy($id)
       {
-          if (!empty($this->__json['nodes'])) {
-              foreach ($this->__json['nodes'] as $k => $node) {
+          if (!empty($this->data['nodes'])) {
+              foreach ($this->data['nodes'] as $k => $node) {
                   if ($node['type'] == 'galaxy' && $node['id'] == $id) {
                       $current_galaxy_id = $k;
                       $tag_name = $node['tag_name'];
@@ -205,7 +211,7 @@
               $current_galaxy_id = $this->__addGalaxy($id);
           }
           $this->cleanLinks();
-          $events = $this->__eventModel->EventTag->Tag->fetchSimpleEventsForTag($this->__json['nodes'][$current_galaxy_id]['tag_name'], $this->__user, true);
+          $events = $this->__eventModel->EventTag->Tag->fetchSimpleEventsForTag($this->data['nodes'][$current_galaxy_id]['tag_name'], $this->__user, true);
           foreach ($events as $event) {
               $current_event_id = $this->__createNode('event', $event);
               $this->__addLink($current_event_id, $current_galaxy_id);
@@ -229,7 +235,7 @@
       {
           $link = $this->graphJsonContainsLink($from_id, $to_id);
           if ($link === false) {
-              $this->__json['links'][] = array('source' => $from_id, 'target' => $to_id, 'linkDistance' => $linkDistance);
+              $this->data['links'][] = array('source' => $from_id, 'target' => $to_id, 'linkDistance' => $linkDistance);
           }
       }
 
@@ -240,7 +246,7 @@
           if ($from_uuid == $to_uuid) {
               return false;
           }
-          foreach ($this->__json['nodes'] as $k => $node) {
+          foreach ($this->data['nodes'] as $k => $node) {
               if ($node['uuid'] === $from_uuid) {
                   $from_id = $k;
               }
@@ -277,10 +283,9 @@
             );
             break;
           case 'event':
-            if ($this->orgImgExists($data['Orgc']['name'])) {
-                $image = Configure::read('MISP.baseurl') . '/img/orgs/' . h($data['Orgc']['name']) . '.png';
-            } else {
-                $image = Configure::read('MISP.baseurl') . '/img/orgs/MISP.png';
+            $orgImage = $this->orgImgHelper->getOrgLogoAsBase64($data['Orgc']);
+            if ($orgImage === null) {
+                $orgImage = Configure::read('MISP.baseurl') . '/img/misp-org.png';
             }
             $node = array(
               'unique_id' => 'event-' . $data['id'],
@@ -289,7 +294,7 @@
               'id' => $data['id'],
               'expanded' => $expand,
               'uuid' => $data['uuid'],
-              'image' => $image,
+              'image' => $orgImage,
               'info' => $data['info'],
               'org' => $data['Orgc']['name'],
               'analysis' => $this->__lookupTables['analysisLevels'][$data['analysis']],
@@ -345,11 +350,11 @@
             );
             break;
         }
-              $this->__json['nodes'][] = $node;
-              $current_id = count($this->__json['nodes'])-1;
+              $this->data['nodes'][] = $node;
+              $current_id = count($this->data['nodes'])-1;
           } else {
               if ($expand) {
-                  $this->__json['nodes'][$current_id]['expanded'] = 1;
+                  $this->data['nodes'][$current_id]['expanded'] = 1;
               }
           }
           return $current_id;
@@ -357,11 +362,11 @@
 
       public function cleanLinks()
       {
-          if (isset($this->__json['nodes']) && isset($this->__json['links'])) {
+          if (isset($this->data['nodes']) && isset($this->data['links'])) {
               $links = array();
-              foreach ($this->__json['links'] as $link) {
+              foreach ($this->data['links'] as $link) {
                   $temp = array();
-                  foreach ($this->__json['nodes'] as $k => $node) {
+                  foreach ($this->data['nodes'] as $k => $node) {
                       if ($link['source'] == $node) {
                           $temp['source'] = $k;
                       }
@@ -372,32 +377,24 @@
                   $temp['linkDistance'] = $link['linkDistance'];
                   $links[] = $temp;
               }
-              $this->__json['links'] = $links;
+              $this->data['links'] = $links;
           } else {
-              if (!isset($this->__json['links'])) {
-                  $this->__json['links'] = array();
+              if (!isset($this->data['links'])) {
+                  $this->data['links'] = array();
               }
-              if (!isset($this->__json['nodes'])) {
-                  $this->__json['nodes'] = array();
+              if (!isset($this->data['nodes'])) {
+                  $this->data['nodes'] = array();
               }
           }
           return true;
       }
 
-      public function orgImgExists($org)
-      {
-          if (file_exists(APP . 'webroot' . DS . 'img' . DS . 'orgs' . DS . $org . '.png')) {
-              return true;
-          }
-          return false;
-      }
-
       public function graphJsonContains($type, $element)
       {
-          if (!isset($this->__json['nodes'])) {
+          if (!isset($this->data['nodes'])) {
               return false;
           }
-          foreach ($this->__json['nodes'] as $k => $node) {
+          foreach ($this->data['nodes'] as $k => $node) {
               if ($type == 'event' && $node['type'] == 'event' && $node['id'] == $element['id']) {
                   return $k;
               }
@@ -418,10 +415,10 @@
       }
       public function graphJsonContainsLink($id1, $id2)
       {
-          if (!isset($this->__json['links'])) {
+          if (!isset($this->data['links'])) {
               return false;
           }
-          foreach ($this->__json['links'] as $k => $link) {
+          foreach ($this->data['links'] as $k => $link) {
               if (($link['source'] == $id1 && $link['target'] == $id2) || ($link['source'] == $id2 && $link['target'] == $id1)) {
                   return $k;
               }

app/Lib/Tools/CurlClient.php

@@ -0,0 +1,354 @@
+<?php
+App::uses('HttpSocketExtended', 'Tools');
+
+class CurlClient extends HttpSocketExtended
+{
+    /** @var resource */
+    private $ch;
+
+    /**
+     * Maximum time the transfer is allowed to complete in seconds
+     * 300 seconds is recommended timeout for MISP servers
+     * @var int
+     */
+    private $timeout = 300;
+
+    /** @var string|null */
+    private $caFile;
+
+    /** @var string|null */
+    private $localCert;
+
+    /** @var int */
+    private $cryptoMethod;
+
+    /** @var bool */
+    private $allowSelfSigned;
+
+    /** @var bool */
+    private $verifyPeer;
+
+    /** @var bool */
+    private $compress = true;
+
+    /** @var array */
+    private $proxy = [];
+
+    /** @var array */
+    private $defaultOptions;
+
+    /**
+     * @param array $params
+     * @noinspection PhpMissingParentConstructorInspection
+     */
+    public function __construct(array $params)
+    {
+        if (isset($params['timeout'])) {
+            $this->timeout = $params['timeout'];
+        }
+        if (isset($params['ssl_cafile'])) {
+            $this->caFile = $params['ssl_cafile'];
+        }
+        if (isset($params['ssl_local_cert'])) {
+            $this->localCert = $params['ssl_local_cert'];
+        }
+        if (isset($params['compress'])) {
+            $this->compress = $params['compress'];
+        }
+        if (isset($params['ssl_crypto_method'])) {
+            $this->cryptoMethod = $this->convertCryptoMethod($params['ssl_crypto_method']);
+        }
+        if (isset($params['ssl_allow_self_signed'])) {
+            $this->allowSelfSigned = $params['ssl_allow_self_signed'];
+        }
+        if (isset($params['ssl_verify_peer'])) {
+            $this->verifyPeer = $params['ssl_verify_peer'];
+        }
+        $this->defaultOptions = $this->generateDefaultOptions();
+    }
+
+    /**
+     * @param string $uri
+     * @param array $query
+     * @param array $request
+     * @return HttpSocketResponseExtended
+     */
+    public function head($uri = null, $query = [], $request = [])
+    {
+        return $this->internalRequest('HEAD', $uri, $query, $request);
+    }
+
+    /**
+     * @param string $uri
+     * @param array $query
+     * @param array $request
+     * @return HttpSocketResponseExtended
+     */
+    public function get($uri = null, $query = [], $request = [])
+    {
+        return $this->internalRequest('GET', $uri, $query, $request);
+    }
+
+    /**
+     * @param string $uri
+     * @param array $data
+     * @param array $request
+     * @return HttpSocketResponseExtended
+     */
+    public function post($uri = null, $data = [], $request = [])
+    {
+        return $this->internalRequest('POST', $uri, $data, $request);
+    }
+
+    /**
+     * @param string $uri
+     * @param array$data
+     * @param $request
+     * @return HttpSocketResponseExtended
+     */
+    public function put($uri = null, $data = [], $request = [])
+    {
+        return $this->internalRequest('PUT', $uri, $data, $request);
+    }
+
+    /**
+     * @param string $uri
+     * @param array $data
+     * @param array $request
+     * @return HttpSocketResponseExtended
+     */
+    public function patch($uri = null, $data = [], $request = [])
+    {
+        return $this->internalRequest('PATCH', $uri, $data, $request);
+    }
+
+    /**
+     * @param string $uri
+     * @param array $data
+     * @param array $request
+     * @return HttpSocketResponseExtended
+     */
+    public function delete($uri = null, $data = array(), $request = array())
+    {
+        return $this->internalRequest('DELETE', $uri, $data, $request);
+    }
+
+    public function url($url = null, $uriTemplate = null)
+    {
+        throw new Exception('Not implemented');
+    }
+
+    public function request($request = array())
+    {
+        throw new Exception('Not implemented');
+    }
+
+    public function setContentResource($resource)
+    {
+        throw new Exception('Not implemented');
+    }
+
+    public function getMetaData()
+    {
+        return null; // not supported by curl extension
+    }
+
+    /**
+     * @param string $host
+     * @param int $port
+     * @param string $method
+     * @param string $user
+     * @param string $pass
+     * @return void
+     */
+    public function configProxy($host, $port = 3128, $method = null, $user = null, $pass = null)
+    {
+        if (empty($host)) {
+            $this->proxy = [];
+            return;
+        }
+        if (is_array($host)) {
+            $this->proxy = $host + ['host' => null];
+            return;
+        }
+        $this->proxy = compact('host', 'port', 'method', 'user', 'pass');
+        $this->defaultOptions = $this->generateDefaultOptions(); // regenerate default options in case proxy setting is changed
+    }
+
+    /**
+     * @param string $method
+     * @param string $url
+     * @param array|string $query
+     * @param array $request
+     * @return HttpSocketResponseExtended
+     */
+    private function internalRequest($method, $url, $query, $request)
+    {
+        if (empty($url)) {
+            throw new InvalidArgumentException("No URL provided.");
+        }
+
+        if (!$this->ch) {
+            // Share handle between requests to allow keep connection alive between requests
+            $this->ch = curl_init();
+            if (!$this->ch) {
+                throw new \RuntimeException("Could not initialize curl");
+            }
+        } else {
+            // Reset options, so we can do another request
+            curl_reset($this->ch);
+        }
+
+        if (($method === 'GET' || $method === 'HEAD') && !empty($query)) {
+            $url .= '?' . http_build_query($query, '', '&', PHP_QUERY_RFC3986);
+        }
+
+        $options = $this->defaultOptions; // this will copy default options
+        $options[CURLOPT_URL] = $url;
+        $options[CURLOPT_CUSTOMREQUEST] = $method;
+
+        if (($method === 'POST' || $method === 'DELETE' || $method === 'PUT' || $method === 'PATCH') && !empty($query)) {
+            $options[CURLOPT_POSTFIELDS] = $query;
+        }
+
+        if ($method === 'HEAD') {
+            $options[CURLOPT_NOBODY] = true;
+        }
+
+        if (!empty($request['header'])) {
+            $headers = [];
+            foreach ($request['header'] as $key => $value) {
+                if (is_array($value)) {
+                    $value = implode(', ', $value);
+                }
+                $headers[] = "$key: $value";
+            }
+            $options[CURLOPT_HTTPHEADER] = $headers;
+        }
+
+        // Parse response headers
+        $responseHeaders = [];
+        $options[CURLOPT_HEADERFUNCTION] = function ($curl, $header) use (&$responseHeaders){
+            $len = strlen($header);
+            $header = explode(':', $header, 2);
+            if (count($header) < 2) { // ignore invalid headers
+                return $len;
+            }
+            $key = strtolower(trim($header[0]));
+            $value = trim($header[1]);
+
+            if (isset($responseHeaders[$key])) {
+                $responseHeaders[$key] = array_merge((array)$responseHeaders[$key], [$value]);
+            } else {
+                $responseHeaders[$key] = $value;
+            }
+            return $len;
+        };
+        if (!curl_setopt_array($this->ch, $options)) {
+            throw new \RuntimeException('curl error: Could not set options');
+        }
+
+        // Download the given URL, and return output
+        $output = curl_exec($this->ch);
+
+        if ($output === false) {
+            $errorCode = curl_errno($this->ch);
+            $errorMessage = curl_error($this->ch);
+            if (!empty($errorMessage)) {
+                $errorMessage = ": $errorMessage";
+            }
+            throw new SocketException("curl error $errorCode '" . curl_strerror($errorCode) . "'" . $errorMessage);
+        }
+
+        $code = curl_getinfo($this->ch, CURLINFO_HTTP_CODE);
+        return $this->constructResponse($output, $responseHeaders, $code);
+    }
+
+    public function disconnect()
+    {
+        if ($this->ch) {
+            curl_close($this->ch);
+            $this->ch = null;
+        }
+    }
+
+    /**
+     * @param string $body
+     * @param array $headers
+     * @param int $code
+     * @return HttpSocketResponseExtended
+     */
+    private function constructResponse($body, array $headers, $code)
+    {
+        $response = new HttpSocketResponseExtended();
+        $response->code = $code;
+        $response->body = $body;
+        $response->headers = $headers;
+        return $response;
+    }
+
+    /**
+     * @param int $cryptoMethod
+     * @return int
+     */
+    private function convertCryptoMethod($cryptoMethod)
+    {
+        switch ($cryptoMethod) {
+            case STREAM_CRYPTO_METHOD_TLSv1_0_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_1_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
+                return CURL_SSLVERSION_TLSv1;
+            case STREAM_CRYPTO_METHOD_TLSv1_1_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
+                return CURL_SSLVERSION_TLSv1_1;
+            case STREAM_CRYPTO_METHOD_TLSv1_2_CLIENT | STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
+                return CURL_SSLVERSION_TLSv1_2;
+            case STREAM_CRYPTO_METHOD_TLSv1_3_CLIENT:
+                return CURL_SSLVERSION_TLSv1_3;
+            default:
+                throw new InvalidArgumentException("Unsupported crypto method value $cryptoMethod");
+        }
+    }
+
+    /**
+     * @return array
+     */
+    private function generateDefaultOptions()
+    {
+        $options = [
+            CURLOPT_FOLLOWLOCATION => true, // Allows to follow redirect
+            CURLOPT_MAXREDIRS => 10,
+            CURLOPT_RETURNTRANSFER => true, // Should cURL return or print out the data? (true = return, false = print)
+            CURLOPT_HEADER => false, // Include header in result?
+            CURLOPT_TIMEOUT => $this->timeout, // Timeout in seconds
+            CURLOPT_PROTOCOLS => CURLPROTO_HTTPS | CURLPROTO_HTTP, // be sure that only HTTP and HTTPS protocols are enabled
+        ];
+
+        if ($this->caFile) {
+            $options[CURLOPT_CAINFO] = $this->caFile;
+        }
+
+        if ($this->localCert) {
+            $options[CURLOPT_SSLCERT] = $this->localCert;
+        }
+
+        if ($this->cryptoMethod) {
+            $options[CURLOPT_SSLVERSION] = $this->cryptoMethod;
+        }
+
+        if ($this->compress) {
+            $options[CURLOPT_ACCEPT_ENCODING] = ''; // empty string means all encodings supported by curl
+        }
+
+        if ($this->allowSelfSigned) {
+            $options[CURLOPT_SSL_VERIFYPEER] = $this->verifyPeer;
+            $options[CURLOPT_SSL_VERIFYHOST] = 0;
+        }
+
+        if (!empty($this->proxy)) {
+            $options[CURLOPT_PROXY] = "{$this->proxy['host']}:{$this->proxy['port']}";
+            if (!empty($this->proxy['method']) && isset($this->proxy['user'], $this->proxy['pass'])) {
+                $options[CURLOPT_PROXYUSERPWD] = "{$this->proxy['user']}:{$this->proxy['pass']}";
+            }
+        }
+
+        return $options;
+    }
+}

app/Lib/Tools/EventTimelineTool.php

@@ -146,7 +146,7 @@
                         'group' => 'object_attribute',
                         'timestamp' => $obj_attr['timestamp'],
                         'attribute_type' => $obj_attr['type'],
-                        'date_sighting' => $sightingsAttributeMap[$attr['id']] ?? [],
+                        'date_sighting' => $sightingsAttributeMap[$obj_attr['id']] ?? [],
                         'is_image' => $this->__eventModel->Attribute->isImage($obj_attr),
                     );
                     $toPush_obj['Attribute'][] = $toPush_attr;

app/Lib/Tools/GitTool.php

@@ -12,11 +12,7 @@ class GitTool
     public static function getLatestTags(HttpSocketExtended $HttpSocket)
     {
         $url = 'https://api.github.com/repos/MISP/MISP/tags?per_page=10';
-        $response = $HttpSocket->get($url);
-        if (!$response->isOk()) {
-            throw new HttpSocketHttpException($response, $url);
-        }
-        return $response->json();
+        return self::gitHubRequest($HttpSocket, $url);
     }
 
     /**
@@ -28,11 +24,7 @@ class GitTool
     public static function getLatestCommit(HttpSocketExtended $HttpSocket)
     {
         $url = 'https://api.github.com/repos/MISP/MISP/commits?per_page=1';
-        $response = $HttpSocket->get($url);
-        if (!$response->isOk()) {
-            throw new HttpSocketHttpException($response, $url);
-        }
-        $data = $response->json();
+        $data = self::gitHubRequest($HttpSocket, $url);
         if (!isset($data[0]['sha'])) {
             throw new Exception("Response do not contains requested data.");
         }
@@ -40,20 +32,49 @@ class GitTool
     }
 
     /**
+     * @param HttpSocketExtended $HttpSocket
+     * @param string $url
+     * @return array
+     * @throws HttpSocketHttpException
+     * @throws HttpSocketJsonException
+     */
+    private static function gitHubRequest(HttpSocketExtended $HttpSocket, $url)
+    {
+        $response = $HttpSocket->get($url, [], ['header' => ['User-Agent' => 'MISP']]);
+        if (!$response->isOk()) {
+            throw new HttpSocketHttpException($response, $url);
+        }
+        return $response->json();
+    }
+
+    /**
+     * Returns current SHA1 hash of current commit
      * `git rev-parse HEAD`
+     * @param string $repoPath
      * @return string
      * @throws Exception
      */
-    public static function currentCommit()
+    public static function currentCommit($repoPath)
     {
-        $head = rtrim(FileAccessTool::readFromFile(ROOT . '/.git/HEAD'));
+        if (is_file($repoPath . '/.git')) {
+            $fileContent = FileAccessTool::readFromFile($repoPath . '/.git');
+            if (substr($fileContent, 0, 8) === 'gitdir: ') {
+                $gitDir = $repoPath . '/' . trim(substr($fileContent, 8)) . '/';
+            } else {
+                throw new Exception("$repoPath/.git is file, but contains non expected content $fileContent");
+            }
+        } else {
+            $gitDir = $repoPath . '/.git/';
+        }
+
+        $head = rtrim(FileAccessTool::readFromFile($gitDir . 'HEAD'));
         if (substr($head, 0, 5) === 'ref: ') {
             $path = substr($head, 5);
-            return rtrim(FileAccessTool::readFromFile(ROOT . '/.git/' . $path));
+            return rtrim(FileAccessTool::readFromFile($gitDir . $path));
         }  else if (strlen($head) === 40) {
             return $head;
         } else {
-            throw new Exception("Invalid head $head");
+            throw new Exception("Invalid head '$head' in $gitDir/HEAD");
         }
     }
 
@@ -94,30 +115,18 @@ class GitTool
         return $output;
     }
 
-    /**
-     * @param string $submodule Path to Git repo
-     * @return string|null
-     */
-    public static function submoduleCurrentCommit($submodule)
-    {
-        try {
-            $commit = ProcessTool::execute(['git', 'rev-parse', 'HEAD'], $submodule);
-        } catch (ProcessException $e) {
-            return null;
-        }
-        return rtrim($commit);
-    }
-
     /**
      * @param string $commit
      * @param string|null $submodule Path to Git repo
      * @return int|null
+     * @throws Exception
      */
     public static function commitTimestamp($commit, $submodule = null)
     {
         try {
             $timestamp = ProcessTool::execute(['git', 'show', '-s', '--pretty=format:%ct', $commit], $submodule);
         } catch (ProcessException $e) {
+            CakeLog::notice("Could not get Git commit timestamp for $submodule: {$e->getMessage()}");
             return null;
         }
         return (int)rtrim($timestamp);

app/Lib/Tools/HttpSocketExtended.php

@@ -18,10 +18,15 @@ class HttpSocketHttpException extends Exception
     {
         $this->response = $response;
         $this->url = $url;
+
         $message = "Remote server returns HTTP error code $response->code";
         if ($url) {
             $message .= " for URL $url";
         }
+        if ($response->body) {
+            $message .= ': ' . substr(ltrim($response->body), 0, 100);
+        }
+
         parent::__construct($message, (int)$response->code);
     }
 
@@ -109,10 +114,15 @@ class HttpSocketResponseExtended extends HttpSocketResponse
      */
     public function json()
     {
+        if (strlen($this->body) === 0) {
+            throw new HttpSocketJsonException('Could not parse empty response as JSON.', $this);
+        }
+
         try {
             return JsonTool::decode($this->body);
         } catch (Exception $e) {
-            throw new HttpSocketJsonException('Could not parse response as JSON.', $this, $e);
+            $contentType = $this->getHeader('content-type');
+            throw new HttpSocketJsonException("Could not parse HTTP response as JSON. Received Content-Type $contentType.", $this, $e);
         }
     }
 }

app/Lib/Tools/JSONConverterTool.php

@@ -21,7 +21,7 @@ class JSONConverterTool
 
     public static function convertObject($object, $isSiteAdmin = false, $raw = false)
     {
-        $toRearrange = array('SharingGroup', 'Attribute', 'ShadowAttribute', 'Event', 'CryptographicKey');
+        $toRearrange = array('SharingGroup', 'Attribute', 'ShadowAttribute', 'Event', 'CryptographicKey', 'Note', 'Opinion', 'Relationship');
         foreach ($toRearrange as $element) {
             if (isset($object[$element])) {
                 $object['Object'][$element] = $object[$element];
@@ -40,7 +40,7 @@ class JSONConverterTool
 
     public static function convert($event, $isSiteAdmin=false, $raw = false)
     {
-        $toRearrange = array('Org', 'Orgc', 'SharingGroup', 'Attribute', 'ShadowAttribute', 'RelatedAttribute', 'RelatedEvent', 'Galaxy', 'Object', 'EventReport', 'CryptographicKey');
+        $toRearrange = array('Org', 'Orgc', 'SharingGroup', 'Attribute', 'ShadowAttribute', 'RelatedAttribute', 'RelatedEvent', 'Galaxy', 'Object', 'EventReport', 'CryptographicKey', 'Note', 'Opinion', 'Relationship');
         foreach ($toRearrange as $object) {
             if (isset($event[$object])) {
                 $event['Event'][$object] = $event[$object];
@@ -69,6 +69,16 @@ class JSONConverterTool
             }
         }
 
+        if (isset($event['Event']['Note'])) {
+            $event['Event']['Note'] = self::__cleanAnalystData($event['Event']['Note']);
+        }
+        if (isset($event['Event']['Opinion'])) {
+            $event['Event']['Opinion'] = self::__cleanAnalystData($event['Event']['Opinion']);
+        }
+        if (isset($event['Event']['Relationship'])) {
+            $event['Event']['Relationship'] = self::__cleanAnalystData($event['Event']['Relationship']);
+        }
+
         // cleanup the array from things we do not want to expose
         $tempSightings = array();
         if (!empty($event['Sighting'])) {
@@ -153,7 +163,7 @@ class JSONConverterTool
             return;
         }
         yield '{"Event":{';
-        $firstKey = key($event['Event']);
+        $firstKey = array_key_first($event['Event']);
         foreach ($event['Event'] as $key => $value) {
             if ($key === 'Attribute' || $key === 'Object') { // Encode every object or attribute separately
                 yield ($firstKey === $key ? '' : ',') . json_encode($key) . ":[";
@@ -209,6 +219,17 @@ class JSONConverterTool
         return $objects;
     }
 
+    private function __cleanAnalystData($data)
+    {
+        foreach ($data as $k => $entry) {
+            if (empty($entry['SharingGroup'])) {
+                unset($data[$k]['SharingGroup']);
+            }
+        }
+        $data = array_values($data);
+        return $data;
+    }
+
     public static function arrayPrinter($array, $root = true)
     {
         if (is_array($array)) {