Merge branch 'develop' into develop

.github/dependabot.yml

@@ -0,0 +1,17 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "composer"
+    directory: "/app"
+    schedule:
+      interval: "daily"
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      # Check for updates to GitHub Actions every weekday
+      interval: "daily"

.github/workflows/codeql-analysis.yml

@@ -35,11 +35,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -50,7 +50,7 @@ jobs:
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@v1
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -64,4 +64,4 @@ jobs:
     #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2

.github/workflows/main.yml

@@ -6,7 +6,7 @@ name: misp
 # events but only for the 2.4 and develop branches
 on:
   push:
-    branches: [ 2.4, develop, misp-stix ]
+    branches: [ 2.4, develop, misp-stix, taxii ]
   pull_request:
     branches: [ 2.4, develop, misp-stix ]
 
@@ -20,13 +20,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04]
+        os: [ubuntu-22.04]
         php: ['7.2', '7.3', '7.4']
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           submodules: 'recursive'
 
@@ -75,7 +75,8 @@ jobs:
         run: |
               sudo chown $USER:www-data $HOME/.composer
               pushd app
-              sudo -H -u $USER php composer.phar install --no-progress
+              sudo -H -u $USER composer config --no-plugins allow-plugins.composer/installers true
+              sudo -H -u $USER composer install --no-progress
               popd
               cp -fa INSTALL/setup/config.php app/Plugin/CakeResque/Config/config.php
               # Set perms
@@ -138,6 +139,13 @@ jobs:
               sudo chown -R $USER:www-data `pwd`/app/Config
               sudo chmod -R 777 `pwd`/app/Config
 
+              # fix perms (?)
+              namei -m  /home/runner/work
+              sudo chmod +x /home/runner/work
+              sudo chmod +x /home/runner
+              sudo chmod +x /home
+              sudo chmod +x /
+
       - name: DB Update
         run: |
               sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.osuser" $USER'
@@ -192,6 +200,9 @@ jobs:
       - name: Turn MISP live
         run: sudo -E su $USER -c 'app/Console/cake Live 1'
 
+      - name: Check if Redis is ready
+        run: sudo -E su $USER -c 'app/Console/cake Admin redisReady'
+
       - name: Start workers
         run: |
               sudo chmod +x app/Console/worker/start.sh
@@ -221,12 +232,11 @@ jobs:
               echo 'key = "'${AUTH}'"' >> tests/keys.py
               cat tests/keys.py
               popd
-
-      - name: Build test
-        run: |
               . ./venv/bin/activate
               pushd tests
-              ./build-test.sh
+              bash ./build-test.sh
+              popd
+              deactivate
 
       - name: Run PHP tests
         run: |
@@ -240,17 +250,15 @@ jobs:
               popd
 
               sudo chmod -R g+ws `pwd`/app/tmp/logs
-          
+
               . ./venv/bin/activate
               pushd PyMISP
               python tests/testlive_comprehensive.py
+              python tests/test_mispevent.py
               popd
               python tests/testlive_security.py -v
               python tests/testlive_sync.py
               python tests/testlive_comprehensive_local.py -v
-              pushd PyMISP
-              python tests/test_mispevent.py
-              popd
               cp PyMISP/tests/keys.py PyMISP/examples/events/
               pushd PyMISP/examples/events/
               python ./create_massive_dummy_events.py -l 5 -a 30
@@ -264,3 +272,5 @@ jobs:
               tail -n +1 `pwd`/app/tmp/logs/*
               tail -n +1 /var/log/apache2/*.log
 
+              sudo -u $USER app/Console/cake Log export /tmp/logs.json.gz --without-changes
+              zcat /tmp/logs.json.gz

.gitignore

@@ -104,6 +104,7 @@ app/Lib/EventWarning/Custom/*
 /app/tmp/cached_exports/sha256/*
 /app/tmp/cached_exports/bro/*
 /app/Plugin/CakeResque
+/app/Plugin/DebugKit
 .gnupg
 .smime
 *.swp

CODINGSTYLE.md

@@ -6,11 +6,11 @@ Maintaining proper coding style is very important for any large software project
 - It allows others (as well as the future you!) to easily understand fragments of code and what they were supposed to do, and thus makes it easier to later extend them with newer functionality or bug fixes
 - It allows others to easily review the code and catch bugs
 - It provides for an aesthetically pleasing experience when one reads the code
- 
+
 ## General typographic conventions
 - Maintain a maximum line length of 80 characters. Even though today’s monitors often are very wide and it’s often not a problem to have 120 characters displayed in an editor, maintaining shorter line lengths improves readability. It also allows others to have two parallel windows open, side by side, each with different parts of the source code.
 - Naming conventions:
-    - `ClassName`, 
+    - `ClassName`,
     - `someVariable`, `someFunction`, `someArgument`
 - Maintain a decent amount of horizontal spacing, e.g. add a space after `if` or before `{` in PHP, Python, JavaScript, and similar in other languages. Whether and where to also use spaces within expressions, such as `(y*4+8)` vs. `(y * 4 + 8)` is left to the developer’s judgment. Do not put spaces immediately after or before the brackets in expressions, so avoid constructs like this: `if ( condition )` and use ones like this: `if (condition)` instead.
 - Use descriptive names for variables and functions. At a time when most editors have auto-completion features, there is no excuse for using short variable names.
@@ -25,7 +25,7 @@ Maintaining proper coding style is very important for any large software project
 ## File naming conventions
 - Never use spaces within file names
 - **PHP:** Write file names in title case ,e.g. `AttachmentTool.php`
-- **Python:** Write file names with small letters, use a dash to separate words, rather than underscores, e.g. `load_warninglists.py`
+- **Python:** Write file names with small letters, use an underscore to separate words, rather than dashes, e.g. `load_warninglists.py`
 - **JavaScript:** Write file names with small letters, use dashes to separate words, rather than underscores, e.g. `bootstrap-colorpicker.js`
 
 ## General programming style guidelines
@@ -41,7 +41,7 @@ Maintaining proper coding style is very important for any large software project
         return style;
       }
 - In production code, there should be little to no commented or disabled code fragments. Do not use comments to disable code fragments, unless you need to. But generally, there is little excuse to keep old, unused code fragments in the code. Instead, use the functionality provided by the source code management system, such as git. For example, create a special branch for storing the old, unused code – this way you will always be able to merge this code into upstream in the future.
-- Try not to hardcode values in the code. 
+- Try not to hardcode values in the code.
 
 ## Commit message guidelines
 

INSTALL/INSTALL.sh

@@ -1509,9 +1509,17 @@ coreCAKE () {
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "GnuPG.homedir" "${PATH_TO_MISP}/.gnupg"
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "GnuPG.password" "${GPG_PASSPHRASE}"
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "GnuPG.obscure_subject" true
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "GnuPG.key_fetching_disabled" false
   # FIXME: what if we have not gpg binary but a gpg2 one?
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "GnuPG.binary" "$(which gpg)"
 
+  # LinOTP
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "LinOTPAuth.enabled" false
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "LinOTPAuth.baseUrl" "https://<your-linotp-baseUrl>"
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "LinOTPAuth.realm" "lino"
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "LinOTPAuth.verifyssl" true
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "LinOTPAuth.mixedauth" false
+
   # Enable installer org and tune some configurables
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.host_org_id" 1
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.email" "info@admin.test"
@@ -1870,7 +1878,7 @@ mispmodules () {
 modulesCAKE () {
   # Enable Enrichment, set better timeouts
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_services_enable" true
-  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_hover_enable" true
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_hover_enable" false
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_hover_popover_only" false
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_hover_timeout" 150
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_timeout" 300
@@ -2543,7 +2551,7 @@ apacheConfig_RHEL7 () {
   #sudo sed -i "s/SetHandler/\#SetHandler/g" /etc/httpd/conf.d/misp.ssl.conf
   sudo rm /etc/httpd/conf.d/ssl.conf
   sudo chmod 644 /etc/httpd/conf.d/misp.ssl.conf
-  sudo sed -i '/Listen 80/a Listen 443' /etc/httpd/conf/httpd.conf
+  sudo sed -i '/Listen 443/!s/Listen 80/a Listen 443/' /etc/httpd/conf/httpd.conf
 
   # If a valid SSL certificate is not already created for the server, create a self-signed certificate:
   echo "The Common Name used below will be: ${OPENSSL_CN}"
@@ -2591,7 +2599,7 @@ apacheConfig_RHEL8 () {
   #sudo sed -i "s/SetHandler/\#SetHandler/g" /etc/httpd/conf.d/misp.ssl.conf
   sudo rm /etc/httpd/conf.d/ssl.conf
   sudo chmod 644 /etc/httpd/conf.d/misp.ssl.conf
-  sudo sed -i '/Listen 80/a Listen 443' /etc/httpd/conf/httpd.conf
+  sudo sed -i '/Listen 443/!s/Listen 80/a Listen 443/' /etc/httpd/conf/httpd.conf
 
   # If a valid SSL certificate is not already created for the server, create a self-signed certificate:
   echo "The Common Name used below will be: ${OPENSSL_CN}"

INSTALL/INSTALL.sh.sfv

@@ -1,5 +1,5 @@
-; Generated by RHash v1.4.2 on 2022-05-23 at 12:45.34
+; Generated by RHash v1.4.2 on 2023-07-01 at 17:15.04
 ; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/
 ;
-;       160126  12:45.34 2022-05-23 INSTALL.sh
-INSTALL.sh 4296D40B11B3002DF3FDFD69A508ED5ECACB8C13 D32E5A4B0F37F4C937CD4F85927E998D917BCBE89E4E0E864FFD7EA09E29ADEF BD093D8018C351E3D3722646E269C4B60E6DA19F42150338CE6FD72FEE293B8B89AA69D48A84B19D3EFDDAE25EC9E646 ECACC3071E130058C3DDECC86E1CBF27DD4F11389D10F43B14293B1915F7A24F02D0DA51E299706A38C00F2D2A7505B0FE46E33B705E53594383CE65461F2B08
+;       160686  17:15.04 2023-07-01 INSTALL.sh
+INSTALL.sh 9576C31EC5BD942E1C9B12413E6408E4623252F7 78B708FE1FC6B39BE081B9F05C6AA5E1478F8762CAF5A8A7671A12EBA4D3C1C5 27991471FB5788F42AF3BBF86FC80A95341AA17AE9487016EEC94961A48437172702EB8E2D6CB300387E87D9E8E0E3E5 C1C21FD491AEFD662C87C3EF62837D769E63E9CF2446B9BD607CCEF8AFD72528824A8F408C6892FD51109390104010EF90DA7F4828950A8671D2986A6B8E216F

INSTALL/INSTALL.sh.sha1

@@ -1 +1 @@
-4296d40b11b3002df3fdfd69a508ed5ecacb8c13  INSTALL.sh
+9576c31ec5bd942e1c9b12413e6408e4623252f7  INSTALL.sh

INSTALL/INSTALL.sh.sha256

@@ -1 +1 @@
-d32e5a4b0f37f4c937cd4f85927e998d917bcbe89e4e0e864ffd7ea09e29adef  INSTALL.sh
+78b708fe1fc6b39be081b9f05c6aa5e1478f8762caf5a8a7671a12eba4d3c1c5  INSTALL.sh

INSTALL/INSTALL.sh.sha384

@@ -1 +1 @@
-bd093d8018c351e3d3722646e269c4b60e6da19f42150338ce6fd72fee293b8b89aa69d48a84b19d3efddae25ec9e646  INSTALL.sh
+27991471fb5788f42af3bbf86fc80a95341aa17ae9487016eec94961a48437172702eb8e2d6cb300387e87d9e8e0e3e5  INSTALL.sh

INSTALL/INSTALL.sh.sha512

@@ -1 +1 @@
-ecacc3071e130058c3ddecc86e1cbf27dd4f11389d10f43b14293b1915f7a24f02d0da51e299706a38c00f2d2a7505b0fe46e33b705e53594383ce65461f2b08  INSTALL.sh
+c1c21fd491aefd662c87c3ef62837d769e63e9cf2446b9bd607ccef8afd72528824a8f408c6892fd51109390104010ef90da7f4828950a8671d2986a6b8e216f  INSTALL.sh

INSTALL/MYSQL.sql

@@ -88,6 +88,12 @@ CREATE TABLE IF NOT EXISTS `attribute_tags` (
   INDEX `tag_id` (`tag_id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
+-- -------------------------------------------------------
+
+--
+-- Table structure for table `auth_keys`
+--
+
 CREATE TABLE IF NOT EXISTS `auth_keys` (
     `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
     `uuid` varchar(40) COLLATE utf8mb4_unicode_ci NOT NULL,
@@ -98,6 +104,8 @@ CREATE TABLE IF NOT EXISTS `auth_keys` (
     `expiration` int(10) unsigned NOT NULL,
     `user_id` int(10) unsigned NOT NULL,
     `comment` text COLLATE utf8mb4_unicode_ci,
+    `allowed_ips` text COLLATE utf8mb4_unicode_ci,
+    `unique_ips` text COLLATE utf8mb4_unicode_ci,
     PRIMARY KEY (`id`),
     KEY `authkey_start` (`authkey_start`),
     KEY `authkey_end` (`authkey_end`),
@@ -1523,16 +1531,16 @@ INSERT IGNORE INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `pe
 VALUES (2, 'Org Admin', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 0, 1, 0);
 
 INSERT IGNORE INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`)
-VALUES (3, 'User', NOW(), NOW(), 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1);
+VALUES (3, 'User', NOW(), NOW(), 1, 1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 1);
 
 INSERT IGNORE INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`)
-VALUES (4, 'Publisher', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0);
+VALUES (4, 'Publisher', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 1, 0);
 
 INSERT IGNORE INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`)
-VALUES (5, 'Sync user', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0);
+VALUES (5, 'Sync user', NOW(), NOW(), 1, 1, 1, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0);
 
 INSERT IGNORE INTO `roles` (`id`, `name`, `created`, `modified`, `perm_add`, `perm_modify`, `perm_modify_org`, `perm_publish`, `perm_publish_zmq`, `perm_publish_kafka`, `perm_sync`, `perm_admin`, `perm_audit`, `perm_full`, `perm_auth`, `perm_regexp_access`, `perm_tagger`, `perm_site_admin`, `perm_template`, `perm_sharing_group`, `perm_tag_editor`, `perm_delegate`, `perm_sighting`, `perm_object_template`, `perm_decaying`, `default_role`)
-VALUES (6, 'Read Only', NOW(), NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
+VALUES (6, 'Read Only', NOW(), NOW(), 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0);
 
 -- --------------------------------------------------------
 
@@ -1661,4 +1669,6 @@ INSERT IGNORE INTO `org_blocklists` (`org_uuid`, `created`, `org_name`, `comment
 ('58d38339-7b24-4386-b4b4-4c0f950d210f', NOW(), 'Setec Astrononomy', 'default example'),
 ('58d38326-eda8-443a-9fa8-4e12950d210f', NOW(), 'Acme Finance', 'default example');
 
-INSERT IGNORE INTO `admin_settings` (`setting`, `value`) VALUES ('fix_login', NOW());
+INSERT IGNORE INTO `admin_settings` (`setting`, `value`) VALUES 
+('fix_login', NOW()),
+('default_role', 3);

INSTALL/old/INSTALL.ubuntu1604.txt

@@ -1 +0,0 @@
-../../docs/archive/INSTALL.ubuntu1604.md

PyMISP

@@ -1 +1 @@
-Subproject commit 98bb5ebd49cf1ab3eb725922c5bbbc6369657b05
+Subproject commit 94983c01ecced6086df28133a38a297111534142

README.md

@@ -41,7 +41,7 @@ The objective of MISP is to foster the sharing of structured information within
 
 MISP, Malware Information Sharing Platform and Threat Sharing, core functionalities are:
 
-- An **efficient IOC and indicators** database allowing to store technical and non-technical information about malware samples, incidents, attackers and intelligence.
+- An **efficient IOC and indicators** database, allowing to store technical and non-technical information about malware samples, incidents, attackers and intelligence.
 - Automatic **correlation** finding relationships between attributes and indicators from malware, attack campaigns or analysis. The correlation engine includes correlation between attributes and more advanced correlations like Fuzzy hashing correlation (e.g. ssdeep) or CIDR block matching. Correlation can also be enabled or event disabled per attribute.
 - A **flexible data model** where complex [objects](https://www.misp-project.org/objects.html) can be expressed and **linked together to express threat intelligence, incidents or connected elements**.
 - Built-in **sharing functionality** to ease data sharing using different model of distributions. MISP can automatically synchronize events and attributes among different MISP instances. Advanced filtering functionalities can be used to meet each organization's sharing policy including a **flexible sharing group** capacity and an attribute level distribution mechanisms.
@@ -50,9 +50,9 @@ MISP, Malware Information Sharing Platform and Threat Sharing, core functionalit
 - **export**: generating IDS, OpenIOC, plain text, CSV, MISP XML or JSON output to integrate with other systems (network IDS, host IDS, custom tools), Cache format (used for forensic tools), STIX (XML and JSON) 1 and 2, NIDS export (Suricata, Snort and Bro/Zeek) or RPZ zone. Many other formats can be easily added via the [misp-modules](https://github.com/MISP/misp-modules).
 - **import**: bulk-import, batch-import, import from OpenIOC, GFI sandbox, ThreatConnect CSV, MISP standard format or STIX 1.1/2.0. Many other formats easily added via the [misp-modules](https://github.com/MISP/misp-modules).
 - Flexible **free text import** tool to ease the integration of unstructured reports into MISP.
-- A gentle system to **collaborate** on events and attributes allowing MISP users to propose changes or updates to attributes/indicators.
+- A user-friendly system to **collaborate** on events and attributes allowing MISP users to propose changes or updates to attributes/indicators.
 - **data-sharing**: automatically exchange and synchronize with other parties and trust-groups using MISP.
-- **delegating of sharing**: allows a simple pseudo-anonymous mechanism to delegate publication of event/indicators to another organization.
+- **delegating of sharing**: allows for a simple, pseudo-anonymous mechanism to delegate publication of event/indicators to another organization.
 - Flexible **API** to integrate MISP with your own solutions. MISP is bundled with [PyMISP](https://github.com/MISP/PyMISP) which is a flexible Python Library to fetch, add or update events attributes, handle malware samples or search for attributes. An exhaustive restSearch API to easily search for indicators in MISP and exports those in all the format supported by MISP.
 - **Adjustable taxonomy** to classify and tag events following your own classification schemes or [existing classification](https://github.com/MISP/misp-taxonomies). The taxonomy can be local to your MISP but also shareable among MISP instances.
 - **Intelligence vocabularies** called MISP galaxy and bundled with existing [threat actors, malware, RAT, ransomware or MITRE ATT&CK](https://www.misp-project.org/galaxy.html) which can be easily linked with events and attributes in MISP.
@@ -108,16 +108,16 @@ License
 
 This software is licensed under [GNU Affero General Public License version 3](http://www.gnu.org/licenses/agpl-3.0.html)
 
-* Copyright (C) 2012-2022 Christophe Vandeplas
+* Copyright (C) 2012-2023 Christophe Vandeplas
 * Copyright (C) 2012 Belgian Defence
 * Copyright (C) 2012 NATO / NCIRC
-* Copyright (C) 2013-2022 Andras Iklody
-* Copyright (C) 2015-2022 CIRCL - Computer Incident Response Center Luxembourg
+* Copyright (C) 2013-2023 Andras Iklody
+* Copyright (C) 2015-2023 CIRCL - Computer Incident Response Center Luxembourg
 * Copyright (C) 2016 Andreas Ziegler
-* Copyright (C) 2018-2022 Sami Mokaddem
-* Copyright (C) 2018-2022 Christian Studer
-* Copyright (C) 2015-2022 Alexandre Dulaunoy
+* Copyright (C) 2018-2023 Sami Mokaddem
+* Copyright (C) 2018-2023 Christian Studer
+* Copyright (C) 2015-2023 Alexandre Dulaunoy
 * Copyright (C) 2018-2022 Steve Clement
-* Copyright (C) 2020-2022 Jakub Onderka
+* Copyright (C) 2020-2023 Jakub Onderka
 
 For more information, [the list of authors and contributors](AUTHORS) is available.

VERSION.json

@@ -1 +1 @@
-{"major":2, "minor":4, "hotfix":164}
+{"major":2, "minor":4, "hotfix":174}

app/.htaccess

@@ -1,5 +1,5 @@
 <IfModule mod_rewrite.c>
     RewriteEngine on
-    RewriteRule    ^$    webroot/    [L]
-    RewriteRule    (.*) webroot/$1    [L]
-</IfModule>
+    RewriteRule    ^$    webroot/    "[B= ,L]"
+    RewriteRule    (.*) webroot/$1    "[B= ,L]"
+</IfModule>

app/Config/config.default.php

@@ -213,12 +213,13 @@ $config = array(
     // Warning: The following is a 3rd party contribution and still untested (including security) by the MISP-project team.
     // Feel free to enable it and report back to us if you run into any issues.
     //
-    // Uncomment the following to enable Kerberos authentication
+    // Uncomment the following to enable Kerberos/LDAP authentication
     // needs PHP LDAP support enabled (e.g. compile flag --with-ldap or Debian package php5-ldap)
     /*
-    'ApacheSecureAuth' => array( // Configuration for kerberos authentication
+    'ApacheSecureAuth' => array( // Configuration for kerberos/LDAP authentication
         'apacheEnv'          => 'REMOTE_USER',           // If proxy variable = HTTP_REMOTE_USER, If BasicAuth ldap = PHP_AUTH_USER
-        'ldapServer'         => 'ldap://example.com',   // FQDN or IP
+        'ldapServer'         => 'ldap://example.com',   // FQDN or IP, ldap:// for LDAP or LDAP+STARTTLS, ldaps:// for LDAPS
+        'starttls'           => true, // true for STARTTLS, ignored for LDAPS
         'ldapProtocol'       => 3,
         'ldapNetworkTimeout' => -1,  // use -1 for unlimited network timeout
         'ldapReaderUser'     => 'cn=userWithReadAccess,ou=users,dc=example,dc=com', // DN ou RDN LDAP with reader user right

app/Config/routes.php

@@ -33,6 +33,7 @@
 	Router::connect('/roles/admin_index/*', array('controller' => 'roles', 'action' => 'index', 'admin' => true));
 	Router::connect('/logs/admin_search/*', array('controller' => 'logs', 'action' => 'search', 'admin' => true));
 	Router::connect('/audit_logs/admin_index/*', array('controller' => 'audit_logs', 'action' => 'index', 'admin' => true));
+	Router::connect('/access_logs/admin_index/*', array('controller' => 'access_logs', 'action' => 'index', 'admin' => true));
 	Router::connect('/logs/admin_index/*', array('controller' => 'logs', 'action' => 'index', 'admin' => true));
 	Router::connect('/regexp/admin_index/*', array('controller' => 'regexp', 'action' => 'index', 'admin' => true));
 

app/Console/Command/AdminShell.php

@@ -112,6 +112,18 @@ class AdminShell extends AppShell
         return $parser;
     }
 
+    public function jobForgot()
+    {
+        if (empty($this->args[0])) {
+            die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Forgot'] . PHP_EOL);
+        }
+
+        $email = $this->args[0];
+        $ip = empty($this->args[1]) ? null : $this->args[1];
+        $jobId = empty($this->args[2]) ? null : $this->args[2];
+        $this->User->forgot($email, $ip, $jobId);
+    }
+
     public function jobGenerateCorrelation()
     {
         if (empty($this->args[0])) {
@@ -524,6 +536,7 @@ class AdminShell extends AppShell
             $this->out('Executing all updates to bring the database up to date with the current version.');
             $processId = empty($this->args[0]) ? false : $this->args[0];
             $this->Server->runUpdates(true, false, $processId);
+            $this->Server->cleanCacheFiles();
             $this->out('All updates completed.');
         } else {
             $this->error('This OS user is not allowed to run this command.', 'Run it under `www-data` or `httpd` or `apache` or `wwwrun` or set MISP.osuser in the configuration.' . PHP_EOL . 'You tried to run this command as: ' . $whoami);
@@ -554,8 +567,21 @@ class AdminShell extends AppShell
     public function redisReady()
     {
         try {
-            RedisTool::init()->ping();
-            $this->out('Successfully connected to Redis.');
+            $redis = RedisTool::init();
+            for ($i = 0; $i < 10; $i++) {
+                $persistence = $redis->info('persistence');
+                if (isset($persistence['loading']) && $persistence['loading']) {
+                    $this->out('Redis is still loading...');
+                    sleep(1);
+                } else {
+                    break;
+                }
+            }
+            if ($i === 9) {
+                $this->out('Redis is still loading, but we will continue.');
+            } else {
+                $this->out('Successfully connected to Redis.');
+            }
         } catch (Exception $e) {
             $this->error('Redis connection is not available', $e->getMessage());
         }
@@ -845,7 +871,7 @@ class AdminShell extends AppShell
         ]);
 
         foreach ($tables as $table) {
-            $dataSource->query('OPTIMISE TABLE ' . $dataSource->name($table));
+            $dataSource->query('OPTIMIZE TABLE ' . $dataSource->name($table));
             $progress->increment();
             $progress->draw();
         }

app/Console/Command/AppShell.php

@@ -89,7 +89,11 @@ abstract class AppShell extends Shell
     protected function getBackgroundJobsTool()
     {
         if (!isset($this->BackgroundJobsTool)) {
-            $this->BackgroundJobsTool = new BackgroundJobsTool(Configure::read('SimpleBackgroundJobs'));
+            $settings = ['enabled' => false];
+            if (!empty(Configure::read('SimpleBackgroundJobs.enabled'))) {
+                $settings = Configure::read('SimpleBackgroundJobs');
+            }
+            $this->BackgroundJobsTool = new BackgroundJobsTool($settings);
         }
         return $this->BackgroundJobsTool;
     }

app/Console/Command/EventShell.php

@@ -68,7 +68,7 @@ class EventShell extends AppShell
         $user = $this->getUser($userId);
 
         if (!file_exists($path)) {
-            $this->error("File '$path' does not exists.");
+            $this->error("File '$path' does not exist.");
         }
         if (!is_readable($path)) {
             $this->error("File '$path' is not readable.");
@@ -405,7 +405,17 @@ class EventShell extends AppShell
         $jobId = $this->args[2];
         $userId = $this->args[3];
         $user = $this->getUser($userId);
-        $job = $this->Job->read(null, $jobId);
+        $job = $this->Job->find('first', [
+            'recursive' => -1,
+            'conditions' => [
+                'Job.id' => $jobId
+            ]
+        ]);
+        if (empty($job)) {
+            $log = ClassRegistry::init('Log');
+            $log->createLogEntry($user, 'publish', 'Event', $id, 'Event (' . $id . '): could not be published - valid job not found.', '');
+            return true;
+        }
         $this->Event->Behaviors->unload('SysLogLogable.SysLogLogable');
         $result = $this->Event->publish($id, $passAlong);
         $job['Job']['progress'] = 100;
@@ -632,7 +642,7 @@ class EventShell extends AppShell
     {
         $user = $this->User->getAuthUser($userId, true);
         if (empty($user)) {
-            $this->error("User with ID $userId does not exists.");
+            $this->error("User with ID $userId does not exist.");
         }
         Configure::write('CurrentUserId', $user['id']); // for audit logging purposes
         return $user;

app/Console/Command/LogShell.php

@@ -3,26 +3,44 @@
 /**
  * @property Log $Log
  * @property AuditLog $AuditLog
+ * @property AccessLog $AccessLog
  * @property Server $Server
  */
 class LogShell extends AppShell
 {
-    public $uses = ['Log', 'AuditLog', 'Server'];
+    public $uses = ['Log', 'AuditLog', 'AccessLog', 'Server'];
 
     public function getOptionParser()
     {
         $parser = parent::getOptionParser();
         $parser->addSubcommand('auditStatistics', [
-            'help' => __('Show statistics from audit logs.'),
+            'help' => __('Show statistics for audit logs.'),
+        ]);
+        $parser->addSubcommand('accessStatistics', [
+            'help' => __('Show statistics for access logs.'),
         ]);
         $parser->addSubcommand('statistics', [
-            'help' => __('Show statistics from logs.'),
+            'help' => __('Show statistics for application logs.'),
         ]);
         $parser->addSubcommand('export', [
-            'help' => __('Export logs to compressed file in JSON Lines format (one JSON encoded line per entry).'),
+            'help' => __('Export application logs to compressed file in JSON Lines format (one JSON encoded line per entry).'),
+            'parser' => [
+                'arguments' => [
+                    'file' => ['help' => __('Path to output file'), 'required' => true],
+                ],
+                'options' => [
+                    'without-changes' => ['boolean' => true, 'help' => __('Do not include add, edit or delete actions.')],
+                ],
+            ],
+        ]);
+        $parser->addSubcommand('recompress', [
+            'help' => __('Recompress compressed data in logs.'),
+        ]);
+        $parser->addSubcommand('accessLogRetention', [
+            'help' => __('Delete logs that are older than specified duration.'),
             'parser' => array(
                 'arguments' => array(
-                    'file' => ['help' => __('Path to output file'), 'required' => true],
+                    'duration' => ['help' => __('Duration in days'), 'required' => true],
                 ),
             ),
         ]);
@@ -32,6 +50,7 @@ class LogShell extends AppShell
     public function export()
     {
         list($path) = $this->args;
+        $withoutChanges = $this->param('without-changes');
 
         if (file_exists($path)) {
             $this->error("File $path already exists");
@@ -42,21 +61,24 @@ class LogShell extends AppShell
             $this->error("Could not open $path for writing");
         }
 
-        $rows = $this->Log->query("SELECT TABLE_ROWS FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'logs';");
         /** @var ProgressShellHelper $progress */
         $progress = $this->helper('progress');
         $progress->init([
-            'total' => $rows[0]['TABLES']['TABLE_ROWS'], // just estimate, but fast
+            'total' => $this->Log->tableRows(), // just estimate, but fast
             'width' => 50,
         ]);
 
         $lastId = 0;
         while (true) {
+            $conditions = ['Log.id >' => $lastId]; // much faster than offset
+            if ($withoutChanges) {
+                $conditions['NOT'] = ['Log.action' => ['add', 'edit', 'delete']];
+            }
             $logs = $this->Log->find('all', [
-                'conditions' => ['id >' => $lastId], // much faster than offset
+                'conditions' => $conditions,
                 'recursive' => -1,
                 'limit' => 100000,
-                'order' => ['id ASC'],
+                'order' => ['Log.id ASC'],
             ]);
             if (empty($logs)) {
                 break;
@@ -77,7 +99,7 @@ class LogShell extends AppShell
                 if ($log['id'] > $lastId) {
                     $lastId = $log['id'];
                 }
-                $lines .= json_encode($log, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_THROW_ON_ERROR) . "\n";
+                $lines .= JsonTool::encode($log) . "\n";
             }
             if (gzwrite($file, $lines) === false) {
                 $this->error("Could not write data to $path");
@@ -144,7 +166,48 @@ class LogShell extends AppShell
         $this->out('Change field:');
         $this->out('-------------');
         $this->out(str_pad(__('Compressed items:'), 20) . $this->AuditLog->compressionStats['compressed']);
+        $this->out(str_pad(__('Total size:'), 20) . CakeNumber::toReadableSize($this->AuditLog->compressionStats['bytes_total']));
         $this->out(str_pad(__('Uncompressed size:'), 20) . CakeNumber::toReadableSize($this->AuditLog->compressionStats['bytes_uncompressed']));
         $this->out(str_pad(__('Compressed size:'), 20) . CakeNumber::toReadableSize($this->AuditLog->compressionStats['bytes_compressed']));
     }
+
+    public function accessStatistics()
+    {
+        $count = $this->AccessLog->find('count');
+        $first = $this->AccessLog->find('first', [
+            'recursive' => -1,
+            'fields' => ['created'],
+            'order' => ['id ASC'],
+        ]);
+        $last = $this->AccessLog->find('first', [
+            'recursive' => -1,
+            'fields' => ['created'],
+            'order' => ['id DESC'],
+        ]);
+
+        $this->out(str_pad(__('Count:'), 20) . $count);
+        $this->out(str_pad(__('First:'), 20) . $first['AccessLog']['created']);
+        $this->out(str_pad(__('Last:'), 20) . $last['AccessLog']['created']);
+
+        $usage = $this->Server->dbSpaceUsage()['access_logs'];
+        $this->out(str_pad(__('Data size:'), 20) . CakeNumber::toReadableSize($usage['data_in_bytes']));
+        $this->out(str_pad(__('Index size:'), 20) . CakeNumber::toReadableSize($usage['index_in_bytes']));
+        $this->out(str_pad(__('Reclaimable size:'), 20) . CakeNumber::toReadableSize($usage['reclaimable_in_bytes']), 2);
+    }
+
+    public function recompress()
+    {
+        $this->AuditLog->recompress();
+    }
+
+    public function accessLogRetention()
+    {
+        list($duration) = $this->args;
+        if ($duration <= 0 || !is_numeric($duration)) {
+            $this->error("Invalid duration specified.");
+        }
+        $duration = new DateTime("-$duration days");
+        $deleted = $this->AccessLog->deleteOldLogs($duration);
+        $this->out(__n("Deleted %s entry", "Deleted %s entries", $deleted, $deleted));
+    }
 }

app/Console/Command/Ls22Shell.php

@@ -29,6 +29,8 @@ class Ls22Shell extends AppShell
 
     public function getOptionParser()
     {
+        $this->stdout->styles('green', array('text' => 'green'));
+
         $parser = parent::getOptionParser();
         $parser->addSubcommand('enableTaxonomy', [
             'help' => __('Enable a taxonomy with all its tags.'),
@@ -101,6 +103,33 @@ class Ls22Shell extends AppShell
                 ),
             ),
         ]);
+        $parser->addSubcommand('setSetting', [
+            'help' => __('Set a setting on the given MISP instance(s).'),
+            'parser' => array(
+                'options' => array(
+                    'instances' => [
+                        'help' => 'Path to the instance file, by default "instances.csv" from the local directory',
+                        'short' => 'i',
+                        'required' => true
+                    ],
+                    'misp_url_filter' => [
+                        'help' => 'The url of the instance to execute changes on. If not set, all are updated.',
+                        'short' => 'm',
+                        'required' => false
+                    ],
+                    'setting' => [
+                        'help' => 'The setting to modify',
+                        'short' => 's',
+                        'required' => true
+                    ],
+                    'value' => [
+                        'help' => 'The value to set for the given setting',
+                        'short' => 'v',
+                        'required' => true
+                    ]
+                ),
+            ),
+        ]);
         $parser->addSubcommand('addWarninglist', [
             'help' => __('Inject warninglist'),
             'parser' => array(
@@ -363,6 +392,26 @@ class Ls22Shell extends AppShell
         }
     }
 
+    public function setSetting()
+    {
+        $setting = $this->param('setting');
+        $value = $this->param('value');
+        $this->__getInstances($this->param('instances'));
+        foreach ($this->__servers as $server) {
+            $HttpSocket = $this->Server->setupHttpSocket($server, null);
+            $request = $this->Server->setupSyncRequest($server, 'Server');
+            $payload = ["value" => $value];
+            $response = $HttpSocket->post($server['Server']['url'] . '/server/serverSettingsEdit/' . $setting, json_encode($value), $request);
+            $statusWrapped = sprintf(
+                '<%s>%s</%s>',
+                $response->isOk() ? 'info' : 'error',
+                $response->isOk() ? 'OK' : 'Setting updated',
+                $response->isOk() ? 'info' : 'error'
+            );
+            $this->out($server['Server']['url'] . ': ' . $statusWrapped, 1, Shell::NORMAL);
+        }
+    }
+
     public function scores()
     {
         $results = [];
@@ -379,7 +428,7 @@ class Ls22Shell extends AppShell
         }
         $HttpSocket = $this->Server->setupHttpSocket($server, null);
         $request = $this->Server->setupSyncRequest($server);
-        $response = $HttpSocket->get($server['Server']['url'] . '/organisations/index/scope:all', false, $request);
+        $response = $HttpSocket->get($server['Server']['url'] . '/organisations/index/scope:local', false, $request);
         $orgs = json_decode($response->body(), true);
         $this->out(__('Organisations fetched. %d found.', count($orgs)), 1, Shell::VERBOSE);
         $org_mapping = [];
@@ -390,21 +439,31 @@ class Ls22Shell extends AppShell
             if ($org['Organisation']['name'] === 'YT') {
                 continue;
             }
+            if ($org['Organisation']['name'] === 'ORGNAME') {
+                continue;
+            }
             $org_mapping[$org['Organisation']['name']] = $org['Organisation']['id'];
         }
-        if (!empty($this->param['from'])) {
-            $time_range[] = $this->param['from'];
+        $time_range = [];
+        if (!empty($this->param('from'))) {
+            $time_range[] = $this->param('from');
         }
-        if (!empty($this->param['to'])) {
+        if (!empty($this->param('to'))) {
             if (empty($time_range)) {
                 $time_range[] = '365d';
             }
-            $time_range[] = $this->param['to'];
+            $time_range[] = $this->param('to');
+        } else {
+            if (!empty($time_range)) {
+                $time_range[] = '0h';
+            }
         }
+        $event_extended_uuids = [];
+        $event_uuid_per_org = [];
         foreach ($org_mapping as $org_name => $org_id) {
-            $time_range = [];
             $params = [
-                'org' => $org_id
+                'org' => $org_id,
+                'includeWarninglistHits' => true,
             ];
             if (!empty($time_range)) {
                 $params['publish_timestamp'] = $time_range;
@@ -415,6 +474,7 @@ class Ls22Shell extends AppShell
             $results[$org_name] = [
                 'attribute_count' => 0,
                 'object_count' => 0,
+                'event_count' => count($events['response']),
                 'connected_elements' => 0,
                 'event_tags' => 0,
                 'attribute_tags' => 0,
@@ -423,9 +483,16 @@ class Ls22Shell extends AppShell
                 'attribute_attack' => 0,
                 'attribute_other' => 0,
                 'score' => 0,
-                'warnings' => 0
+                'warnings' => 0,
+                'events_extended' => 0,
+                'extending_events' => 0,
             ];
             foreach ($events['response'] as $event) {
+                $event_uuid_per_org[$event['Event']['uuid']] = $event['Event']['Orgc']['name'];
+                if (!empty($event['Event']['extends_uuid'])) {
+                    $event_extended_uuids[$event['Event']['Orgc']['name']][] = $event['Event']['extends_uuid'];
+                }
+
                 if (!empty($event['Event']['Tag'])) {
                     foreach ($event['Event']['Tag'] as $tag) {
                         if (substr($tag['name'], 0, 32) === 'misp-galaxy:mitre-attack-pattern') {
@@ -458,7 +525,7 @@ class Ls22Shell extends AppShell
                         }
                     }
                     if (!empty($attribute['warnings'])) {
-                        $result[$org_name]['warnings'] += 1;
+                        $results[$org_name]['warnings'] += 1;
                     }
                 }
                 $results[$org_name]['attribute_count'] += count($event['Event']['Attribute']);
@@ -485,6 +552,22 @@ class Ls22Shell extends AppShell
 
             }
         }
+
+        foreach ($event_extended_uuids as $orgc => $uuids) {
+            foreach ($uuids as $uuid) {
+                if (!empty($event_uuid_per_org[$uuid])) {
+                    $org_name =  $event_uuid_per_org[$uuid];
+                    if ($orgc != $org_name) {
+                        // Add point for org extending another event
+                        $results[$orgc]['extending_events'] += 1;
+                        // Add point for org getting their event extended
+                        $results[$org_name]['events_extended'] += 1;
+                    }
+                }
+            }
+        }
+
+
         $scores = [];
         foreach ($results as $k => $result) {
             $totalCount = $result['attribute_count'] + $result['object_count'];
@@ -499,8 +582,10 @@ class Ls22Shell extends AppShell
                 $results[$k]['metrics']['connectedness'] = 100 * ($result['connected_elements'] / ($result['attribute_count'] + $result['object_count']));
                 $results[$k]['metrics']['attack_weight'] = 100 * (2*($result['attack']) + $result['attribute_attack']) / ($result['attribute_count'] + $result['object_count']);
                 $results[$k]['metrics']['other_weight'] = 100 * (2*($result['other']) + $result['attribute_other']) / ($result['attribute_count'] + $result['object_count']);
+                $results[$k]['metrics']['collaboration'] = 100 * ((2*$result['events_extended'] + $result['extending_events']) / $result['event_count']);
+                $results[$k]['metrics']['collaboration'] = 100 * (2*(2*$result['events_extended'] + $result['extending_events']) / $result['event_count']);
             }
-            foreach (['connectedness', 'attack_weight', 'other_weight', 'warnings'] as $metric) {
+            foreach (['connectedness', 'attack_weight', 'other_weight', 'warnings', 'collaboration'] as $metric) {
                 if (empty($results[$k]['metrics'][$metric])) {
                     $results[$k]['metrics'][$metric] = 0;
                 }
@@ -512,13 +597,15 @@ class Ls22Shell extends AppShell
                     20 * $results[$k]['metrics']['warnings'] +
                     20 * $results[$k]['metrics']['connectedness'] +
                     40 * $results[$k]['metrics']['attack_weight'] +
-                    20 * $results[$k]['metrics']['other_weight']
+                    10 * $results[$k]['metrics']['other_weight'] +
+                    10 * $results[$k]['metrics']['collaboration']
                 ) / 100;
             $scores[$k]['total'] = $results[$k]['score'];
             $scores[$k]['warnings'] = round(20 * $results[$k]['metrics']['warnings']);
             $scores[$k]['connectedness'] = round(20 * $results[$k]['metrics']['connectedness']);
             $scores[$k]['attack_weight'] = round(40 * $results[$k]['metrics']['attack_weight']);
-            $scores[$k]['other_weight'] = round(20 * $results[$k]['metrics']['other_weight']);
+            $scores[$k]['other_weight'] = round(10 * $results[$k]['metrics']['other_weight']);
+            $scores[$k]['collaboration'] = round(10 * $results[$k]['metrics']['collaboration']);
         }
         arsort($scores, SORT_DESC);
         $this->out(str_repeat('=', 128), 1, Shell::NORMAL);
@@ -534,15 +621,17 @@ class Ls22Shell extends AppShell
             $score_string[1] = str_repeat('█', round($score['connectedness']/100));
             $score_string[2] = str_repeat('█', round($score['attack_weight']/100));
             $score_string[3] = str_repeat('█', round($score['other_weight']/100));
+            $score_string[4] = str_repeat('█', round($score['collaboration']/100));
             $this->out(sprintf(
                 '| %s | %s | %s |',
                 str_pad($org, 10, ' ', STR_PAD_RIGHT),
                 sprintf(
-                    '<error>%s</error><warning>%s</warning><question>%s</question><info>%s</info>%s',
+                    '<error>%s</error><warning>%s</warning><question>%s</question><info>%s</info><green>%s</green>%s',
                     $score_string[0],
                     $score_string[1],
                     $score_string[2],
                     $score_string[3],
+                    $score_string[4],
                     str_repeat(' ', 100 - mb_strlen(implode('', $score_string)))
                 ),
                 str_pad($score['total'] . '%', 8, ' ', STR_PAD_RIGHT)
@@ -555,6 +644,7 @@ class Ls22Shell extends AppShell
             '<warning>█: Connectedness</warning>',
             '<question>█: ATT&CK context</question>',
             '<info>█: Other Context</info>',
+            '<green>█: Collaboration</green>',
             str_repeat(' ', 52)
         ), 1, Shell::NORMAL);
         $this->out(str_repeat('=', 128), 1, Shell::NORMAL);

app/Console/Command/ServerShell.php

@@ -11,7 +11,7 @@ require_once 'AppShell.php';
  */
 class ServerShell extends AppShell
 {
-    public $uses = array('Server', 'Task', 'Job', 'User', 'Feed');
+    public $uses = array('Server', 'Task', 'Job', 'User', 'Feed', 'TaxiiServer');
 
     public function getOptionParser()
     {
@@ -790,4 +790,30 @@ class ServerShell extends AppShell
         }
         return $server;
     }
+
+    public function push_taxii()
+    {
+        if (empty($this->args[0]) || empty($this->args[1])) {
+            die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Push Taxii'] . PHP_EOL);
+        }
+
+        $userId = $this->args[0];
+        $user = $this->getUser($userId);
+        $serverId = $this->args[1];
+        if (!empty($this->args[2])) {
+            $jobId = $this->args[2];
+        } else {
+            $jobId = $this->Job->createJob($user, Job::WORKER_DEFAULT, 'push_taxii', 'Server: ' . $serverId, 'Pushing.');
+        }
+        $this->Job->read(null, $jobId);
+
+        $result = $this->TaxiiServer->push($serverId, $user, $jobId);
+        if ($result !== true && !is_array($result)) {
+            $message = 'Job failed. Reason: ' . $result;
+            $this->Job->saveStatus($jobId, false, $message);
+        } else {
+            $message = 'Job done.';
+            $this->Job->saveStatus($jobId, true, $message);
+        }
+    }
 }

app/Controller/AccessLogsController.php

@@ -0,0 +1,228 @@
+<?php
+App::uses('AppController', 'Controller');
+
+/**
+ * @property AccessLog $AccessLog
+ */
+class AccessLogsController extends AppController
+{
+    public $components = [
+        'RequestHandler',
+    ];
+
+    public $paginate = [
+        'recursive' => -1,
+        'limit' => 60,
+        'fields' => ['id', 'created', 'user_id', 'org_id', 'authkey_id', 'ip', 'request_method', 'user_agent', 'request_id', 'controller', 'action', 'url', 'response_code', 'memory_usage', 'duration', 'query_count'],
+        'contain' => [
+            'User' => ['fields' => ['id', 'email', 'org_id']],
+            'Organisation' => ['fields' => ['id', 'name', 'uuid']],
+        ],
+        'order' => [
+            'AccessLog.id' => 'DESC'
+        ],
+    ];
+
+    public function admin_index()
+    {
+        $params = $this->IndexFilter->harvestParameters([
+            'created',
+            'ip',
+            'user',
+            'org',
+            'request_id',
+            'authkey_id',
+            'api_request',
+            'request_method',
+            'controller',
+            'action',
+            'url',
+            'user_agent',
+            'memory_usage',
+            'duration',
+            'query_count',
+            'response_code',
+        ]);
+
+        $conditions =  $this->__searchConditions($params);
+
+        if ($this->_isRest()) {
+            $list = $this->AccessLog->find('all', [
+                'conditions' => $conditions,
+                'contain' => $this->paginate['contain'],
+            ]);
+            foreach ($list as &$item) {
+                if (!empty($item['AccessLog']['request'])) {
+                    $item['AccessLog']['request'] = base64_encode($item['AccessLog']['request']);
+                }
+            }
+            return $this->RestResponse->viewData($list, 'json');
+        }
+        if (empty(Configure::read('MISP.log_skip_access_logs_in_application_logs'))) {
+            $this->Flash->warning(__('Access logs are logged in both application logs and access logs. Make sure you reconfigure your log monitoring tools and update MISP.log_skip_access_logs_in_application_logs.'));
+        }
+
+        $this->AccessLog->virtualFields['has_query_log'] = 'query_log IS NOT NULL';
+        $this->paginate['fields'][] = 'has_query_log';
+        $this->paginate['conditions'] = $conditions;
+        $list = $this->paginate();
+
+        $this->set('list', $list);
+        $this->set('title_for_layout', __('Access logs'));
+    }
+
+    public function admin_request($id)
+    {
+        $request = $this->AccessLog->find('first', [
+            'conditions' => ['AccessLog.id' => $id],
+            'fields' => ['AccessLog.request'],
+        ]);
+        if (empty($request)) {
+            throw new NotFoundException(__('Access log not found'));
+        }
+
+        if (empty($request['AccessLog']['request'])) {
+            throw new NotFoundException(__('Request body is empty'));
+        }
+
+        $contentType = explode(';', $request['AccessLog']['request_content_type'], 2)[0];
+        if ($contentType === 'application/x-www-form-urlencoded' || $contentType === 'multipart/form-data') {
+            parse_str($request['AccessLog']['request'], $output);
+            // highlight PHP array
+            $highlighted = highlight_string("<?php " . var_export($output, true), true);
+            $highlighted = trim($highlighted);
+            $highlighted = preg_replace("|^\\<code\\>\\<span style\\=\"color\\: #[a-fA-F0-9]{0,6}\"\\>|", "", $highlighted, 1);  // remove prefix
+            $highlighted = preg_replace("|\\</code\\>\$|", "", $highlighted, 1);  // remove suffix 1
+            $highlighted = trim($highlighted);  // remove line breaks
+            $highlighted = preg_replace("|\\</span\\>\$|", "", $highlighted, 1);  // remove suffix 2
+            $highlighted = trim($highlighted);  // remove line breaks
+            $highlighted = preg_replace("|^(\\<span style\\=\"color\\: #[a-fA-F0-9]{0,6}\"\\>)(&lt;\\?php&nbsp;)(.*?)(\\</span\\>)|", "\$1\$3\$4", $highlighted);  // remove custom added "<?php "
+            $data = $highlighted;
+        } else {
+            $data = h($request['AccessLog']['request']);
+        }
+
+        $this->set('request', $data);
+    }
+
+    public function admin_queryLog($id)
+    {
+        $request = $this->AccessLog->find('first', [
+            'conditions' => ['AccessLog.id' => $id],
+            'fields' => ['AccessLog.query_log'],
+        ]);
+        if (empty($request)) {
+            throw new NotFoundException(__('Access log not found'));
+        }
+
+        if (empty($request['AccessLog']['query_log'])) {
+            throw new NotFoundException(__('Query log is empty'));
+        }
+
+        $this->set('queryLog', $request['AccessLog']['query_log']);
+    }
+
+    /**
+     * @param array $params
+     * @return array
+     */
+    private function __searchConditions(array $params)
+    {
+        $qbRules = [];
+        foreach ($params as $key => $value) {
+            if ($key === 'created') {
+                $qbRules[] = [
+                    'id' => $key,
+                    'operator' => is_array($value) ? 'between' : 'greater_or_equal',
+                    'value' => $value,
+                ];
+            } else {
+                if (is_array($value)) {
+                    $value = implode('||', $value);
+                }
+                $qbRules[] = [
+                    'id' => $key,
+                    'value' => $value,
+                ];
+            }
+        }
+        $this->set('qbRules', $qbRules);
+
+        $conditions = [];
+        if (isset($params['user'])) {
+            if (is_numeric($params['user'])) {
+                $conditions['AccessLog.user_id'] = $params['user'];
+            } else {
+                $user = $this->User->find('first', [
+                    'conditions' => ['User.email' => $params['user']],
+                    'fields' => ['id'],
+                ]);
+                if (!empty($user)) {
+                    $conditions['AccessLog.user_id'] = $user['User']['id'];
+                } else {
+                    $conditions['AccessLog.user_id'] = -1;
+                }
+            }
+        }
+        if (isset($params['ip'])) {
+            $conditions['AccessLog.ip'] = inet_pton($params['ip']);
+        }
+        foreach (['authkey_id', 'request_id', 'controller', 'action'] as $field) {
+            if (isset($params[$field])) {
+                $conditions['AccessLog.' . $field] = $params[$field];
+            }
+        }
+        if (isset($params['url'])) {
+            $conditions['AccessLog.url LIKE'] = "%{$params['url']}%";
+        }
+        if (isset($params['user_agent'])) {
+            $conditions['AccessLog.user_agent LIKE'] = "%{$params['user_agent']}%";
+        }
+        if (isset($params['memory_usage'])) {
+            $conditions['AccessLog.memory_usage >='] = ($params['memory_usage'] * 1024);
+        }
+        if (isset($params['memory_usage'])) {
+            $conditions['AccessLog.memory_usage >='] = ($params['memory_usage'] * 1024);
+        }
+        if (isset($params['duration'])) {
+            $conditions['AccessLog.duration >='] = $params['duration'];
+        }
+        if (isset($params['query_count'])) {
+            $conditions['AccessLog.query_count >='] = $params['query_count'];
+        }
+        if (isset($params['request_method'])) {
+            $methodId = array_flip(AccessLog::REQUEST_TYPES)[$params['request_method']] ?? -1;
+            $conditions['AccessLog.request_method'] = $methodId;
+        }
+        if (isset($params['org'])) {
+            if (is_numeric($params['org'])) {
+                $conditions['AccessLog.org_id'] = $params['org'];
+            } else {
+                $org = $this->AccessLog->Organisation->fetchOrg($params['org']);
+                if ($org) {
+                    $conditions['AccessLog.org_id'] = $org['id'];
+                } else {
+                    $conditions['AccessLog.org_id'] = -1;
+                }
+            }
+        }
+        if (isset($params['created'])) {
+            $tempData = is_array($params['created']) ? $params['created'] : [$params['created']];
+            foreach ($tempData as $k => $v) {
+                $tempData[$k] = $this->AccessLog->resolveTimeDelta($v);
+            }
+            if (count($tempData) === 1) {
+                $conditions['AccessLog.created >='] = date("Y-m-d H:i:s", $tempData[0]);
+            } else {
+                if ($tempData[0] < $tempData[1]) {
+                    $temp = $tempData[1];
+                    $tempData[1] = $tempData[0];
+                    $tempData[0] = $temp;
+                }
+                $conditions['AND'][] = ['AccessLog.created <=' => date("Y-m-d H:i:s", $tempData[0])];
+                $conditions['AND'][] = ['AccessLog.created >=' => date("Y-m-d H:i:s", $tempData[1])];
+            }
+        }
+        return $conditions;
+    }
+}

app/Controller/AppController.php

@@ -33,8 +33,8 @@ class AppController extends Controller
 
     public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
 
-    private $__queryVersion = '146';
-    public $pyMispVersion = '2.4.162';
+    private $__queryVersion = '153';
+    public $pyMispVersion = '2.4.174';
     public $phpmin = '7.2';
     public $phprec = '7.4';
     public $phptoonew = '8.0';
@@ -43,7 +43,6 @@ class AppController extends Controller
     private $isApiAuthed = false;
 
     public $baseurl = '';
-    public $sql_dump = false;
 
     public $restResponsePayload = null;
 
@@ -102,7 +101,9 @@ class AppController extends Controller
     {
         $controller = $this->request->params['controller'];
         $action = $this->request->params['action'];
-
+        if (empty($this->Session->read('creation_timestamp'))) {
+            $this->Session->write('creation_timestamp', time());
+        }
         if (Configure::read('MISP.system_setting_db')) {
             App::uses('SystemSetting', 'Model');
             SystemSetting::setGlobalSetting();
@@ -136,17 +137,12 @@ class AppController extends Controller
             $this->response->header('X-XSS-Protection', '1; mode=block');
         }
 
-        if (!empty($this->request->params['named']['sql'])) {
-            $this->sql_dump = intval($this->request->params['named']['sql']);
-        }
-
         $this->_setupDatabaseConnection();
 
         $this->set('debugMode', Configure::read('debug') >= 1 ? 'debugOn' : 'debugOff');
         $isAjax = $this->request->is('ajax');
         $this->set('ajax', $isAjax);
         $this->set('queryVersion', $this->__queryVersion);
-        $this->User = ClassRegistry::init('User');
 
         $language = Configure::read('MISP.language');
         if (!empty($language) && $language !== 'eng') {
@@ -155,6 +151,26 @@ class AppController extends Controller
             Configure::write('Config.language', 'eng');
         }
 
+        $this->User = ClassRegistry::init('User');
+
+        if (!empty($this->request->params['named']['disable_background_processing'])) {
+            Configure::write('MISP.background_jobs', 0);
+        }
+
+        Configure::write('CurrentController', $controller);
+        Configure::write('CurrentAction', $action);
+        $versionArray = $this->User->checkMISPVersion();
+        $this->mispVersion = implode('.', $versionArray);
+        $this->Security->blackHoleCallback = 'blackHole';
+
+        // send users away that are using ancient versions of IE
+        // Make sure to update this if IE 20 comes out :)
+        if (isset($_SERVER['HTTP_USER_AGENT'])) {
+            if (preg_match('/(?i)msie [2-8]/', $_SERVER['HTTP_USER_AGENT']) && !strpos($_SERVER['HTTP_USER_AGENT'], 'Opera')) {
+                throw new MethodNotAllowedException('You are using an unsecure and outdated version of IE, please download Google Chrome, Mozilla Firefox or update to a newer version of IE. If you are running IE9 or newer and still receive this error message, please make sure that you are not running your browser in compatibility mode. If you still have issues accessing the site, get in touch with your administration team at ' . Configure::read('MISP.contact'));
+            }
+        }
+
         // For fresh installation (salt empty) generate a new salt
         if (!Configure::read('Security.salt')) {
             $this->User->Server->serverSettingsSaveValue('Security.salt', $this->User->generateRandomPassword(32));
@@ -165,6 +181,10 @@ class AppController extends Controller
             $this->User->Server->serverSettingsSaveValue('MISP.uuid', CakeText::uuid());
         }
 
+        /**
+         * Authentication related activities
+         */
+
         // Check if Apache provides kerberos authentication data
         $authUserFields = $this->User->describeAuthFields();
         $envvar = Configure::read('ApacheSecureAuth.apacheEnv');
@@ -180,22 +200,7 @@ class AppController extends Controller
         } else {
             $this->Auth->authenticate[AuthComponent::ALL]['userFields'] = $authUserFields;
         }
-        if (!empty($this->request->params['named']['disable_background_processing'])) {
-            Configure::write('MISP.background_jobs', 0);
-        }
-        Configure::write('CurrentController', $controller);
-        Configure::write('CurrentAction', $action);
-        $versionArray = $this->User->checkMISPVersion();
-        $this->mispVersion = implode('.', $versionArray);
-        $this->Security->blackHoleCallback = 'blackHole';
 
-        // send users away that are using ancient versions of IE
-        // Make sure to update this if IE 20 comes out :)
-        if (isset($_SERVER['HTTP_USER_AGENT'])) {
-            if (preg_match('/(?i)msie [2-8]/', $_SERVER['HTTP_USER_AGENT']) && !strpos($_SERVER['HTTP_USER_AGENT'], 'Opera')) {
-                throw new MethodNotAllowedException('You are using an unsecure and outdated version of IE, please download Google Chrome, Mozilla Firefox or update to a newer version of IE. If you are running IE9 or newer and still receive this error message, please make sure that you are not running your browser in compatibility mode. If you still have issues accessing the site, get in touch with your administration team at ' . Configure::read('MISP.contact'));
-            }
-        }
         $userLoggedIn = false;
         if (Configure::read('Plugin.CustomAuth_enable')) {
             $userLoggedIn = $this->__customAuthentication($_SERVER);
@@ -217,16 +222,19 @@ class AppController extends Controller
             !$userLoggedIn &&
             (
                 $controller !== 'users' ||
-                $action !== 'register' ||
-                empty(Configure::read('Security.allow_self_registration'))
+                (
+                    ($action !== 'register' || empty(Configure::read('Security.allow_self_registration'))) &&
+                    (!in_array($action, ['forgot', 'password_reset']) || empty(Configure::read('Security.allow_password_forgotten')))
+                )
             )
         ) {
             // REST authentication
             if ($this->_isRest() || $this->_isAutomation()) {
                 // disable CSRF for REST access
                 $this->Security->csrfCheck = false;
-                if ($this->__loginByAuthKey() === false || $this->Auth->user() === null) {
-                    if ($this->__loginByAuthKey() === null) {
+                $loginByAuthKeyResult = $this->__loginByAuthKey();
+                if ($loginByAuthKeyResult === false || $this->Auth->user() === null) {
+                    if ($loginByAuthKeyResult === null) {
                         $this->loadModel('Log');
                         $this->Log->createLogEntry('SYSTEM', 'auth_fail', 'User', 0, "Failed API authentication. No authkey was provided.");
                     }
@@ -304,10 +312,14 @@ class AppController extends Controller
             $this->__accessMonitor($user);
 
         } else {
-            $preAuthActions = array('login', 'register', 'getGpgPublicKey');
+            $preAuthActions = array('login', 'register', 'getGpgPublicKey', 'logout401', 'otp');
             if (!empty(Configure::read('Security.email_otp_enabled'))) {
                 $preAuthActions[] = 'email_otp';
             }
+            if (!empty(Configure::read('Security.allow_password_forgotten'))) {
+                $preAuthActions[] = 'forgot';
+                $preAuthActions[] = 'password_reset';
+            }
             if (!$this->_isControllerAction(['users' => $preAuthActions, 'servers' => ['cspReport']])) {
                 if ($isAjax) {
                     $response = $this->RestResponse->throwException(401, "Unauthorized");
@@ -411,9 +423,12 @@ class AppController extends Controller
                 }
             }
             if ($foundMispAuthKey) {
-                $authKeyToStore = substr($authKey, 0, 4)
+                $start = substr($authKey, 0, 4);
+                $end = substr($authKey, -4);
+                $authKeyToStore = $start
                     . str_repeat('*', 32)
-                    . substr($authKey, -4);
+                    . $end;
+                $this->__logApiKeyUse($start . $end);
                 if ($user) {
                     // User found in the db, add the user info to the session
                     if (Configure::read('MISP.log_auth')) {
@@ -430,10 +445,7 @@ class AppController extends Controller
                         );
                         $this->Log->save($log);
                     }
-                    $storeAPITime = Configure::read('MISP.store_api_access_time');
-                    if (!empty($storeAPITime) && $storeAPITime) {
-                        $this->User->updateAPIAccessTime($user);
-                    }
+                    $this->User->updateAPIAccessTime($user);
                     $this->Session->renew();
                     $this->Session->write(AuthComponent::$sessionKey, $user);
                     $this->isApiAuthed = true;
@@ -447,6 +459,9 @@ class AppController extends Controller
                     }
                     $this->Session->destroy();
                 }
+            } else {
+                    $this->loadModel('Log');
+                    $this->Log->createLogEntry('SYSTEM', 'auth_fail', 'User', 0, "Failed authentication using an API key of incorrect length.");
             }
             return false;
         }
@@ -483,7 +498,6 @@ class AppController extends Controller
         if (!$userFromDb) {
             $message = __('Something went wrong. Your user account that you are authenticated with doesn\'t exist anymore.');
             if ($this->_isRest()) {
-                // TODO: Why not exception?
                 $response = $this->RestResponse->throwException(401, $message);
                 $response->send();
                 $this->_stop();
@@ -513,12 +527,24 @@ class AppController extends Controller
                 }
                 $this->Flash->info($message);
                 $this->Auth->logout();
-                throw new MethodNotAllowedException($message);//todo this should pb be removed?
+                $this->_redirectToLogin();
+                return false;
             } else {
                 $this->Flash->error(__('Warning: MISP is currently disabled for all users. Enable it in Server Settings (Administration -> Server Settings -> MISP tab -> live). An update might also be in progress, you can see the progress in ') , array('params' => array('url' => $this->baseurl . '/servers/updateProgress/', 'urlName' => __('Update Progress')), 'clear' => 1));
             }
         }
 
+        // kill existing sessions for a user if the admin/instance decides so
+        // exclude API authentication as it doesn't make sense
+        if (!$this->isApiAuthed && $this->User->checkForSessionDestruction($user['id'])) {
+            $this->Auth->logout();
+            $this->Session->destroy();
+            $message = __('User deauthenticated on administrator request. Please reauthenticate.');
+            $this->Flash->warning($message);
+            $this->_redirectToLogin();
+            return false;
+        }
+
         // Force logout doesn't make sense for API key authentication
         if (!$this->isApiAuthed && $user['force_logout']) {
             $this->User->id = $user['id'];
@@ -581,6 +607,12 @@ class AppController extends Controller
             return true;
         }
 
+        // Check if user must create TOTP secret, force them to be on that page as long as needed.
+        if (empty($user['totp']) && Configure::read('Security.otp_required') && !$this->_isControllerAction(['users' => ['terms', 'change_pw', 'logout', 'login', 'totp_new']])) {  // TOTP is mandatory for users, prevent login until the user has configured their TOTP
+            $this->redirect(array('controller' => 'users', 'action' => 'totp_new', 'admin' => false));
+            return false;
+        }
+
         // Check if user accepted terms and conditions
         if (!$user['termsaccepted'] && !empty(Configure::read('MISP.terms_file')) && !$this->_isControllerAction(['users' => ['terms', 'logout', 'login', 'downloadTerms']])) {
             //if ($this->_isRest()) throw new MethodNotAllowedException('You have not accepted the terms of use yet, please log in via the web interface and accept them.');
@@ -621,6 +653,15 @@ class AppController extends Controller
         return in_array($this->request->params['action'], $actionsToCheck[$controller], true);
     }
 
+    private function __logApiKeyUse($apikey)
+    {
+        $redis = $this->User->setupRedis();
+        if (!$redis) {
+            return;
+        }
+        $redis->zIncrBy('misp:authkey_log:' . date("Ymd"), 1, $apikey);
+    }
+
     /**
      * User access monitoring
      * @param array $user
@@ -665,11 +706,28 @@ class AppController extends Controller
     {
         $userMonitoringEnabled = Configure::read('Security.user_monitoring_enabled');
         if ($userMonitoringEnabled) {
-            $redis = $this->User->setupRedis();
-            $userMonitoringEnabled = $redis && $redis->sismember('misp:monitored_users', $user['id']);
+            try {
+                $userMonitoringEnabled = RedisTool::init()->sismember('misp:monitored_users', $user['id']);
+            } catch (Exception $e) {
+                $userMonitoringEnabled = false;
+            }
         }
 
-        if (Configure::read('MISP.log_paranoid') || $userMonitoringEnabled) {
+        $shouldBeLogged = $userMonitoringEnabled ||
+            Configure::read('MISP.log_paranoid') ||
+            (Configure::read('MISP.log_paranoid_api') && isset($user['logged_by_authkey']) && $user['logged_by_authkey']);
+
+        if ($shouldBeLogged) {
+            $includeRequestBody = !empty(Configure::read('MISP.log_paranoid_include_post_body')) || $userMonitoringEnabled;
+            /** @var AccessLog $accessLog */
+            $accessLog = ClassRegistry::init('AccessLog');
+            $accessLog->logRequest($user, $this->_remoteIp(), $this->request, $includeRequestBody);
+        }
+
+        if (
+            empty(Configure::read('MISP.log_skip_access_logs_in_application_logs')) &&
+            $shouldBeLogged
+        ) {
             $change = 'HTTP method: ' . $_SERVER['REQUEST_METHOD'] . PHP_EOL . 'Target: ' . $this->request->here;
             if (
                 (
@@ -684,7 +742,7 @@ class AppController extends Controller
                 $payload = $this->request->input();
                 $change .= PHP_EOL . 'Request body: ' . $payload;
             }
-            $this->Log = ClassRegistry::init('Log');
+            $this->loadModel('Log');
             $this->Log->createLogEntry($user, 'request', 'User', $user['id'], 'Paranoid log entry', $change);
         }
     }
@@ -1041,7 +1099,7 @@ class AppController extends Controller
                 $headerNamespace = '';
             }
             if (isset($server[$headerNamespace . $header]) && !empty($server[$headerNamespace . $header])) {
-                if (Configure::read('Plugin.CustomAuth_only_allow_source') && Configure::read('Plugin.CustomAuth_only_allow_source') !== $server['REMOTE_ADDR']) {
+                if (Configure::read('Plugin.CustomAuth_only_allow_source') && Configure::read('Plugin.CustomAuth_only_allow_source') !== $this->_remoteIp()) {
                     $this->Log = ClassRegistry::init('Log');
                     $this->Log->create();
                     $log = array(
@@ -1050,7 +1108,7 @@ class AppController extends Controller
                             'model_id' => 0,
                             'email' => 'SYSTEM',
                             'action' => 'auth_fail',
-                            'title' => 'Failed authentication using external key (' . trim($server[$headerNamespace . $header]) . ') - the user has not arrived from the expected address. Instead the request came from: ' . $server['REMOTE_ADDR'],
+                            'title' => 'Failed authentication using external key (' . trim($server[$headerNamespace . $header]) . ') - the user has not arrived from the expected address. Instead the request came from: ' . $this->_remoteIp(),
                             'change' => null,
                     );
                     $this->Log->save($log);
@@ -1060,7 +1118,7 @@ class AppController extends Controller
                 $user['User'] = $temp;
                 if ($user['User']) {
                     $this->User->updateLoginTimes($user['User']);
-                    $this->Session->renew();
+                    //$this->Session->renew();
                     $this->Session->write(AuthComponent::$sessionKey, $user['User']);
                     if (Configure::read('MISP.log_auth')) {
                         $this->Log = ClassRegistry::init('Log');
@@ -1356,9 +1414,10 @@ class AppController extends Controller
     protected function _remoteIp()
     {
         $ipHeader = Configure::read('MISP.log_client_ip_header') ?: 'REMOTE_ADDR';
-        return isset($_SERVER[$ipHeader]) ? trim($_SERVER[$ipHeader]) : null;
+        return isset($_SERVER[$ipHeader]) ? trim($_SERVER[$ipHeader]) : $_SERVER['REMOTE_ADDR'];
     }
 
+
     /**
      * @param string $key
      * @return bool Returns true if the same log defined by $key was not stored in last hour
@@ -1452,7 +1511,7 @@ class AppController extends Controller
     protected function __setPagingParams(int $page, int $limit, int $current, string $type = 'named')
     {
         $this->request->params['paging'] = [
-            'Correlation' => [
+            $this->modelClass => [
                 'page' => $page,
                 'limit' => $limit,
                 'current' => $current,

app/Controller/AttributesController.php

@@ -73,6 +73,9 @@ class AttributesController extends AppController
     {
         $user = $this->Auth->user();
         $this->paginate['conditions']['AND'][] = $this->Attribute->buildConditions($user);
+
+        $this->__setIndexFilterConditions();
+
         $attributes = $this->paginate();
 
         if ($this->_isRest()) {
@@ -299,7 +302,7 @@ class AttributesController extends AppController
         $conditions['Attribute.type'] = array('attachment', 'malware-sample');
         $attributes = $this->Attribute->fetchAttributes($this->Auth->user(), array('conditions' => $conditions, 'flatten' => true));
         if (empty($attributes)) {
-            throw new UnauthorizedException(__('Attribute does not exists or you do not have the permission to download this attribute.'));
+            throw new UnauthorizedException(__('Attribute does not exist or you do not have the permission to download this attribute.'));
         }
         return $this->__downloadAttachment($attributes[0]['Attribute']);
     }
@@ -1536,6 +1539,7 @@ class AttributesController extends AppController
         $user = $this->Auth->user();
         $exception = null;
         $filters = $this->__getSearchFilters($exception);
+        $this->set('passedArgsArray', ['results' => $continue]);
         if ($this->request->is('post') || !empty($this->request->params['named']['tags'])) {
             if ($filters === false) {
                 return $exception;
@@ -1763,7 +1767,7 @@ class AttributesController extends AppController
         $conditions['Attribute.type'] = array('attachment', 'malware-sample');
         $attributes = $this->Attribute->fetchAttributes($user, array('conditions' => $conditions, 'flatten' => true));
         if (empty($attributes)) {
-            throw new UnauthorizedException(__('Attribute does not exists or you do not have the permission to download this attribute.'));
+            throw new UnauthorizedException(__('Attribute does not exist or you do not have the permission to download this attribute.'));
         }
         return $this->__downloadAttachment($attributes[0]['Attribute']);
     }
@@ -1961,7 +1965,7 @@ class AttributesController extends AppController
     public function fetchViewValue($id, $field = null)
     {
         $user = $this->_closeSession();
-        $validFields = ['value', 'comment', 'type', 'category', 'to_ids', 'distribution', 'timestamp', 'first_seen', 'last_seen'];
+        $validFields = ['value', 'comment', 'type', 'category', 'distribution', 'timestamp', 'first_seen', 'last_seen'];
         if (!isset($field) || !in_array($field, $validFields, true)) {
             throw new MethodNotAllowedException(__('Invalid field requested.'));
         }
@@ -1989,9 +1993,7 @@ class AttributesController extends AppController
         $attribute = $attribute[0];
         $result = $attribute['Attribute'][$field];
         if ($field === 'distribution') {
-            $result = $this->Attribute->shortDist[$result];
-        } elseif ($field === 'to_ids') {
-            $result = $result == 0 ? 'No' : 'Yes';
+            $this->set('shortDist', $this->Attribute->shortDist);
         } elseif ($field === 'value') {
             $this->loadModel('Warninglist');
             $attribute['Attribute'] = $this->Warninglist->checkForWarning($attribute['Attribute']);
@@ -2006,23 +2008,27 @@ class AttributesController extends AppController
 
     public function fetchEditForm($id, $field = null)
     {
-        $validFields = array('value', 'comment', 'type', 'category', 'to_ids', 'distribution', 'first_seen', 'last_seen');
-        if (!isset($field) || !in_array($field, $validFields)) {
-            throw new MethodNotAllowedException(__('Invalid field requested.'));
-        }
         if (!$this->request->is('ajax')) {
             throw new MethodNotAllowedException(__('This function can only be accessed via AJAX.'));
         }
-        $fields = array('id', 'distribution', 'event_id');
-        if ($field == 'category' || $field == 'type') {
-            $fields[] = 'type';
-            $fields[] = 'category';
+
+        $validFields = array('value', 'comment', 'type', 'category', 'to_ids', 'distribution', 'first_seen', 'last_seen');
+        if (!isset($field) || !in_array($field, $validFields, true)) {
+            throw new NotFoundException(__('Invalid field requested.'));
+        }
+        $fieldsToFetch = array('id', 'event_id');
+        if ($field === 'category' || $field === 'type') {
+            $fieldsToFetch[] = 'type';
+            $fieldsToFetch[] = 'category';
+            if ($field === 'type') {
+                $fieldsToFetch[] = 'value';
+            }
         } else {
-            $fields[] = $field;
+            $fieldsToFetch[] = $field;
         }
         $params = array(
             'conditions' => array('Attribute.id' => $id),
-            'fields' => $fields,
+            'fields' => $fieldsToFetch,
             'flatten' => 1,
             'contain' => array(
                 'Event' => array(
@@ -2044,15 +2050,28 @@ class AttributesController extends AppController
             unset($distributionLevels[4]);
             $this->set('distributionLevels', $distributionLevels);
         } elseif ($field === 'category') {
-            $typeCategory = array();
+            $possibleCategories = [];
             foreach ($this->Attribute->categoryDefinitions as $k => $category) {
-                foreach ($category['types'] as $type) {
-                    $typeCategory[$type][] = $k;
+                if (in_array($attribute['Attribute']['type'], $category['types'], true)) {
+                    $possibleCategories[] = $k;
                 }
             }
-            $this->set('typeCategory', $typeCategory);
+            $this->set('possibleCategories', $possibleCategories);
         } elseif ($field === 'type') {
-            $this->set('categoryDefinitions', $this->Attribute->categoryDefinitions);
+            $possibleTypes = $this->Attribute->categoryDefinitions[$attribute['Attribute']['category']]['types'];
+            $validTypes = AttributeValidationTool::validTypesForValue($possibleTypes, $this->Attribute->getCompositeTypes(), $attribute['Attribute']['value']);
+            $options = [];
+            foreach ($possibleTypes as $possibleType) {
+                if ($this->Attribute->typeIsAttachment($possibleType)) {
+                    continue; // skip attachment types
+                }
+                $options[] = [
+                    'name' => $possibleType,
+                    'value' => $possibleType,
+                    'disabled' => !in_array($possibleType, $validTypes, true),
+                ];
+            }
+            $this->set('options', $options);
         }
         $this->set('object', $attribute['Attribute']);
         $fieldURL = ucfirst($field);
@@ -2798,10 +2817,7 @@ class AttributesController extends AppController
                 'fields' => ['Attribute.deleted', 'Attribute.event_id', 'Attribute.id', 'Attribute.object_id', 'Event.orgc_id', 'Event.user_id'],
                 'contain' => ['Event'],
             ]);
-            if (empty($attribute)) {
-                throw new NotFoundException(__('Invalid attribute'));
-            }
-            if ($attribute['Attribute']['deleted']) {
+            if (empty($attribute) || $attribute['Attribute']['deleted']) {
                 throw new NotFoundException(__('Invalid attribute'));
             }
             if (empty($tag_id)) {
@@ -2831,19 +2847,19 @@ class AttributesController extends AppController
             if (!$this->__canModifyTag($attribute, !empty($attributeTag['AttributeTag']['local']))) {
                 return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'You do not have permission to do that.')), 'status' => 200, 'type' => 'json'));
             }
-
             if (empty($attributeTag)) {
                 return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Invalid attribute - tag combination.')), 'status' => 200, 'type' => 'json'));
             }
-            $tag = $this->Attribute->AttributeTag->Tag->find('first', array(
-                'conditions' => array('Tag.id' => $tag_id),
-                'recursive' => -1,
-                'fields' => array('Tag.name')
-            ));
             if ($this->Attribute->AttributeTag->delete($attributeTag['AttributeTag']['id'])) {
                 if (empty($attributeTag['AttributeTag']['local'])) {
                     $this->Attribute->touch($attribute);
                 }
+
+                $tag = $this->Attribute->AttributeTag->Tag->find('first', array(
+                    'conditions' => array('Tag.id' => $tag_id),
+                    'recursive' => -1,
+                    'fields' => array('Tag.name')
+                ));
                 $log = ClassRegistry::init('Log');
                 $log->createLogEntry($this->Auth->user(), 'tag', 'Attribute', $id, 'Removed tag (' . $tag_id . ') "' . $tag['Tag']['name'] . '" from attribute (' . $id . ')', 'Attribute (' . $id . ') untagged of Tag (' . $tag_id . ')');
                 return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => 'Tag removed.', 'check_publish' => empty($attributeTag['AttributeTag']['local']))), 'status' => 200, 'type'=> 'json'));
@@ -3005,4 +3021,18 @@ class AttributesController extends AppController
         $sg = $this->Attribute->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', true, $sharingGroupId);
         return !empty($sg);
     }
+
+    private function __setIndexFilterConditions()
+    {
+        // search by attribute value
+        if (isset($this->request->params['named']['searchvalue'])) {
+            $v = $this->request->params['named']['searchvalue'];
+            $this->paginate['conditions']['AND'][] = [
+                'OR' => [
+                    ['Attribute.value1' => $v],
+                    ['Attribute.value2' => $v],
+                ]
+            ];
+        }
+    }
 }

app/Controller/AuditLogsController.php

@@ -100,7 +100,9 @@ class AuditLogsController extends AppController
         if ($this->_isRest()) {
             $this->paginate['fields'][] = 'request_id';
         }
-
+        if (!Configure::read('MISP.log_new_audit')) {
+            $this->Flash->warning(__("Audit log is not enabled. See 'MISP.log_new_audit' in the Server Settings. (Administration -> Server Settings -> MISP tab)"));
+        }
         $params = $this->IndexFilter->harvestParameters([
             'ip',
             'user',
@@ -190,10 +192,13 @@ class AuditLogsController extends AppController
             $list[$k]['AuditLog']['action_human'] = $this->actions[$item['AuditLog']['action']];
         }
 
-        $this->set('list', $list);
+        $this->set('data', $list);
         $this->set('event', $event);
         $this->set('mayModify', $this->__canModifyEvent($event));
-        $this->set('title_for_layout', __('Audit logs for event #%s', $event['Event']['id']));
+        $this->set('menuData', [
+            'menuList' => 'event',
+            'menuItem' => 'eventLog'
+        ]);
     }
 
     public function fullChange($id)

app/Controller/AuthKeysController.php

@@ -18,41 +18,13 @@ class AuthKeysController extends AppController
         )
     );
 
-    public function index($id = false)
+    public function index($user_id = false)
     {
         $conditions = $this->__prepareConditions();
-        $canCreateAuthkey = true;
-        if ($id) {
-            $this->set('user_id', $id);
-            if ($this->_isAdmin()) {
-                if ($this->_isSiteAdmin()) {
-                    $canCreateAuthkey = true;
-                } else {
-                    $user = $this->AuthKey->User->find('first', [
-                        'recursive' => -1,
-                        'conditions' => [
-                            'User.id' => $id,
-                            'User.disabled' => false
-                        ],
-                        'fields' => ['User.id', 'User.org_id', 'User.disabled'],
-                        'contain' => [
-                            'Role' => [
-                                'fields' => [
-                                    'Role.perm_site_admin', 'Role.perm_admin'
-                                ]
-                            ]
-                        ]
-                    ]);
-                    if ($user['Role']['perm_site_admin'] || ($user['Role']['perm_admin'] && $user['User']['id'] !== $this->Auth->user('id'))) {
-                        $canCreateAuthkey = false;
-                    } else {
-                        $canCreateAuthkey = true;
-                    }
-                }
-            } else {
-                $canCreateAuthkey = (int)$id === (int)$this->Auth->user('id');
-            }
-            $conditions['AND'][] = ['AuthKey.user_id' => $id];
+        $canCreateAuthkey = $this->__canCreateAuthKeyForUser($user_id);
+        if ($user_id) {
+            $this->set('user_id', $user_id);
+            $conditions['AND'][] = ['AuthKey.user_id' => $user_id];
         }
         $this->set('canCreateAuthkey', $canCreateAuthkey);
         $keyUsageEnabled = Configure::read('MISP.log_user_ips') && Configure::read('MISP.log_user_ips_authkeys');
@@ -90,6 +62,9 @@ class AuthKeysController extends AppController
 
     public function delete($id)
     {
+        if(!$this->__canEditAuthKey($id)) {
+            throw new MethodNotAllowedException(__('Invalid user or insufficient privileges to interact with an authkey for the given user.'));
+        }
         $this->CRUD->delete($id, [
             'conditions' => $this->__prepareConditions(),
             'contain' => ['User'],
@@ -101,6 +76,9 @@ class AuthKeysController extends AppController
 
     public function edit($id)
     {
+        if(!$this->__canEditAuthKey($id)) {
+            throw new MethodNotAllowedException(__('Invalid user or insufficient privileges to interact with an authkey for the given user.'));
+        }
         $this->CRUD->edit($id, [
             'conditions' => $this->__prepareConditions(),
             'afterFind' => function (array $authKey) {
@@ -152,44 +130,16 @@ class AuthKeysController extends AppController
             $user_id = $this->Auth->user('id');
         }
         $selectConditions = [];
-        if (!$this->_isSiteAdmin()) {
-            if ($this->_isAdmin()) {
-                $role_ids = $this->AuthKey->User->Role->find('column', [
-                    'fields' => ['Role.id'],
-                    'conditions' => [
-                        'AND' => [
-                            'Role.perm_site_admin' => false,
-                            'Role.perm_auth' => true,
-                            'Role.perm_admin' => false
-                        ]
-                    ]
-                ]);
-                $user_ids = $this->AuthKey->User->find('column', [
-                    'fields' => ['User.id'],
-                    'conditions' => [
-                        'User.org_id' => $this->Auth->user('org_id'),
-                        'OR' => [
-                            'User.role_id' => $role_ids,
-                            'User.id' => $this->Auth->user('id')
-                        ]
-                    ]
-                ]);
-                if (!empty($user_id)) {
-                    if (in_array($user_id, $user_ids)) {
-                        $user_ids = [$user_id];
-                    } else {
-                        throw new MethodNotAllowedException(__('Invalid user or insufficient privileges to create an authkey for the given user.'));
-                    }
-                }
-                $selectConditions['AND'][] = ['User.id' => $user_ids];
-                $params['override']['user_id'] = $user_ids[0];
+        if ($user_id) {
+            if ($this->__canCreateAuthKeyForUser($user_id)) {
+                $selectConditions['AND'][] = ['User.id' => $user_id];
+                $params['override']['user_id'] = $user_id;
             } else {
-                $selectConditions['AND'][] = ['User.id' => $this->Auth->user('id')];
-                $params['override']['user_id'] = $this->Auth->user('id');
+                throw new MethodNotAllowedException(__('Invalid user or insufficient privileges to interact with an authkey for the given user.'));
             }
-        } else if ($user_id) {
-            $selectConditions['AND'][] = ['User.id' => $user_id];
-            $params['override']['user_id'] = $user_id;
+        } else {
+            $selectConditions['AND'][] = ['User.id' => $this->Auth->user('id')];
+            $params['override']['user_id'] = $this->Auth->user('id');        
         }
         $this->CRUD->add($params);
         if ($this->IndexFilter->isRest()) {
@@ -238,6 +188,33 @@ class AuthKeysController extends AppController
         ]);
     }
 
+    public function pin($id, $ip) {
+        if(!$this->__canEditAuthKey($id)) {
+            throw new MethodNotAllowedException(__('Invalid user or insufficient privileges to interact with an authkey for the given user.'));
+        }
+        if ($this->request->is('post')) {
+            // find entry, to confirm user is authorized
+            $conditions = $this->__prepareConditions();
+            $conditions['AND'][]['AuthKey.id'] = $id;
+            $authKey = $this->AuthKey->find(
+                'first',
+                ['conditions' => $conditions,
+                'recursive'=> 1
+                ]
+            );
+            // update the key with the source IP
+            if ($authKey) {
+                $authKey['AuthKey']['allowed_ips'] = $ip;
+                $this->AuthKey->save($authKey, ['fieldList' => ['allowed_ips']]);
+                $this->Flash->success(__('IP address set as allowed source for the Key.'));
+            } else {
+                $this->Flash->error(__('Failed to set IP as source'));
+            }
+        }
+        $this->redirect($this->referer());
+        // $this->redirect(['controller' => 'auth_keys', 'view' => 'index']);
+    }
+
     /**
      * Return conditions according to current user permission.
      * @return array
@@ -246,7 +223,7 @@ class AuthKeysController extends AppController
     {
         $user = $this->Auth->user();
         if ($user['Role']['perm_site_admin']) {
-            $conditions = []; // site admin can see all keys
+            $conditions = []; // site admin can see/edit all keys
         } else if ($user['Role']['perm_admin']) {
             $conditions['AND'][]['User.org_id'] = $user['org_id']; // org admin can see his/her user org auth keys
         } else {
@@ -254,4 +231,54 @@ class AuthKeysController extends AppController
         }
         return $conditions;
     }
+
+    private function __canCreateAuthKeyForUser($user_id)
+    {
+        if (!$user_id) 
+            return true;
+        if ($this->_isAdmin()) {
+            if ($this->_isSiteAdmin()) {
+                return true;   // site admin is OK for all
+            } else {
+                // org admin only for non-admin users and themselves
+                $user = $this->AuthKey->User->find('first', [
+                    'recursive' => -1,
+                    'conditions' => [
+                        'User.id' => $user_id,
+                        'User.disabled' => false
+                    ],
+                    'fields' => ['User.id', 'User.org_id', 'User.disabled'],
+                    'contain' => [
+                        'Role' => [
+                            'fields' => [
+                                'Role.perm_site_admin', 'Role.perm_admin', 'Role.perm_auth'
+                            ]
+                        ]
+                    ]
+                ]);
+                if ($user['Role']['perm_site_admin'] || 
+                    ($user['Role']['perm_admin'] && $user['User']['id'] !== $this->Auth->user('id')) ||
+                    !$user['Role']['perm_auth']) {
+                    // no create/edit for site_admin or other org admin
+                    return false;
+                } else {
+                    // ok for themselves or users
+                    return true;
+                }
+            }
+        } else {
+            // user for themselves
+            return (int)$user_id === (int)$this->Auth->user('id');
+        }
+    }
+
+    private function __canEditAuthKey($key_id)
+    {
+        $user_id = $this->AuthKey->find('column', [
+            'fields' => ['AuthKey.user_id'],
+            'conditions' => [
+                'AuthKey.id' => $key_id
+            ]]);
+        return $this->__canCreateAuthKeyForUser($user_id);
+    }
 }

app/Controller/CommunitiesController.php

@@ -1,15 +1,13 @@
 <?php
 App::uses('AppController', 'Controller');
 
+/**
+ * @property Community $Community
+ */
 class CommunitiesController extends AppController
 {
     public $components = array('Session', 'RequestHandler');
 
-    public function beforeFilter()
-    {
-        parent::beforeFilter();
-    }
-
     public $paginate = array(
             'limit' => 60,
             'maxLimit' => 9999

app/Controller/Component/ACLComponent.php

@@ -72,8 +72,9 @@ class ACLComponent extends Component
                 'add' => ['AND' => ['perm_auth', 'not_read_only_authkey']],
                 'delete' => ['AND' => ['perm_auth', 'not_read_only_authkey']],
                 'edit' => ['AND' => ['perm_auth', 'not_read_only_authkey']],
+                'pin' => ['AND' => ['perm_auth', 'not_read_only_authkey']],
                 'index' => ['perm_auth'],
-                'view' => ['perm_auth']
+                'view' => ['perm_auth'],
             ],
             'cerebrates' => [
                 'add' => [],
@@ -125,7 +126,7 @@ class ACLComponent extends Component
             'decayingModel' => array(
                 "update" => array(),
                 "export" => array('*'),
-                "import" => array('*'),
+                "import" => array('OR' => array('perm_admin', 'perm_decaying')),
                 "view" => array('*'),
                 "index" => array('*'),
                 "add" => array( 'OR' => array('perm_admin', 'perm_decaying')),
@@ -384,23 +385,30 @@ class ACLComponent extends Component
                     'event_index' => array('*'),
                     'returnDates' => array('*'),
                     'testForStolenAttributes' => array(),
-                    'pruneUpdateLogs' => array()
+                    'pruneUpdateLogs' => array(),
+                    'index' => array('perm_audit')
             ),
-      'auditLogs' => [
-          'admin_index' => ['perm_audit'],
-          'fullChange' => ['perm_audit'],
-          'eventIndex' => ['*'],
-          'returnDates' => ['*'],
-      ],
-      'modules' => array(
-        'index' => array('perm_auth'),
-        'queryEnrichment' => array('perm_auth'),
-      ),
+        'auditLogs' => [
+            'admin_index' => ['perm_audit'],
+            'fullChange' => ['perm_audit'],
+            'eventIndex' => ['*'],
+            'returnDates' => ['*'],
+        ],
+        'accessLogs' => [
+            'admin_index' => [],
+            'admin_request' => [],
+            'admin_queryLog' => [],
+        ],
+        'modules' => array(
+            'index' => array('perm_auth'),
+            'queryEnrichment' => array('perm_auth'),
+        ),
             'news' => array(
-                    'add' => array(),
-                    'edit' => array(),
-                    'delete' => array(),
-                    'index' => array('*'),
+                'add' => array(),
+                'edit' => array(),
+                'delete' => array(),
+                'admin_index' => array(),
+                'index' => ['*'],
             ),
             'noticelists' => array(
                     'delete' => array(),
@@ -429,6 +437,7 @@ class ACLComponent extends Component
                     'groupAttributesIntoObject' => array('perm_add'),
                     'revise_object' => array('perm_add'),
                     'view' => array('*'),
+                'createFromFreetext' => ['perm_add'],
             ),
             'objectReferences' => array(
                 'add' => array('perm_add'),
@@ -446,9 +455,9 @@ class ACLComponent extends Component
                 'objectChoice' => array('*'),
                 'objectMetaChoice' => array('perm_add'),
                 'view' => array('*'),
-                'viewElements' => array('*'),
                 'index' => array('*'),
-                'update' => array()
+                'update' => array(),
+                'possibleObjectTemplates' => ['*'],
             ),
             'objectTemplateElements' => array(
                 'viewElements' => array('*')
@@ -475,9 +484,9 @@ class ACLComponent extends Component
                 'display' => array('*'),
             ),
             'posts' => array(
-                'add' => array('not_read_only_authkey'),
-                'delete' => array('not_read_only_authkey'),
-                'edit' => array('not_read_only_authkey'),
+                'add' => ['AND' => ['not_read_only_authkey', 'discussion_enabled']],
+                'delete' => ['AND' => ['not_read_only_authkey', 'discussion_enabled']],
+                'edit' => ['AND' => ['not_read_only_authkey', 'discussion_enabled']],
                 'pushMessageToZMQ' => array()
             ),
             'regexp' => array(
@@ -604,7 +613,7 @@ class ACLComponent extends Component
             ),
             'sightings' => array(
                 'add' => array('perm_sighting'),
-                'restSearch' => array('perm_sighting'),
+                'restSearch' => array('*'),
                 'advanced' => array('perm_sighting'),
                 'delete' => ['AND' => ['perm_sighting', 'perm_modify_org']],
                 'index' => array('*'),
@@ -669,6 +678,7 @@ class ACLComponent extends Component
                 'taxonomyMassHide' => array('perm_tagger'),
                 'taxonomyMassUnhide' => array('perm_tagger'),
                 'toggleRequired' => array(),
+                'toggleHighlighted' => array(),
                 'update' => array(),
                 'import' => [],
                 'export' => ['*'],
@@ -677,6 +687,19 @@ class ACLComponent extends Component
                 'hideTag' => array('perm_tagger'),
                 'normalizeCustomTagsToTaxonomyFormat' => [],
             ),
+            'taxiiServers' => [
+                'add' => ['perm_site_admin'],
+                'edit' => ['perm_site_admin'],
+                'collectionsIndex' => ['perm_site_admin'],
+                'index' => ['perm_site_admin'],
+                'objectsIndex' => ['perm_site_admin'],
+                'objectView' => ['perm_site_admin'],
+                'delete' => ['perm_site_admin'],
+                'view' => ['perm_site_admin'],
+                'push' => ['perm_site_admin'],
+                'getRoot' => ['perm_site_admin'],
+                'getCollections' => ['perm_site_admin']
+            ],
             'templateElements' => array(
                 'add' => array('perm_template'),
                 'delete' => array('perm_template'),
@@ -698,14 +721,15 @@ class ACLComponent extends Component
                 'view' => array('*'),
             ),
             'threads' => array(
-                'index' => array('*'),
-                'view' => array('*'),
-                'viewEvent' => array('*'),
+                'index' => array('discussion_enabled'),
+                'view' => array('discussion_enabled'),
+                'viewEvent' => array('discussion_enabled'),
             ),
             'users' => array(
                 'acceptRegistrations' => array(),
                 'admin_add' => ['AND' => ['perm_admin', 'add_user_enabled']],
                 'admin_delete' => array('perm_admin'),
+                'admin_destroy' => array(),
                 'admin_edit' => array('perm_admin'),
                 'admin_email' => array('perm_admin'),
                 'admin_filterUserIndex' => array('perm_admin'),
@@ -724,13 +748,20 @@ class ACLComponent extends Component
                 'downloadTerms' => array('*'),
                 'edit' => array('self_management_enabled'),
                 'email_otp' => array('*'),
+                'forgot' => array('*'),
+                'otp' => array('*'),
+                'hotp' => array('*'),
+                'totp_new' => array('*'),
+                'totp_delete' => array('perm_admin'),
                 'searchGpgKey' => array('*'),
                 'fetchGpgKey' => array('*'),
                 'histogram' => array('*'),
                 'initiatePasswordReset' => ['AND' => ['perm_admin', 'password_change_enabled']],
                 'login' => array('*'),
                 'logout' => array('*'),
+                'logout401' => array('*'),
                 'notificationSettings' => ['*'],
+                'password_reset' => array('*'),
                 'register' => array('*'),
                 'registrations' => array(),
                 'resetAllSyncAuthKeys' => array(),
@@ -759,7 +790,7 @@ class ACLComponent extends Component
                 'eventIndexColumnToggle' => ['*'],
             ),
             'warninglists' => array(
-                'checkValue' => array('perm_auth'),
+                'checkValue' => ['*'],
                 'delete' => ['perm_warninglist'],
                 'enableWarninglist' => ['perm_warninglist'],
                 'getToggleField' => ['perm_warninglist'],
@@ -849,6 +880,9 @@ class ACLComponent extends Component
         $this->dynamicChecks['delegation_enabled'] = function (array $user) {
             return (bool)Configure::read('MISP.delegation');
         };
+        $this->dynamicChecks['discussion_enabled'] = function (array $user) {
+            return !Configure::read('MISP.discussion_disable');
+        };
         // Returns true if current user is not using advanced auth key or if authkey is not read only
         $this->dynamicChecks['not_read_only_authkey'] = function (array $user) {
             return !isset($user['authkey_read_only']) || !$user['authkey_read_only'];
@@ -998,7 +1032,7 @@ class ACLComponent extends Component
      */
     public function canEditEventReport(array $user, array $eventReport)
     {
-        if (!isset($report['Event'])) {
+        if (!isset($eventReport['Event'])) {
             throw new InvalidArgumentException('Passed object does not contain an Event.');
         }
         if ($user['Role']['perm_site_admin']) {

app/Controller/Component/Auth/ApacheAuthenticate.php

@@ -38,10 +38,10 @@ class ApacheAuthenticate extends BaseAuthenticate
         }
         return $returnCode;
     }
-	
+
     private function getEmailAddress($ldapEmailField, $ldapUserData)
     {
-	// return the email address of an LDAP user if one of the fields in $ldapEmaiLField exists
+        // return the email address of an LDAP user if one of the fields in $ldapEmaiLField exists
         foreach($ldapEmailField as $field) {
             if (isset($ldapUserData[0][$field][0])) {
                 return $ldapUserData[0][$field][0];
@@ -73,6 +73,14 @@ class ApacheAuthenticate extends BaseAuthenticate
         ldap_set_option($ldapconn, LDAP_OPT_PROTOCOL_VERSION, Configure::read('ApacheSecureAuth.ldapProtocol'));
         ldap_set_option($ldapconn, LDAP_OPT_REFERRALS, Configure::read('ApacheSecureAuth.ldapAllowReferrals', true));
 
+        if (Configure::read('ApacheSecureAuth.starttls', false) == true) {
+            # Default is false, sine STARTTLS support is a new feature
+            # Ignored on ldaps://, but can trigger problems for orgs
+            # using unencrypted LDAP. Loose comparison allows users to
+            # use # true / 1 / etc.
+            ldap_start_tls($ldapconn);
+        }
+
         if ($ldapconn) {
             // LDAP bind
             $ldapbind = ldap_bind($ldapconn, $ldaprdn, $ldappass);
@@ -105,7 +113,6 @@ class ApacheAuthenticate extends BaseAuthenticate
             } else {
                 die("User not found in LDAP");
             }
-		
             // close LDAP connection
             ldap_close($ldapconn);
         }

app/Controller/Component/BetterSecurityComponent.php

@@ -13,6 +13,15 @@ class BetterSecurityComponent extends SecurityComponent
      */
     public $doNotGenerateToken = false;
 
+    public function blackHole(Controller $controller, $error = '', SecurityException $exception = null)
+    {
+        $action = $controller->request->params['action'];
+        $unlockedActions = JsonTool::encode($this->unlockedActions);
+        $isRest = $controller->IndexFilter->isRest() ? '1' : '0';
+        $this->log("Blackhole exception when accessing $controller->here (isRest: $isRest, action: $action, unlockedActions: $unlockedActions): {$exception->getMessage()}"); // log blackhole exception
+        return parent::blackHole($controller, $error, $exception);
+    }
+
     public function generateToken(CakeRequest $request)
     {
         if (isset($request->params['requested']) && $request->params['requested'] === 1) {

app/Controller/Component/CRUDComponent.php

@@ -114,7 +114,7 @@ class CRUDComponent extends Component
                     $this->Controller->Flash->success($message);
                     if (!empty($params['displayOnSuccess'])) {
                         $this->Controller->set('entity', $data);
-                        $this->Controller->set('referer', $this->Controller->referer());
+                        $this->Controller->set('referer', $this->Controller->referer(['action' => 'view', $model->id], true));
                         $this->Controller->render($params['displayOnSuccess']);
                         return;
                     }

app/Controller/Component/CompressedRequestHandlerComponent.php

@@ -1,19 +1,16 @@
 <?php
 class CompressedRequestHandlerComponent extends Component
 {
-    // Maximum size of uncompressed data to prevent zip bombs
-    const MAX_SIZE = 1024 * 1024 * 100;
-
     public function startup(Controller $controller)
     {
-        $contentEncoding = CakeRequest::header('CONTENT_ENCODING');
+        $contentEncoding = $_SERVER['HTTP_CONTENT_ENCODING'] ?? null;
         if (!empty($contentEncoding)) {
             if ($contentEncoding === 'br') {
                 $controller->request->setInput($this->decodeBrotliEncodedContent($controller));
             } else if ($contentEncoding === 'gzip') {
                 $controller->request->setInput($this->decodeGzipEncodedContent($controller));
             } else {
-                throw new MethodNotAllowedException("Unsupported content encoding '$contentEncoding'.");
+                throw new BadRequestException("Unsupported content encoding '$contentEncoding'.");
             }
         }
     }
@@ -24,10 +21,10 @@ class CompressedRequestHandlerComponent extends Component
     public function supportedEncodings()
     {
         $supportedEncodings = [];
-        if (function_exists('gzdecode') || function_exists('inflate_init')) {
+        if (function_exists('gzdecode')) {
             $supportedEncodings[] = 'gzip';
         }
-        if (function_exists('brotli_uncompress') || function_exists('brotli_uncompress_init')) {
+        if (function_exists('brotli_uncompress')) {
             $supportedEncodings[] = 'br';
         }
         return $supportedEncodings;
@@ -36,44 +33,17 @@ class CompressedRequestHandlerComponent extends Component
     /**
      * @return string
      * @throws Exception
-     * @see CakeRequest::_readInput()
      */
     private function decodeGzipEncodedContent(Controller $controller)
     {
-        if (function_exists('inflate_init')) {
-            // Decompress data on the fly if supported
-            $resource = inflate_init(ZLIB_ENCODING_GZIP);
-            if ($resource === false) {
-                throw new Exception('GZIP incremental uncompress init failed.');
-            }
-            $uncompressed = '';
-            foreach ($this->streamInput() as $data) {
-                $uncompressedChunk = inflate_add($resource, $data);
-                if ($uncompressedChunk === false) {
-                    throw new MethodNotAllowedException('Invalid compressed data.');
-                }
-                $uncompressed .= $uncompressedChunk;
-                if (strlen($uncompressed) > self::MAX_SIZE) {
-                    throw new Exception("Uncompressed data are bigger than is limit.");
-                }
-            }
-            $uncompressedChunk = inflate_add($resource, '', ZLIB_FINISH);
-            if ($uncompressedChunk === false) {
-                throw new MethodNotAllowedException('Invalid compressed data.');
-            }
-            return $uncompressed . $uncompressedChunk;
-
-        } else if (function_exists('gzdecode')) {
-            $decoded = gzdecode($controller->request->input(), self::MAX_SIZE);
+        if (function_exists('gzdecode')) {
+            $decoded = gzdecode($controller->request->input());
             if ($decoded === false) {
-                throw new MethodNotAllowedException('Invalid compressed data.');
-            }
-            if (strlen($decoded) >= self::MAX_SIZE) {
-                throw new Exception("Uncompressed data are bigger than is limit.");
+                throw new BadRequestException('Invalid compressed data.');
             }
             return $decoded;
         } else {
-            throw new MethodNotAllowedException("This server doesn't support GZIP compressed requests.");
+            throw new BadRequestException("This server doesn't support GZIP compressed requests.");
         }
     }
 
@@ -81,65 +51,17 @@ class CompressedRequestHandlerComponent extends Component
      * @param Controller $controller
      * @return string
      * @throws Exception
-     * @see CakeRequest::_readInput()
      */
     private function decodeBrotliEncodedContent(Controller $controller)
     {
-        if (function_exists('brotli_uncompress_init')) {
-            // Decompress data on the fly if supported
-            $resource = brotli_uncompress_init();
-            if ($resource === false) {
-                throw new Exception('Brotli incremental uncompress init failed.');
-            }
-            $uncompressed = '';
-            foreach ($this->streamInput() as $data) {
-                $uncompressedChunk = brotli_uncompress_add($resource, $data, BROTLI_PROCESS);
-                if ($uncompressedChunk === false) {
-                    throw new MethodNotAllowedException('Invalid compressed data.');
-                }
-                $uncompressed .= $uncompressedChunk;
-                if (strlen($uncompressed) > self::MAX_SIZE) {
-                    throw new Exception("Uncompressed data are bigger than is limit.");
-                }
-            }
-            $uncompressedChunk = brotli_uncompress_add($resource, '', BROTLI_FINISH);
-            if ($uncompressedChunk === false) {
-                throw new MethodNotAllowedException('Invalid compressed data.');
-            }
-            return $uncompressed . $uncompressedChunk;
-
-        } else if (function_exists('brotli_uncompress')) {
-            $decoded = brotli_uncompress($controller->request->input(), self::MAX_SIZE);
+        if (function_exists('brotli_uncompress')) {
+            $decoded = brotli_uncompress($controller->request->input());
             if ($decoded === false) {
-                throw new MethodNotAllowedException('Invalid compressed data.');
-            }
-            if (strlen($decoded) >= self::MAX_SIZE) {
-                throw new Exception("Uncompressed data are bigger than is limit.");
+                throw new BadRequestException('Invalid compressed data.');
             }
             return $decoded;
         } else {
-            throw new MethodNotAllowedException("This server doesn't support brotli compressed requests.");
+            throw new BadRequestException("This server doesn't support brotli compressed requests.");
         }
     }
-
-    /**
-     * @param int $chunkSize
-     * @return Generator<string>
-     * @throws Exception
-     */
-    private function streamInput($chunkSize = 8192)
-    {
-        $fh = fopen('php://input', 'rb');
-        if ($fh === false) {
-            throw new Exception("Could not open PHP input for reading.");
-        }
-        while (!feof($fh)) {
-            $data = fread($fh, $chunkSize);
-            if ($data === false) {
-                throw new Exception("Could not read PHP input.");
-            }
-            yield $data;
-        }
-        fclose($fh);
-    }
 }

app/Controller/Component/IOCImportComponent.php

@@ -190,7 +190,7 @@ class IOCImportComponent extends Component
 
         $duplicateFilter = array();
         // check if we have any attributes, if yes, add their UUIDs to our list of success-array
-        if (count($event['Attribute']) > 0) {
+        if (isset($event['Attribute']) && count($event['Attribute']) > 0) {
             foreach ($event['Attribute'] as $k => $attribute) {
                 $condensed = strtolower($attribute['value']) . $attribute['category'] . $attribute['type'];
                 if (!in_array($condensed, $duplicateFilter)) {

app/Controller/Component/IndexFilterComponent.php

@@ -44,15 +44,21 @@ class IndexFilterComponent extends Component
                 }
             }
         }
+
+        $data = $this->__massageData($data, $request, $paramArray);
+
+        $this->Controller->set('passedArgs', json_encode($this->Controller->passedArgs));
+        return $data;
+    }
+
+    private function __massageData($data, $request, $paramArray)
+    {
+        $data = array_filter($data, function($paramName) use ($paramArray) {
+            return !empty($paramArray[$paramName]);
+        }, ARRAY_FILTER_USE_KEY);
+
         if (!empty($paramArray)) {
             foreach ($paramArray as $p) {
-                if (
-                    isset($options['ordered_url_params'][$p]) &&
-                    (!in_array(strtolower((string)$options['ordered_url_params'][$p]), array('null', '0', false, 'false', null)))
-                ) {
-                    $data[$p] = $options['ordered_url_params'][$p];
-                    $data[$p] = str_replace(';', ':', $data[$p]);
-                }
                 if (isset($request->params['named'][$p])) {
                     $data[$p] = str_replace(';', ':', $request->params['named'][$p]);
                 }
@@ -67,28 +73,8 @@ class IndexFilterComponent extends Component
             }
         }
         unset($v);
-        if (!empty($options['additional_delimiters'])) {
-            if (!is_array($options['additional_delimiters'])) {
-                $options['additional_delimiters'] = array($options['additional_delimiters']);
-            }
-            foreach ($data as $k => $v) {
-                $found = false;
-                foreach ($options['additional_delimiters'] as $delim) {
-                    if (strpos($v, $delim) !== false) {
-                        $found = true;
-                        break;
-                    }
-                }
-                if ($found) {
-                    $data[$k] = explode($options['additional_delimiters'][0], str_replace($options['additional_delimiters'], $options['additional_delimiters'][0], $v));
-                    foreach ($data[$k] as $k2 => $value) {
-                        $data[$k][$k2] = trim($data[$k][$k2]);
-                    }
-                }
-            }
-        }
-        $this->Controller->set('passedArgs', json_encode($this->Controller->passedArgs));
         return $data;
+
     }
 
     public function isRest()

app/Controller/Component/RestResponseComponent.php

@@ -608,37 +608,47 @@ class RestResponseComponent extends Component
             $type = 'csv';
         } else {
             $type = $format;
-            $dumpSql = !empty($this->Controller->sql_dump) && Configure::read('debug') > 1;
+
+            $dumpSql = intval($this->Controller->request->params['named']['sql'] ?? 0);
+            if ($dumpSql && Configure::read('debug') < 2) {
+                $dumpSql = 0; // disable dumping SQL if debugging is off
+            }
+
             if (!$raw) {
                 if (is_string($response)) {
                     $response = array('message' => $response);
                 }
                 if ($dumpSql) {
-                    $this->Log = ClassRegistry::init('Log');
-                    if ($this->Controller->sql_dump === 2) {
-                        $response = array('sql_dump' => $this->Log->getDataSource()->getLog(false, false));
+                    if ($dumpSql === 2) {
+                        $response = ['sql_dump' => $this->getSqlLog()];
                     } else {
-                        $response['sql_dump'] = $this->Log->getDataSource()->getLog(false, false);
+                        $response['sql_dump'] = $this->getSqlLog();
                     }
                 }
 
-                $flags = JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE;
-                if (!$this->isAutomaticTool()) {
-                    $flags |= JSON_PRETTY_PRINT; // Do not pretty print response for automatic tools
+                // If response is big array, encode items separately to save memory
+                if (is_array($response) && count($response) > 10000) {
+                    $output = new TmpFileTool();
+                    $output->write('[');
+
+                    foreach ($response as $item) {
+                        $output->writeWithSeparator(JsonTool::encode($item), ',');
+                    }
+
+                    $output->write(']');
+                    $response = $output;
+                } else {
+                    $prettyPrint = !$this->isAutomaticTool(); // Do not pretty print response for automatic tools
+                    $response = JsonTool::encode($response, $prettyPrint);
                 }
-                if (defined('JSON_THROW_ON_ERROR')) {
-                    $flags |= JSON_THROW_ON_ERROR; // Throw exception on error if supported
-                }
-                $response = json_encode($response, $flags);
             } else {
                 if ($dumpSql) {
-                    $this->Log = ClassRegistry::init('Log');
-                    if ($this->Controller->sql_dump === 2) {
-                        $response = json_encode(array('sql_dump' => $this->Log->getDataSource()->getLog(false, false)));
+                    if ($dumpSql === 2) {
+                        $response = JsonTool::encode(['sql_dump' => $this->getSqlLog()]);
                     } else {
                         $response = substr_replace(
                             $response,
-                            sprintf(', "sql_dump": %s}', json_encode($this->Log->getDataSource()->getLog(false, false))),
+                            sprintf(', "sql_dump": %s}', JsonTool::encode($this->getSqlLog())),
                             -2
                         );
                     }
@@ -2104,4 +2114,12 @@ class RestResponseComponent extends Component
         }
         return '/' . $admin_routing . $controller . '/' . $action;
     }
+
+    /**
+     * @return array
+     */
+    private function getSqlLog()
+    {
+        return $this->Controller->User->getDataSource()->getLog(false, false);
+    }
 }

app/Controller/DashboardsController.php

@@ -88,22 +88,24 @@ class DashboardsController extends AppController
 
     public function getForm($action = 'edit')
     {
-        if ($this->request->is('post') || $this->request->is('put')) {
+        if ($this->request->is(['post', 'put'])) {
             $data = $this->request->data;
-            if ($action === 'edit' && !isset($data['widget'])) {
-                throw new InvalidArgumentException(__('No widget name passed.'));
-            }
             if (empty($data['config'])) {
                 $data['config'] = '';
             }
             if ($action === 'add') {
                 $data['widget_options'] = $this->Dashboard->loadAllWidgets($this->Auth->user());
-            } else {
+            } else if ($action === 'edit') {
+                if (!isset($data['widget'])) {
+                    throw new BadRequestException(__('No widget name passed.'));
+                }
                 $dashboardWidget = $this->Dashboard->loadWidget($this->Auth->user(), $data['widget']);
                 $data['description'] = empty($dashboardWidget->description) ? '' : $dashboardWidget->description;
                 $data['params'] = empty($dashboardWidget->params) ? array() : $dashboardWidget->params;
                 $data['params'] = array_merge($data['params'], array('widget_config' => __('Configuration of the widget that will be passed to the render. Check the view for more information')));
                 $data['params'] = array_merge(array('alias' => __('Alias to use as the title of the widget')), $data['params']);
+            } else {
+                throw new BadRequestException(__('Invalid action provided, just add or edit is supported.'));
             }
             $this->set('data', $data);
             $this->layout = false;
@@ -183,13 +185,17 @@ class DashboardsController extends AppController
         } else {
             $data = $dashboardWidget->handler($user, $valueConfig);
         }
-
+        $renderer = method_exists($dashboardWidget, 'getRenderer') ? $dashboardWidget->getRenderer($valueConfig) : $dashboardWidget->render;
         $config = array(
-            'render' => $dashboardWidget->render,
+            'render' => $renderer,
             'autoRefreshDelay' => empty($dashboardWidget->autoRefreshDelay) ? false : $dashboardWidget->autoRefreshDelay,
             'widget_config' => empty($valueConfig['widget_config']) ? array() : $valueConfig['widget_config']
         );
 
+        if (!empty($this->request->params['named']['exportjson'])) {
+            return $this->RestResponse->viewData($data);
+        }
+
         $this->layout = false;
         $this->set('title', $dashboardWidget->title);
         $this->set('widget_id', $widget_id);
@@ -316,6 +322,8 @@ class DashboardsController extends AppController
     public function listTemplates()
     {
         $conditions = array();
+        // load all widgets for internal use, won't be displayed to the user. Thus we circumvent the ACL on it.
+        $accessible_widgets = array_keys($this->Dashboard->loadAllWidgets($this->Auth->user()));
         if (!$this->_isSiteAdmin()) {
             $permission_flags = array();
             foreach ($this->Auth->user('Role') as $perm => $value) {
@@ -394,6 +402,15 @@ class DashboardsController extends AppController
                 }
                 $element['Dashboard']['widgets'] = array_keys($widgets);
                 sort($element['Dashboard']['widgets']);
+                $temp = [];
+                foreach ($element['Dashboard']['widgets'] as $widget) {
+                    if (in_array($widget, $accessible_widgets)) {
+                        $temp['allow'][] = $widget;
+                    } else {
+                        $temp['deny'][] = $widget;
+                    }
+                }
+                $element['Dashboard']['widgets'] = $temp;
                 if ($element['Dashboard']['user_id'] != $this->Auth->user('id')) {
                     $element['User']['email'] = '';
                 }

app/Controller/DecayingModelController.php

@@ -377,13 +377,15 @@ class DecayingModelController extends AppController
 
             $decaying_model['DecayingModel']['enabled'] = 1;
             if ($this->DecayingModel->save($decaying_model)) {
+                $model = $this->DecayingModel->fetchModel($this->Auth->user(), $id, true, array(), true);
+                if (empty($model)) {
+                    throw new NotFoundException(__('No Decaying Model with the provided ID exists'));
+                }
+                $response = array('data' => $model, 'action' => 'enable');
                 if ($this->request->is('ajax')) {
-                    $model = $this->DecayingModel->fetchModel($this->Auth->user(), $id, true, array(), true);
-                    if (empty($model)) {
-                        throw new NotFoundException(__('No Decaying Model with the provided ID exists'));
-                    }
-                    $response = array('data' => $model, 'action' => 'enable');
                     return $this->RestResponse->viewData($response, $this->response->type());
+                } else if ($this->_isRest()) {
+                    return $this->RestResponse->successResponse($id, __('Decaying model enabled'), $model);
                 }
                 $this->Flash->success(__('Decaying Model enabled.'));
             } else {
@@ -400,7 +402,7 @@ class DecayingModelController extends AppController
                 }
                 $this->Flash->error(__('Error while enabling decaying model'));
             }
-            $this->redirect($this->referer());
+            $this->redirect(array('action' => 'index'));
         } else {
             $this->set('model', $decaying_model['DecayingModel']);
             $this->render('ajax/enable_form');
@@ -420,13 +422,15 @@ class DecayingModelController extends AppController
 
             $decaying_model['DecayingModel']['enabled'] = 0;
             if ($this->DecayingModel->save($decaying_model)) {
+                $model = $this->DecayingModel->fetchModel($this->Auth->user(), $id, true, array(), true);
+                if (empty($model)) {
+                    throw new NotFoundException(__('No Decaying Model with the provided ID exists'));
+                }
+                $response = array('data' => $model, 'action' => 'disable');
                 if ($this->request->is('ajax')) {
-                    $model = $this->DecayingModel->fetchModel($this->Auth->user(), $id, true, array(), true);
-                    if (empty($model)) {
-                        throw new NotFoundException(__('No Decaying Model with the provided ID exists'));
-                    }
-                    $response = array('data' => $model, 'action' => 'disable');
                     return $this->RestResponse->viewData($response, $this->response->type());
+                } else if ($this->_isRest()) {
+                    return $this->RestResponse->successResponse($id, __('Decaying model disabled'), $model);
                 }
                 $this->Flash->success(__('Decaying Model disabled.'));
             } else {

app/Controller/EventReportsController.php

@@ -492,9 +492,9 @@ class EventReportsController extends AppController
         $this->set('sharingGroups', $sgs);
     }
 
-    private function __injectPermissionsToViewContext($user, $report)
+    private function __injectPermissionsToViewContext(array $user, array $report)
     {
-        $canEdit = $this->EventReport->canEditReport($user, $report) === true;
+        $canEdit = $this->ACL->canEditEventReport($user, $report);
         $this->set('canEdit', $canEdit);
     }
 

app/Controller/EventsController.php

@@ -111,6 +111,10 @@ class EventsController extends AppController
         if (in_array($this->request->action, ['checkLocks', 'getDistributionGraph'], true)) {
             $this->Security->doNotGenerateToken = true;
         }
+
+        if (Configure::read('Plugin.CustomAuth_enable') && in_array($this->request->action, ['saveFreeText'], true)) {
+            $this->Security->csrfCheck = false;
+        }
     }
 
     /**
@@ -665,6 +669,25 @@ class EventsController extends AppController
                             [$eventReportQuery]
                         ]
                     ];
+                    break;
+                case 'value':
+                    if ($v == "") {
+                        continue 2;
+                    }
+                    $conditions['OR'] = [
+                        ['Attribute.value1' => $v],
+                        ['Attribute.value2' => $v],
+                    ];
+
+                    $eventIds = $this->Event->Attribute->fetchAttributes($this->Auth->user(), array(
+                        'conditions' => $conditions,
+                        'flatten' => true,
+                        'event_ids' => true,
+                        'list' => true,
+                    ));
+
+                    $this->paginate['conditions']['AND'][] = array('Event.id' => $eventIds);
+
                     break;
                 default:
                     continue 2;
@@ -678,7 +701,7 @@ class EventsController extends AppController
     {
         // list the events
         $urlparams = "";
-        $overrideAbleParams = array('all', 'attribute', 'published', 'eventid', 'datefrom', 'dateuntil', 'org', 'eventinfo', 'tag', 'tags', 'distribution', 'sharinggroup', 'analysis', 'threatlevel', 'email', 'hasproposal', 'timestamp', 'publishtimestamp', 'publish_timestamp', 'minimal');
+        $overrideAbleParams = array('all', 'attribute', 'published', 'eventid', 'datefrom', 'dateuntil', 'org', 'eventinfo', 'tag', 'tags', 'distribution', 'sharinggroup', 'analysis', 'threatlevel', 'email', 'hasproposal', 'timestamp', 'publishtimestamp', 'publish_timestamp', 'minimal', 'value');
         $paginationParams = array('limit', 'page', 'sort', 'direction', 'order');
         $passedArgs = $this->passedArgs;
         if (!empty($this->request->data)) {
@@ -990,7 +1013,7 @@ class EventsController extends AppController
             $possibleColumns[] = 'proposals';
         }
 
-        if (Configure::read('MISP.showDiscussionsCountOnIndex')) {
+        if (Configure::read('MISP.showDiscussionsCountOnIndex') && !Configure::read('MISP.discussion_disable')) {
             $possibleColumns[] = 'discussion';
         }
 
@@ -1022,6 +1045,7 @@ class EventsController extends AppController
         if (in_array('tags', $columns, true) || in_array('clusters', $columns, true)) {
             $events = $this->Event->attachTagsToEvents($events);
             $events = $this->GalaxyCluster->attachClustersToEventIndex($user, $events, true);
+            $events = $this->__attachHighlightedTagsToEvents($events);
         }
 
         if (in_array('correlations', $columns, true)) {
@@ -1036,7 +1060,7 @@ class EventsController extends AppController
             $events = $this->Event->attachProposalsCountToEvents($user, $events);
         }
 
-        if (in_array('discussion', $columns, true)) {
+        if (in_array('discussion', $columns, true) && !Configure::read('MISP.discussion_disable')) {
             $events = $this->Event->attachDiscussionsCountToEvents($user, $events);
         }
 
@@ -1302,6 +1326,7 @@ class EventsController extends AppController
         }
 
         // remove galaxies tags
+        $containsProposals = !empty($event['ShadowAttribute']);;
         $this->loadModel('Taxonomy');
         foreach ($event['Object'] as $k => $object) {
             if (isset($object['Attribute'])) {
@@ -1312,6 +1337,9 @@ class EventsController extends AppController
                         $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']);
                         $event['Object'][$k]['Attribute'][$k2]['tagConflicts'] = $tagConflicts;
                     }
+                    if (!$containsProposals && !empty($attribute['ShadowAttribute'])) {
+                        $containsProposals = true;
+                    }
                 }
             }
         }
@@ -1323,6 +1351,9 @@ class EventsController extends AppController
                 $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']);
                 $attribute['tagConflicts'] = $tagConflicts;
             }
+            if (!$containsProposals && !empty($attribute['ShadowAttribute'])) {
+                $containsProposals = true;
+            }
         }
         if (empty($this->passedArgs['sort'])) {
             $filters['sort'] = 'timestamp';
@@ -1337,6 +1368,7 @@ class EventsController extends AppController
         }
         $this->params->params['paging'] = array($this->modelClass => $params);
         $this->set('event', $event);
+        $this->set('includeOrgColumn', (isset($conditions['extended']) || $containsProposals));
         $this->set('includeSightingdb', (!empty($filters['includeSightingdb']) && Configure::read('Plugin.Sightings_sighting_db_enable')));
         $this->set('deleted', isset($filters['deleted']) && $filters['deleted'] != 0);
         $this->set('attributeFilter', isset($filters['attributeFilter']) ? $filters['attributeFilter'] : 'all');
@@ -1386,32 +1418,11 @@ class EventsController extends AppController
 
         $emptyEvent = (empty($event['Object']) && empty($event['Attribute']));
         $this->set('emptyEvent', $emptyEvent);
-        $attributeCount = isset($event['Attribute']) ? count($event['Attribute']) : 0;
-        $objectCount = isset($event['Object']) ? count($event['Object']) : 0;
-        $oldest_timestamp = false;
+
         // set the data for the contributors / history field
         $contributors = $this->Event->ShadowAttribute->getEventContributors($event['Event']['id']);
         $this->set('contributors', $contributors);
 
-        if ($this->__canPublishEvent($event, $user)) {
-            $proposalStatus = false;
-            if (isset($event['ShadowAttribute']) && !empty($event['ShadowAttribute'])) {
-                $proposalStatus = true;
-            }
-            if (!$proposalStatus && !empty($event['Attribute'])) {
-                foreach ($event['Attribute'] as $temp) {
-                    if (!empty($temp['ShadowAttribute'])) {
-                        $proposalStatus = true;
-                        break;
-                    }
-                }
-            }
-            $mess = $this->Session->read('Message');
-            if ($proposalStatus && empty($mess)) {
-                $this->Flash->info('This event has active proposals for you to accept or discard.');
-            }
-        }
-
         // set the pivot data
         $this->helpers[] = 'Pivot';
         if ($continue) {
@@ -1435,7 +1446,7 @@ class EventsController extends AppController
             }
         }
         foreach ($relatedEventCorrelationCount as $key => $relation) {
-            $relatedEventCorrelationCount[$key] = count($relatedEventCorrelationCount[$key]);
+            $relatedEventCorrelationCount[$key] = count($relation);
         }
 
         $this->Event->removeGalaxyClusterTags($event);
@@ -1449,11 +1460,15 @@ class EventsController extends AppController
         }
         $this->set('tagConflicts', $tagConflicts);
 
+        $attributeCount = isset($event['Attribute']) ? count($event['Attribute']) : 0;
+        $objectCount = isset($event['Object']) ? count($event['Object']) : 0;
+        $oldestTimestamp = PHP_INT_MAX;
+        $containsProposals = !empty($event['ShadowAttribute']);
         $modDate = date("Y-m-d", $event['Event']['timestamp']);
         $modificationMap = array($modDate => 1);
         foreach ($event['Attribute'] as $k => $attribute) {
-            if ($oldest_timestamp === false || $oldest_timestamp > $attribute['timestamp']) {
-                $oldest_timestamp = $attribute['timestamp'];
+            if ($oldestTimestamp > $attribute['timestamp']) {
+                $oldestTimestamp = $attribute['timestamp'];
             }
             $modDate = date("Y-m-d", $attribute['timestamp']);
             $modificationMap[$modDate] = !isset($modificationMap[$modDate]) ? 1 : $modificationMap[$modDate] + 1;
@@ -1470,10 +1485,10 @@ class EventsController extends AppController
                 }
                 $event['Attribute'][$k]['tagConflicts'] = $tagConflicts;
             }
+            if (!$containsProposals && !empty($attribute['ShadowAttribute'])) {
+                $containsProposals = true;
+            }
         }
-        $attributeTagsName = $this->Event->Attribute->AttributeTag->extractAttributeTagsNameFromEvent($event);
-        $this->set('attributeTags', array_values($attributeTagsName['tags']));
-        $this->set('attributeClusters', array_values($attributeTagsName['clusters']));
 
         foreach ($event['Object'] as $k => $object) {
             $modDate = date("Y-m-d", $object['timestamp']);
@@ -1481,8 +1496,8 @@ class EventsController extends AppController
             if (!empty($object['Attribute'])) {
                 $attributeCount += count($object['Attribute']);
                 foreach ($object['Attribute'] as $k2 => $attribute) {
-                    if ($oldest_timestamp === false || $oldest_timestamp > $attribute['timestamp']) {
-                        $oldest_timestamp = $attribute['timestamp'];
+                    if ($oldestTimestamp > $attribute['timestamp']) {
+                        $oldestTimestamp = $attribute['timestamp'];
                     }
 
                     $modDate = date("Y-m-d", $attribute['timestamp']);
@@ -1500,9 +1515,24 @@ class EventsController extends AppController
                         }
                         $event['Object'][$k]['Attribute'][$k2]['tagConflicts'] = $tagConflicts;
                     }
+                    if (!$containsProposals && !empty($attribute['ShadowAttribute'])) {
+                        $containsProposals = true;
+                    }
                 }
             }
         }
+
+        if ($containsProposals && $this->__canPublishEvent($event, $user)) {
+            $mess = $this->Session->read('Message');
+            if (empty($mess)) {
+                $this->Flash->info(__('This event has active proposals for you to accept or discard.'));
+            }
+        }
+
+        $attributeTagsName = $this->Event->Attribute->AttributeTag->extractAttributeTagsNameFromEvent($event);
+        $this->set('attributeTags', array_values($attributeTagsName['tags']));
+        $this->set('attributeClusters', array_values($attributeTagsName['clusters']));
+
         $this->set('warningTagConflicts', $warningTagConflicts);
         $filters['sort'] = 'timestamp';
         $filters['direction'] = 'desc';
@@ -1584,9 +1614,10 @@ class EventsController extends AppController
         if (!empty($filters['includeSightingdb']) && Configure::read('Plugin.Sightings_sighting_db_enable')) {
             $this->set('sightingdbs', $this->Sightingdb->getSightingdbList($user));
         }
+        $this->set('includeOrgColumn', $this->viewVars['extended'] || $containsProposals);
         $this->set('includeSightingdb', !empty($filters['includeSightingdb']) && Configure::read('Plugin.Sightings_sighting_db_enable'));
         $this->set('relatedEventCorrelationCount', $relatedEventCorrelationCount);
-        $this->set('oldest_timestamp', $oldest_timestamp);
+        $this->set('oldest_timestamp', $oldestTimestamp === PHP_INT_MAX ? false : $oldestTimestamp);
         $this->set('missingTaxonomies', $this->Event->missingTaxonomies($event));
         $this->set('currentUri', $attributeUri);
         $this->set('filters', $filters);
@@ -1645,6 +1676,7 @@ class EventsController extends AppController
             $cortex_modules = $this->Module->getEnabledModules($user, false, 'Cortex');
             $this->set('cortex_modules', $cortex_modules);
         }
+        $this->set('sightingsDbEnabled', (bool)Configure::read('Plugin.Sightings_sighting_db_enable'));
     }
 
     public function view($id = null, $continue = false, $fromEvent = null)
@@ -1805,6 +1837,8 @@ class EventsController extends AppController
         $this->set('includeRelatedTags', (!empty($namedParams['includeRelatedTags'])) ? 1 : 0);
         $this->set('includeDecayScore', (!empty($namedParams['includeDecayScore'])) ? 1 : 0);
 
+        $this->__setHighlightedTags($event);
+
         if ($this->_isSiteAdmin() && $event['Event']['orgc_id'] !== $this->Auth->user('org_id')) {
             $this->Flash->info(__('You are currently logged in as a site administrator and about to edit an event not belonging to your organisation. This goes against the sharing model of MISP. Use a normal user account for day to day work.'));
         }
@@ -2273,7 +2307,12 @@ class EventsController extends AppController
         foreach ($analysisLevels as $key => $value) {
             $fieldDesc['analysis'][$key] = $this->Event->analysisDescriptions[$key]['formdesc'];
         }
-        $this->Flash->info(__('The event created will be visible to the organisations having an account on this platform, but not synchronised to other MISP instances until it is published.'));
+
+        if (Configure::read('MISP.unpublishedprivate')) {
+            $this->Flash->info(__('The event created will be visible only to your organisation until it is published.'));
+        } else {
+            $this->Flash->info(__('The event created will be visible to the organisations having an account on this platform, but not synchronised to other MISP instances until it is published.'));
+        }
         $this->set('fieldDesc', $fieldDesc);
         if (isset($this->params['named']['extends'])) {
             $this->set('extends_uuid', $this->params['named']['extends']);
@@ -2300,7 +2339,7 @@ class EventsController extends AppController
         // set the id
         $this->set('id', $id);
         // set whether it is published or not
-        $this->set('published', $this->Event->data['Event']['published']);
+        $this->set('published', $this->Event->data['Event']['published'] ?? false);
     }
 
     public function add_misp_export()
@@ -2308,6 +2347,9 @@ class EventsController extends AppController
         if ($this->request->is('post')) {
             $results = array();
             if (!empty($this->request->data)) {
+                if (empty($this->request->data['Event'])) {
+                    $this->request->data['Event'] = $this->request->data;
+                }
                 if (!empty($this->request->data['Event']['filecontent'])) {
                     $data = $this->request->data['Event']['filecontent'];
                     $isXml = $data[0] === '<';
@@ -2341,29 +2383,64 @@ class EventsController extends AppController
                     $results = $this->Event->addMISPExportFile($this->Auth->user(), $data, $isXml, $takeOwnership, $publish);
                 } catch (Exception $e) {
                     $this->log("Exception during processing MISP file import: {$e->getMessage()}");
-                    $this->Flash->error(__('Could not process MISP export file. Probably file content is invalid.'));
+                    $this->Flash->error(__('Could not process MISP export file. %s.', $e->getMessage()));
                     $this->redirect(['controller' => 'events', 'action' => 'add_misp_export']);
                 }
             }
             $this->set('results', $results);
             $this->render('add_misp_export_result');
         }
+        $this->set('title_for_layout', __('Import from MISP Export File'));
     }
 
-    public function upload_stix($stix_version = '1', $publish = false)
+    public function upload_stix($stix_version = '1', $publish = false, $galaxies_as_tags = true, $debug = false)
     {
+        $sgs = $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1);
+        $initialDistribution = 0;
+        if (Configure::read('MISP.default_event_distribution') != null) {
+            $initialDistribution = Configure::read('MISP.default_event_distribution');
+        }
+        $distributionLevels = $this->Event->distributionLevels;
         if ($this->request->is('post')) {
             if ($this->_isRest()) {
                 if (isset($this->params['named']['publish'])) {
                     $publish = $this->params['named']['publish'];
                 }
+                if (isset($this->params['named']['distribution'])) {
+                    $distribution = intval($this->params['named']['distribution']);
+                    if (array_key_exists($distribution, $distributionLevels)) {
+                        $initialDistribution = $distribution;
+                    } else {
+                        throw new MethodNotAllowedException(__('Wrong distribution level'));
+                    }
+                }
+                $sharingGroupId = null;
+                if ($initialDistribution == 4) {
+                    if (!isset($this->params['named']['sharing_group_id'])) {
+                        throw new MethodNotAllowedException(__('The sharing group id is needed when the distribution is set to 4 ("Sharing group").'));
+                    }
+                    $sharingGroupId = intval($this->params['named']['sharing_group_id']);
+                    if (!array_key_exists($sharingGroupId, $sgs)) {
+                        throw new MethodNotAllowedException(__('Please select a valid sharing group id.'));
+                    }
+                }
+                if (isset($this->params['named']['galaxies_as_tags'])) {
+                    $galaxies_as_tags = $this->params['named']['galaxies_as_tags'];
+                }
+                if (isset($this->params['named']['debugging'])) {
+                    $debug = $this->params['named']['debugging'];
+                }
                 $filePath = FileAccessTool::writeToTempFile($this->request->input());
                 $result = $this->Event->upload_stix(
                     $this->Auth->user(),
                     $filePath,
                     $stix_version,
                     'uploaded_stix_file.' . ($stix_version == '1' ? 'xml' : 'json'),
-                    $publish
+                    $publish,
+                    $initialDistribution,
+                    $sharingGroupId,
+                    $galaxies_as_tags,
+                    $debug
                 );
                 if (is_numeric($result)) {
                     $event = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $result));
@@ -2382,12 +2459,19 @@ class EventsController extends AppController
                     if (!move_uploaded_file($this->data['Event']['stix']['tmp_name'], $filePath)) {
                         throw new Exception("Could not move uploaded STIX file.");
                     }
+                    if (isset($this->data['Event']['debug'])) {
+                        $debug = $this->data['Event']['debug'];
+                    }
                     $result = $this->Event->upload_stix(
                         $this->Auth->user(),
                         $filePath,
                         $stix_version,
                         $original_file,
-                        $this->data['Event']['publish']
+                        $this->data['Event']['publish'],
+                        $this->data['Event']['distribution'],
+                        $this->data['Event']['sharing_group_id'],
+                        !boolval($this->data['Event']['galaxies_parsing']),
+                        $debug
                     );
                     if (is_numeric($result)) {
                         $this->Flash->success(__('STIX document imported.'));
@@ -2405,6 +2489,20 @@ class EventsController extends AppController
             }
         }
         $this->set('stix_version', $stix_version == 2 ? '2.x JSON' : '1.x XML');
+        $this->set('initialDistribution', $initialDistribution);
+        $distributions = array_keys($this->Event->distributionDescriptions);
+        $distributions = $this->_arrayToValuesIndexArray($distributions);
+        $this->set('distributions', $distributions);
+        $fieldDesc = array();
+        if (empty($sgs)) {
+            unset($distributionLevels[4]);
+        }
+        $this->set('distributionLevels', $distributionLevels);
+        foreach ($distributionLevels as $key => $value) {
+            $fieldDesc['distribution'][$key] = $this->Event->distributionDescriptions[$key]['formdesc'];
+        }
+        $this->set('sharingGroups', $sgs);
+        $this->set('fieldDesc', $fieldDesc);
     }
 
     public function merge($target_id=null, $source_id=null)
@@ -2540,7 +2638,7 @@ class EventsController extends AppController
         }
     }
 
-    public function populate($id)
+    public function populate($id, $regenerateUUIDs=false)
     {
         if ($this->request->is('get') && $this->_isRest()) {
             return $this->RestResponse->describe('Events', 'populate', false, $this->response->type());
@@ -2562,15 +2660,31 @@ class EventsController extends AppController
         }
         if ($this->request->is('post') || $this->request->is('put')) {
             if (isset($this->request->data['Event'])) {
+                $regenerateUUIDs = $this->request->data['Event']['regenerate_uuids'] ?? false;
                 $this->request->data = $this->request->data['Event'];
             }
             if (isset($this->request->data['json'])) {
                 $this->request->data = $this->_jsonDecode($this->request->data['json']);
             }
+            if (isset($this->request->data['Event'])) {
+                $this->request->data = $this->request->data['Event'];
+            }
             $eventToSave = $event;
             $capturedObjects = ['Attribute', 'Object', 'Tag', 'Galaxy', 'EventReport'];
             foreach ($capturedObjects as $objectType) {
                 if (!empty($this->request->data[$objectType])) {
+                    if (!empty($regenerateUUIDs)) {
+                        foreach ($this->request->data[$objectType] as $i => $obj) {
+                            unset($this->request->data[$objectType][$i]['id']);
+                            unset($this->request->data[$objectType][$i]['uuid']);
+                            if ($objectType === 'Object' && !empty($this->request->data[$objectType][$i]['Attribute'])) {
+                                foreach ($this->request->data[$objectType][$i]['Attribute'] as $j => $attr) {
+                                    unset($this->request->data[$objectType][$i]['Attribute'][$j]['id']);
+                                    unset($this->request->data[$objectType][$i]['Attribute'][$j]['uuid']);
+                                }
+                            }
+                        }
+                    }
                     $eventToSave['Event'][$objectType] = $this->request->data[$objectType];
                 }
             }
@@ -3494,6 +3608,7 @@ class EventsController extends AppController
                 'category' => 'External analysis',
                 'uuid' =>  CakeText::uuid(),
                 'type' => 'attachment',
+                'sharing_group_id' => '0',
                 'value' => $this->data['Event']['submittedioc']['name'],
                 'to_ids' => false,
                 'distribution' => $dist,
@@ -3508,7 +3623,7 @@ class EventsController extends AppController
 
             $fieldList = array(
                     'Event' => array('published', 'timestamp'),
-                    'Attribute' => array('event_id', 'category', 'type', 'value', 'value1', 'value2', 'to_ids', 'uuid', 'distribution', 'timestamp', 'comment')
+                    'Attribute' => array('event_id', 'category', 'type', 'value', 'value1', 'value2', 'to_ids', 'uuid', 'distribution', 'timestamp', 'comment', 'sharing_group_id')
             );
             // Save it all
             $saveResult = $this->Event->saveAssociated($saveEvent, array('validate' => true, 'fieldList' => $fieldList));
@@ -3985,6 +4100,7 @@ class EventsController extends AppController
             $this->set('mayModify', $this->__canModifyEvent($event));
             $this->set('typeDefinitions', $this->Event->Attribute->typeDefinitions);
             $this->set('typeCategoryMapping', $typeCategoryMapping);
+            $this->set('defaultAttributeDistribution', $this->Event->Attribute->defaultDistribution());
             $this->set('resultArray', $resultArray);
             $this->set('importComment', '');
             $this->set('title_for_layout', __('Freetext Import Results'));
@@ -4290,7 +4406,7 @@ class EventsController extends AppController
             ),
             'bro' => array(
                 'url' => $this->baseurl . '/attributes/bro/download/all/false/' . $id,
-                // 'url' => '/attributes/restSearch/returnFormat:bro/published:1||0/eventid:' . $id,
+                // 'url' => $this->baseurl . '/attributes/restSearch/returnFormat:bro/published:1||0/eventid:' . $id,
                 'text' => __('Bro rules'),
                 'requiresPublished' => false,
                 'checkbox' => false,
@@ -4396,12 +4512,12 @@ class EventsController extends AppController
                 ),
                 'STIX' => array(
                     'url' => $this->baseurl . '/events/upload_stix',
-                    'text' => __('STIX 1.1.1 format (lossy)'),
+                    'text' => __('STIX 1.x format (lossy)'),
                     'ajax' => false,
                 ),
                 'STIX2' => array(
                     'url' => $this->baseurl . '/events/upload_stix/2',
-                    'text' => __('STIX 2.0 format (lossy)'),
+                    'text' => __('STIX 2.x format (lossy)'),
                     'ajax' => false,
                 )
             );
@@ -5094,42 +5210,82 @@ class EventsController extends AppController
         $this->render('index');
     }
 
-    // expects an attribute ID and the module to be used
-    public function queryEnrichment($attribute_id, $module = false, $type = 'Enrichment')
+    // expects a model ID, model type, the module to be used (optional) and the type of enrichment (optional)
+    public function queryEnrichment($id, $module = false, $type = 'Enrichment', $model = 'Attribute')
     {
         if (!Configure::read('Plugin.' . $type . '_services_enable')) {
             throw new MethodNotAllowedException(__('%s services are not enabled.', $type));
         }
-        $attribute = $this->Event->Attribute->fetchAttributes($this->Auth->user(), [
-            'conditions' => [
-                'Attribute.id' => $attribute_id
-            ],
-            'flatten' => 1,
-            'includeEventTags' => 1,
-            'contain' => ['Event' => ['fields' => ['distribution', 'sharing_group_id']]],
-        ]);
-        if (empty($attribute)) {
-            throw new MethodNotAllowedException(__('Attribute not found or you are not authorised to see it.'));
+
+        if (!in_array($model, array('Attribute', 'ShadowAttribute', 'Object', 'Event'))) {
+            throw new MethodNotAllowedException(__('Invalid model.'));
         }
+
         $this->loadModel('Module');
         $enabledModules = $this->Module->getEnabledModules($this->Auth->user(), false, $type);
+        
         if (!is_array($enabledModules) || empty($enabledModules)) {
-            throw new MethodNotAllowedException(__('No valid %s options found for this attribute.', $type));
+            throw new MethodNotAllowedException(__('No valid %s options found for this %s.', $type, strtolower($model)));
         }
+
+        if ($model === 'Attribute' || $model === 'ShadowAttribute') {
+            $attribute = $this->Event->Attribute->fetchAttributes($this->Auth->user(), [
+                'conditions' => [
+                    'Attribute.id' => $id
+                ],
+                'flatten' => 1,
+                'includeEventTags' => 1,
+                'contain' => ['Event' => ['fields' => ['distribution', 'sharing_group_id']]],
+            ]);
+            if (empty($attribute)) {
+                throw new MethodNotAllowedException(__('Attribute not found or you are not authorised to see it.'));
+            }
+        }
+
+        if ($model === 'Object') {
+            $object = $this->Event->Object->fetchObjects($this->Auth->user(), [
+                'conditions' => [
+                    'Object.id' => $id
+                ],
+                'flatten' => 1,
+                'includeEventTags' => 1,
+                'contain' => ['Event' => ['fields' => ['distribution', 'sharing_group_id']]],
+            ]);
+            if (empty($object)) {
+                throw new MethodNotAllowedException(__('Object not found or you are not authorised to see it.'));
+            }
+        }
+        
         if ($this->request->is('ajax')) {
-            $modules = array();
-            foreach ($enabledModules['modules'] as $module) {
-                if (in_array($attribute[0]['Attribute']['type'], $module['mispattributes']['input'])) {
-                    $modules[] = array('name' => $module['name'], 'description' => $module['meta']['description']);
+            $modules = [];
+
+            if ($model === 'Attribute' || $model === 'ShadowAttribute') {
+                foreach ($enabledModules['modules'] as $module) {
+                    if (in_array($attribute[0]['Attribute']['type'], $module['mispattributes']['input'])) {
+                        $modules[] = array('name' => $module['name'], 'description' => $module['meta']['description']);
+                    }
                 }
             }
-            foreach (array('attribute_id', 'modules') as $viewVar) {
-                $this->set($viewVar, $$viewVar);
+
+            if ($model === 'Object') {
+                foreach ($enabledModules['modules'] as $module) {
+                    if (
+                        in_array($object[0]['Object']['name'], $module['mispattributes']['input']) ||
+                        in_array($object[0]['Object']['uuid'], $module['mispattributes']['input'])
+                    ) {
+                        $modules[] = array('name' => $module['name'], 'description' => $module['meta']['description']);
+                    }
+                }
             }
+            
+            $this->set('id', $id);
+            $this->set('modules', $modules);
             $this->set('type', $type);
+            $this->set('model', $model);
             $this->render('ajax/enrichmentChoice');
         } else {
-            $options = array();
+            $options = [];
+            $format = 'simplified';
             foreach ($enabledModules['modules'] as $temp) {
                 if ($temp['name'] == $module) {
                     $format = !empty($temp['mispattributes']['format']) ? $temp['mispattributes']['format'] : 'simplified';
@@ -5151,7 +5307,13 @@ class EventsController extends AppController
             $this->set('title_for_layout', __('Enrichment Results'));
             $this->set('title', __('Enrichment Results'));
             if ($format == 'misp_standard') {
-                $this->__queryEnrichment($attribute, $module, $options, $type);
+                if ($model === 'Attribute' || $model === 'ShadowAttribute') {
+                    $this->__queryEnrichment($attribute, $module, $options, $type);
+                }
+                
+                if ($model === 'Object') {
+                    $this->__queryObjectEnrichment($object, $module, $options, $type);
+                }
             } else {
                 $this->__queryOldEnrichment($attribute, $module, $options, $type);
             }
@@ -5205,6 +5367,57 @@ class EventsController extends AppController
         }
     }
 
+    private function __queryObjectEnrichment($object, $module, $options, $type)
+    {
+        $object[0]['Object']['Attribute'] = $object[0]['Attribute'];
+        foreach($object[0]['Object']['Attribute'] as &$attribute) {
+            if ($this->Event->Attribute->typeIsAttachment($attribute['type'])) {
+                $attribute['data'] = $this->Event->Attribute->base64EncodeAttachment($attribute);
+            }
+        }
+
+        $event_id = $object[0]['Event']['id'];
+        $data = array('module' => $module, 'object' => $object[0]['Object'], 'event_id' => $event_id);
+        if (!empty($options)) {
+            $data['config'] = $options;
+        }
+        $result = $this->Module->queryModuleServer($data, false, $type, false, $object[0]);
+        if (!$result) {
+            throw new InternalErrorException(__('%s service not reachable.', $type));
+        }
+        if (isset($result['error'])) {
+            $this->Flash->error($result['error']);
+        }
+        if (!is_array($result)) {
+            throw new Exception($result);
+        }
+        $event = $this->Event->handleMispFormatFromModuleResult($result);
+        if (empty($event['Attribute']) && empty($event['Object'])) {
+            throw new NotImplementedException(__('No Attribute or Object returned by the module.'));
+        } else {
+            $importComment = !empty($result['comment']) ? $result['comment'] : $object[0]['Object']['value'] . __(': Enriched via the ') . $module . ($type != 'Enrichment' ? ' ' . $type : '')  . ' module';
+            $this->set('importComment', $importComment);
+            $event['Event'] = $object[0]['Event'];
+            $org_name = $this->Event->Orgc->find('first', array(
+                'conditions' => array('Orgc.id' => $event['Event']['orgc_id']),
+                'fields' => array('Orgc.name')
+            ));
+            $event['Event']['orgc_name'] = $org_name['Orgc']['name'];
+            if ($attribute[0]['Object']['id']) {
+                $object_id = $attribute[0]['Object']['id'];
+                $initial_object = $this->Event->fetchInitialObject($event_id, $object_id);
+                if (!empty($initial_object)) {
+                    $event['initialObject'] = $initial_object;
+                }
+            }
+            $this->set('event', $event);
+            $this->set('menuItem', 'enrichmentResults');
+            $this->set('title_for_layout', __('Enrichment Results'));
+            $this->set('title', __('Enrichment Results'));
+            $this->render('resolved_misp_format');
+        }
+    }
+
     private function __queryOldEnrichment($attribute, $module, $options, $type)
     {
         $data = array('module' => $module, $attribute[0]['Attribute']['type'] => $attribute[0]['Attribute']['value'], 'event_id' => $attribute[0]['Attribute']['event_id'], 'attribute_uuid' => $attribute[0]['Attribute']['uuid']);
@@ -5258,6 +5471,7 @@ class EventsController extends AppController
         $this->set('resultArray', $resultArray);
         $this->set('typeDefinitions', $this->Event->Attribute->typeDefinitions);
         $this->set('typeCategoryMapping', $typeCategoryMapping);
+        $this->set('defaultAttributeDistribution', $this->Event->Attribute->defaultDistribution());
         $this->set('importComment', $importComment);
         $this->render('resolved_attributes');
     }
@@ -5292,7 +5506,7 @@ class EventsController extends AppController
         }
     }
 
-    public function importModule($module, $eventId)
+    public function importModule($moduleName, $eventId)
     {
         $event = $this->Event->fetchSimpleEvent($this->Auth->user(), $eventId);
         if (!$event) {
@@ -5302,8 +5516,7 @@ class EventsController extends AppController
         $eventId = $event['Event']['id'];
 
         $this->loadModel('Module');
-        $moduleName = $module;
-        $module = $this->Module->getEnabledModule($module, 'Import');
+        $module = $this->Module->getEnabledModule($moduleName, 'Import');
         if (!is_array($module)) {
             throw new MethodNotAllowedException($module);
         }
@@ -5311,10 +5524,11 @@ class EventsController extends AppController
             $module['mispattributes']['inputSource'] = array('paste');
         }
         if ($this->request->is('post')) {
+            $requestData = $this->request->data['Event'];
             $fail = false;
             $modulePayload = array(
-                    'module' => $module['name'],
-                    'event_id' => $eventId,
+                'module' => $module['name'],
+                'event_id' => $eventId,
             );
             if (isset($module['meta']['config'])) {
                 foreach ($module['meta']['config'] as $conf) {
@@ -5322,11 +5536,11 @@ class EventsController extends AppController
                 }
             }
             if ($moduleName === 'csvimport') {
-                if (empty($this->request->data['Event']['config']['header']) && $this->request->data['Event']['config']['has_header'] === '1') {
-                    $this->request->data['Event']['config']['header'] = ' ';
+                if (empty($requestData['config']['header']) && $requestData['config']['has_header'] === '1') {
+                    $requestData['config']['header'] = ' ';
                 }
-                if (empty($this->request->data['Event']['config']['special_delimiter'])) {
-                    $this->request->data['Event']['config']['special_delimiter'] = ' ';
+                if (empty($requestData['config']['special_delimiter'])) {
+                    $requestData['config']['special_delimiter'] = ' ';
                 }
             }
             if (isset($module['mispattributes']['userConfig'])) {
@@ -5337,18 +5551,19 @@ class EventsController extends AppController
                                 $validation = true;
                             }
                         } else {
-                            $validation = call_user_func_array(array($this->Module, $this->Module->configTypes[$config['type']]['validation']), array($this->request->data['Event']['config'][$configName]));
+                            $validationMethod = Module::CONFIG_TYPES[$config['type']]['validation'];
+                            $validation = $this->Module->{$validationMethod}($requestData['config'][$configName]);
                         }
                         if ($validation !== true) {
                             $fail = ucfirst($configName) . ': ' . $validation;
                         } else {
                             if (isset($config['regex']) && !empty($config['regex'])) {
-                                $fail = preg_match($config['regex'], $this->request->data['Event']['config'][$configName]) ? false : ucfirst($configName) . ': ' . 'Invalid setting' . ($config['errorMessage'] ? ' - ' . $config['errorMessage'] : '');
+                                $fail = preg_match($config['regex'], $requestData['config'][$configName]) ? false : ucfirst($configName) . ': ' . 'Invalid setting' . ($config['errorMessage'] ? ' - ' . $config['errorMessage'] : '');
                                 if (!empty($fail)) {
-                                    $modulePayload['config'][$configName] = $this->request->data['Event']['config'][$configName];
+                                    $modulePayload['config'][$configName] = $requestData['config'][$configName];
                                 }
                             } else {
-                                $modulePayload['config'][$configName] = $this->request->data['Event']['config'][$configName];
+                                $modulePayload['config'][$configName] = $requestData['config'][$configName];
                             }
                         }
                     }
@@ -5356,31 +5571,29 @@ class EventsController extends AppController
             }
             if (!$fail) {
                 if (!empty($module['mispattributes']['inputSource'])) {
-                    if (!isset($this->request->data['Event']['source'])) {
+                    if (!isset($requestData['source'])) {
                         if (in_array('paste', $module['mispattributes']['inputSource'])) {
-                            $this->request->data['Event']['source'] = '0';
+                            $requestData['source'] = '0';
                         } else {
-                            $this->request->data['Event']['source'] = '1';
+                            $requestData['source'] = '1';
                         }
                     }
-                    if ($this->request->data['Event']['source'] == '1') {
-                        if (isset($this->request->data['Event']['data'])) {
-                            $modulePayload['data'] = base64_decode($this->request->data['Event']['data']);
-                        } elseif (!isset($this->request->data['Event']['fileupload']) || empty($this->request->data['Event']['fileupload'])) {
-                            $fail = 'Invalid file upload.';
+                    if ($requestData['source'] == '1') {
+                        if (isset($requestData['data'])) {
+                            $modulePayload['data'] = base64_decode($requestData['data']);
+                        } elseif (empty($requestData['fileupload'])) {
+                            $fail = __('Invalid file upload.');
                         } else {
-                            $fileupload = $this->request->data['Event']['fileupload'];
-                            $tmpfile = new File($fileupload['tmp_name']);
-                            if ((isset($fileupload['error']) && $fileupload['error'] == 0) || (!empty($fileupload['tmp_name']) && $fileupload['tmp_name'] != 'none') && is_uploaded_file($tmpfile->path)) {
+                            $fileupload = $requestData['fileupload'];
+                            if ((isset($fileupload['error']) && $fileupload['error'] == 0) || (!empty($fileupload['tmp_name']) && $fileupload['tmp_name'] != 'none') && is_uploaded_file($fileupload['tmp_name'])) {
                                 $filename = basename($fileupload['name']);
-                                App::uses('FileAccessTool', 'Tools');
-                                $modulePayload['data'] = FileAccessTool::readFromFile($fileupload['tmp_name'], $fileupload['size']);
+                                $modulePayload['data'] = FileAccessTool::readAndDelete($fileupload['tmp_name']);
                             } else {
-                                $fail = 'Invalid file upload.';
+                                $fail = __('Invalid file upload.');
                             }
                         }
                     } else {
-                        $modulePayload['data'] = $this->request->data['Event']['paste'];
+                        $modulePayload['data'] = $requestData['paste'];
                     }
                 } else {
                     $modulePayload['data'] = '';
@@ -5402,13 +5615,13 @@ class EventsController extends AppController
                     }
                     $importComment = !empty($result['comment']) ? $result['comment'] : 'Enriched via the ' . $module['name'] . ' module';
                     if (!empty($module['mispattributes']['format']) && $module['mispattributes']['format'] === 'misp_standard') {
-                        $event = $this->Event->handleMispFormatFromModuleResult($result);
-                        $event['Event'] = array('id' => $eventId);
+                        $resolvedEvent = $this->Event->handleMispFormatFromModuleResult($result);
+                        $resolvedEvent['Event'] = $event['Event'];
                         if ($this->_isRest()) {
-                            $this->Event->processModuleResultsDataRouter($this->Auth->user(), $event, $eventId, $importComment);
-                            return $this->RestResponse->viewData($event, $this->response->type());
+                            $this->Event->processModuleResultsDataRouter($this->Auth->user(), $resolvedEvent, $eventId, $importComment);
+                            return $this->RestResponse->viewData($resolvedEvent, $this->response->type());
                         }
-                        $this->set('event', $event);
+                        $this->set('event', $resolvedEvent);
                         $this->set('menuItem', 'importResults');
                         $render_name = 'resolved_misp_format';
                     } else {
@@ -5433,15 +5646,13 @@ class EventsController extends AppController
                         $this->set('resultArray', $resultArray);
                         $this->set('typeDefinitions', $this->Event->Attribute->typeDefinitions);
                         $this->set('typeCategoryMapping', $typeCategoryMapping);
+                        $this->set('defaultAttributeDistribution', $this->Event->Attribute->defaultDistribution());
                         $render_name = 'resolved_attributes';
                     }
-                    $distributions = $this->Event->Attribute->distributionLevels;
-                    $sgs = $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1);
-                    if (empty($sgs)) {
-                        unset($distributions[4]);
-                    }
-                    $this->set('distributions', $distributions);
-                    $this->set('sgs', $sgs);
+
+                    $distributionData = $this->Event->Attribute->fetchDistributionData($this->Auth->user());
+                    $this->set('distributions', $distributionData['levels']);
+                    $this->set('sgs', $distributionData['sgs']);
                     $this->set('title', __('Import Results'));
                     $this->set('title_for_layout', __('Import Results'));
                     $this->set('importComment', $importComment);
@@ -5452,7 +5663,7 @@ class EventsController extends AppController
                 $this->Flash->error($fail);
             }
         }
-        $this->set('configTypes', $this->Module->configTypes);
+        $this->set('configTypes', Module::CONFIG_TYPES);
         $this->set('module', $module);
         $this->set('eventId', $eventId);
         $this->set('event', $event);
@@ -6217,4 +6428,31 @@ class EventsController extends AppController
             $this->render('/genericTemplates/confirm');
         }
     }
+
+    /**
+     * @param array $event
+     * @return void
+     */
+    private function __setHighlightedTags($event)
+    {
+        $this->loadModel('Taxonomy');
+        $highlightedTags = $this->Taxonomy->getHighlightedTags($this->Taxonomy->getHighlightedTaxonomies(), $event['EventTag']);
+        $this->set('highlightedTags', $highlightedTags);
+    }
+
+    /**
+     *
+     * @param array $events
+     * @return array
+     */
+    private function __attachHighlightedTagsToEvents($events)
+    {
+        $this->loadModel('Taxonomy');
+        $highlightedTaxonomies = $this->Taxonomy->getHighlightedTaxonomies();
+        foreach ($events as $k => $event) {
+            $events[$k]['Event']['highlightedTags'] = $this->Taxonomy->getHighlightedTags($highlightedTaxonomies, $event['EventTag']);
+        }
+
+        return $events;
+    }
 }

app/Controller/FeedsController.php

@@ -311,6 +311,7 @@ class FeedsController extends AppController
         ];
         $this->set('allAttributeTypes', $allTypes['attribute']);
         $this->set('allObjectTypes', $allTypes['object']);
+        $this->set('supportedUrlparams', Feed::SUPPORTED_URL_PARAM_FILTERS);
         $this->set(compact('dropdownData'));
         $this->set('defaultPullRules', json_encode(Feed::DEFAULT_FEED_PULL_RULES));
         $this->set('menuData', array('menuList' => 'feeds', 'menuItem' => 'add'));

app/Controller/GalaxiesController.php

@@ -22,8 +22,16 @@ class GalaxiesController extends AppController
     public function index()
     {
         $aclConditions = array();
-        $filters = $this->IndexFilter->harvestParameters(array('value', 'enabled'));
-        $searchConditions = array();
+        $filterData = array(
+            'request' => $this->request,
+            'named_params' => $this->params['named'],
+            'paramArray' => ['value', 'enabled'],
+            'ordered_url_params' => [],
+            'additional_delimiters' => PHP_EOL
+        );
+        $exception = false;
+        $filters = $this->_harvestParameters($filterData, $exception);
+        $searchConditions = [];
         if (empty($filters['value'])) {
             $filters['value'] = '';
         } else {
@@ -220,14 +228,15 @@ class GalaxiesController extends AppController
             } else {
                 $data = $this->request->data['Galaxy'];
                 $text = FileAccessTool::getTempUploadedFile($data['submittedjson'], $data['json']);
-                $clusters = json_decode($text, true);
-                if ($clusters === null) {
-                    throw new MethodNotAllowedException(__('Error while decoding JSON'));
+                try {
+                    $clusters = JsonTool::decodeArray($text);
+                } catch (Exception $e) {
+                    throw new BadRequestException(__('Error while decoding JSON'));
                 }
             }
             $saveResult = $this->Galaxy->importGalaxyAndClusters($this->Auth->user(), $clusters);
             if ($saveResult['success']) {
-                $message = sprintf(__('Galaxy clusters imported. %s imported, %s ignored, %s failed. %s'), $saveResult['imported'], $saveResult['ignored'], $saveResult['failed'], !empty($saveResult['errors']) ? implode(', ', $saveResult['errors']) : '');
+                $message = __('Galaxy clusters imported. %s imported, %s ignored, %s failed. %s', $saveResult['imported'], $saveResult['ignored'], $saveResult['failed'], !empty($saveResult['errors']) ? implode(', ', $saveResult['errors']) : '');
                 if ($this->_isRest()) {
                     return $this->RestResponse->saveSuccessResponse('Galaxy', 'import', false, $this->response->type(), $message);
                 } else {
@@ -235,7 +244,7 @@ class GalaxiesController extends AppController
                     $this->redirect(array('controller' => 'galaxies', 'action' => 'index'));
                 }
             } else {
-                $message = sprintf(__('Could not import galaxy clusters. %s imported, %s ignored, %s failed. %s'), $saveResult['imported'], $saveResult['ignored'], $saveResult['failed'], !empty($saveResult['errors']) ? implode(', ', $saveResult['errors']) : '');
+                $message = __('Could not import galaxy clusters. %s imported, %s ignored, %s failed. %s', $saveResult['imported'], $saveResult['ignored'], $saveResult['failed'], !empty($saveResult['errors']) ? implode(', ', $saveResult['errors']) : '');
                 if ($this->_isRest()) {
                     return $this->RestResponse->saveFailResponse('Galaxy', 'import', false, $message);
                 } else {
@@ -525,7 +534,7 @@ class GalaxiesController extends AppController
             }
         }
 
-        $result = $this->Galaxy->attachCluster($user, $target_type, $target_id, $cluster_id, $local);
+        $result = $this->Galaxy->attachCluster($user, $target_type, $target, $cluster_id, $local);
         return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => $result, 'check_publish' => true)), 'status'=>200, 'type' => 'json'));
     }
 

app/Controller/GalaxyClustersController.php

@@ -37,7 +37,8 @@ class GalaxyClustersController extends AppController
 
     public function index($galaxyId)
     {
-        $filters = $this->IndexFilter->harvestParameters(array('context', 'searchall'));
+        $galaxyId = $this->Toolbox->findIdByUuid($this->GalaxyCluster->Galaxy, $galaxyId);
+        $filters = $this->_harvestParameters(array('context', 'searchall'));
         $aclConditions = $this->GalaxyCluster->buildConditions($this->Auth->user());
         $contextConditions = array();
         if (empty($filters['context'])) {
@@ -47,7 +48,7 @@ class GalaxyClustersController extends AppController
         }
 
         if ($filters['context'] == 'default') {
-                $contextConditions['GalaxyCluster.default'] = true;
+            $contextConditions['GalaxyCluster.default'] = true;
         } elseif ($filters['context'] == 'custom') {
             $contextConditions['GalaxyCluster.default'] = false;
         } elseif ($filters['context'] == 'org') {
@@ -146,9 +147,8 @@ class GalaxyClustersController extends AppController
                 'GalaxyCluster.default' => 0,
             ]
         ]);
-        $this->loadModel('Attribute');
-        $distributionLevels = $this->Attribute->distributionLevels;
-        unset($distributionLevels[5]);
+        $this->loadModel('Event');
+        $distributionLevels = $this->Event->shortDist;
         $this->set('distributionLevels', $distributionLevels);
         $this->set('list', $clusters);
         $this->set('galaxy_id', $galaxyId);

app/Controller/GalaxyElementsController.php

@@ -1,6 +1,9 @@
 <?php
 App::uses('AppController', 'Controller');
 
+/**
+ * @property GalaxyElement $GalaxyElement
+ */
 class GalaxyElementsController extends AppController
 {
     public $components = array('Session', 'RequestHandler');
@@ -16,8 +19,9 @@ class GalaxyElementsController extends AppController
 
     public function index($clusterId)
     {
+        $user = $this->_closeSession();
         $filters = $this->IndexFilter->harvestParameters(array('context', 'searchall'));
-        $aclConditions = $this->GalaxyElement->buildClusterConditions($this->Auth->user(), $clusterId);
+        $aclConditions = $this->GalaxyElement->buildClusterConditions($user, $clusterId);
         if (empty($filters['context'])) {
             $filters['context'] = 'all';
         }
@@ -44,18 +48,15 @@ class GalaxyElementsController extends AppController
             'context' => $filters['context'],
             'searchall' => isset($filters['searchall']) ? $filters['searchall'] : ''
         ]));
-        $cluster = $this->GalaxyElement->GalaxyCluster->fetchIfAuthorized($this->Auth->user(), $clusterId, array('edit', 'delete'), false, false);
+        $cluster = $this->GalaxyElement->GalaxyCluster->fetchIfAuthorized($user, $clusterId, array('edit', 'delete'), false, false);
         $canModify = !empty($cluster['authorized']);
-        $canModify = true;
         $this->set('canModify', $canModify);
-        if ($filters['context'] == 'JSONView') {
+        if ($filters['context'] === 'JSONView') {
             $expanded = $this->GalaxyElement->getExpandedJSONFromElements($elements);
             $this->set('JSONElements', $expanded);
         }
-        if ($this->request->is('ajax')) {
-            $this->layout = false;
-            $this->render('ajax/index');
-        }
+        $this->layout = false;
+        $this->render('ajax/index');
     }
 
     public function delete($elementId)

app/Controller/LogsController.php

@@ -1,7 +1,9 @@
 <?php
-
 App::uses('AppController', 'Controller');
 
+/**
+ * @property Log $Log
+ */
 class LogsController extends AppController
 {
     public $components = array(
@@ -23,23 +25,24 @@ class LogsController extends AppController
         parent::beforeFilter();
 
         // No need for CSRF tokens for a search
-        if ('admin_search' == $this->request->params['action']) {
+        if ('admin_search' === $this->request->params['action']) {
             $this->Security->csrfCheck = false;
         }
     }
 
-    public function admin_index()
+    public function index()
     {
         $paramArray = array('id', 'title', 'created', 'model', 'model_id', 'action', 'user_id', 'change', 'email', 'org', 'description', 'ip');
         $filterData = array(
             'request' => $this->request,
-            'named_params' => $this->params['named'],
+            'named_params' => $this->request->params['named'],
             'paramArray' => $paramArray,
             'ordered_url_params' => func_get_args()
         );
         $exception = false;
         $filters = $this->_harvestParameters($filterData, $exception);
         unset($filterData);
+
         if ($this->_isRest()) {
             if ($filters === false) {
                 return $exception;
@@ -71,8 +74,14 @@ class LogsController extends AppController
                 }
             }
             if (!$this->_isSiteAdmin()) {
-                $orgRestriction = $this->Auth->user('Organisation')['name'];
-                $conditions['AND']['Log.org'] = $orgRestriction;
+                if ($this->_isAdmin()) {
+                    // ORG admins can see their own org info
+                    $orgRestriction = $this->Auth->user('Organisation')['name'];
+                    $conditions['Log.org'] = $orgRestriction;
+                } else {
+                    // users can see their own info
+                    $conditions['Log.user_id'] = $this->Auth->user('id');
+                }
             }
             $params = array(
                 'conditions' => $conditions,
@@ -86,30 +95,42 @@ class LogsController extends AppController
             }
             $log_entries = $this->Log->find('all', $params);
             return $this->RestResponse->viewData($log_entries, 'json');
-        } else {
-            $this->set('isSearch', 0);
-            $this->recursive = 0;
-            $validFilters = $this->Log->logMeta;
-            if (!$this->_isSiteAdmin()) {
-                $orgRestriction = $this->Auth->user('Organisation')['name'];
-                $conditions['Log.org'] = $orgRestriction;
-                $this->paginate['conditions'] = $conditions;
-            } else {
-                $validFilters = array_merge_recursive($validFilters, $this->Log->logMetaAdmin);
-            }
-            if (isset($this->params['named']['filter']) && in_array($this->params['named']['filter'], array_keys($validFilters))) {
-                $this->paginate['conditions']['Log.action'] = $validFilters[$this->params['named']['filter']]['values'];
-            }
-            foreach ($filters as $key => $value) {
-                if ($key === 'created') {
-                    $key = 'created >=';
-                }
-                $this->paginate['conditions']["Log.$key"] = $value;
-            }
-            $this->set('validFilters', $validFilters);
-            $this->set('filter', isset($this->params['named']['filter']) ? $this->params['named']['filter'] : false);
-            $this->set('list', $this->paginate());
         }
+
+        $this->set('isSearch', 0);
+        $this->recursive = 0;
+        $validFilters = $this->Log->logMeta;
+        if ($this->_isSiteAdmin()) {
+            $validFilters = array_merge_recursive($validFilters, $this->Log->logMetaAdmin);
+        }
+        else if (!$this->_isSiteAdmin() && $this->_isAdmin()) {
+            // ORG admins can see their own org info
+            $orgRestriction = $this->Auth->user('Organisation')['name'];
+            $conditions['Log.org'] = $orgRestriction;
+            $this->paginate['conditions'] = $conditions;
+        } else {
+            // users can see their own info
+            $conditions['Log.email'] = $this->Auth->user('email');
+            $this->paginate['conditions'] = $conditions;
+        }
+        if (isset($this->params['named']['filter']) && in_array($this->params['named']['filter'], array_keys($validFilters))) {
+            $this->paginate['conditions']['Log.action'] = $validFilters[$this->params['named']['filter']]['values'];
+        }
+        foreach ($filters as $key => $value) {
+            if ($key === 'created') {
+                $key = 'created >=';
+            }
+            $this->paginate['conditions']["Log.$key"] = $value;
+        }
+        $this->set('validFilters', $validFilters);
+        $this->set('filter', isset($this->params['named']['filter']) ? $this->params['named']['filter'] : false);
+        $this->set('list', $this->paginate());
+    }
+
+    public function admin_index()
+    {
+        $this->view = 'index';
+        return $this->index();
     }
 
     // Shows a minimalistic history for the currently selected event
@@ -313,7 +334,7 @@ class LogsController extends AppController
                     }
 
                     // set the same view as the index page
-                    $this->render('admin_index');
+                    $this->render('index');
                 }
             } else {
                 // get from Session
@@ -356,7 +377,7 @@ class LogsController extends AppController
                 $this->set('list', $list);
 
                 // set the same view as the index page
-                $this->render('admin_index');
+                $this->render('index');
             }
         } else {
             // no search keyword is given, show the search form

app/Controller/NewsController.php

@@ -1,6 +1,9 @@
 <?php
-App::uses('AppController', 'Controller', 'CRUD');
+App::uses('AppController', 'Controller');
 
+/**
+ * @property News $News
+ */
 class NewsController extends AppController
 {
     public $components = array('Session', 'RequestHandler');
@@ -8,28 +11,51 @@ class NewsController extends AppController
     public $paginate = array(
         'limit' => 5,
         'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page.
-        'order' => array(
+        'order' => [
             'News.id' => 'DESC'
-        ),
+        ],
+        'contain' => [
+            'User' => ['fields' => ['User.email']],
+        ]
     );
 
     public function index()
     {
-        $this->paginate['contain'] = array('User' => array('fields' => array('User.email')));
+        $user = $this->Auth->user();
         $newsItems = $this->paginate();
 
-        $newsread = $this->Auth->user('newsread');
-        foreach ($newsItems as $key => $item) {
-            if ($item['News']['date_created'] > $newsread) {
-                $newsItems[$key]['News']['new'] = true;
-            } else {
-                $newsItems[$key]['News']['new'] = false;
+        $newsread = $user['newsread'];
+        $hasUnreadNews = false;
+        foreach ($newsItems as &$item) {
+            $isNew = $item['News']['date_created'] > $newsread;
+            $item['News']['new'] = $isNew;
+            if ($isNew) {
+                $hasUnreadNews = true;
             }
         }
         $this->set('newsItems', $newsItems);
+        $this->set('hasUnreadNews', $hasUnreadNews);
 
-        $this->loadModel('User');
-        $this->User->updateField($this->Auth->user(), 'newsread', time());
+        if ($hasUnreadNews) {
+            $homepage = $this->User->UserSetting->getValueForUser($user['id'], 'homepage');
+            if (!empty($homepage)) {
+                $this->set('homepage', $homepage);
+            } else {
+                $this->set('homepage', "{$this->baseurl}/events/index");
+            }
+
+            $this->User->updateField($user, 'newsread', time());
+        }
+    }
+
+    public function admin_index()
+    {
+        $user = $this->Auth->user();
+        $this->paginate['limit'] = 25;
+        $newsItems = $this->paginate();
+
+        $this->set('newsItems', $newsItems);
+        $this->set('user', $user);
     }
 
     public function add()
@@ -74,7 +100,6 @@ class NewsController extends AppController
 
     public function delete($id)
     {
-        $this->defaultModel = 'News';
         $this->CRUD->delete($id);
         if ($this->IndexFilter->isRest()) {
             return $this->restResponsePayload;

app/Controller/ObjectTemplatesController.php

@@ -22,9 +22,10 @@ class ObjectTemplatesController extends AppController
     public function beforeFilter()
     {
         parent::beforeFilter();
-        if (in_array($this->request->action, ['objectMetaChoice', 'objectChoice'], true)) {
+        if (in_array($this->request->action, ['objectMetaChoice', 'objectChoice', 'possibleObjectTemplates'], true)) {
             $this->Security->doNotGenerateToken = true;
         }
+        $this->Security->unlockedActions[] = 'possibleObjectTemplates';
     }
 
     public function objectMetaChoice($eventId)
@@ -162,16 +163,6 @@ class ObjectTemplatesController extends AppController
         $this->redirect($this->referer());
     }
 
-    public function viewElements($id, $context = 'all')
-    {
-        $elements = $this->ObjectTemplate->ObjectTemplateElement->find('all', array(
-            'conditions' => array('ObjectTemplateElement.object_template_id' => $id)
-        ));
-        $this->set('list', $elements);
-        $this->layout = false;
-        $this->render('ajax/view_elements');
-    }
-
     public function index($all = false)
     {
         $passedArgsArray = array();
@@ -183,11 +174,12 @@ class ObjectTemplatesController extends AppController
             $this->set('all', true);
         }
         if (!empty($this->params['named']['searchall'])) {
+            $searchTerm = '%' . strtolower($this->request->params['named']['searchall']) . '%';
             $this->paginate['conditions']['AND']['OR'] = array(
-                'ObjectTemplate.uuid LIKE' => '%' . strtolower($this->params['named']['searchall']) . '%',
-                'LOWER(ObjectTemplate.name) LIKE' => '%' . strtolower($this->params['named']['searchall']) . '%',
-                'ObjectTemplate.meta-category LIKE' => '%' . strtolower($this->params['named']['searchall']) . '%',
-                'LOWER(ObjectTemplate.description) LIKE' => '%' . strtolower($this->params['named']['searchall']) . '%'
+                'ObjectTemplate.uuid LIKE' => $searchTerm,
+                'LOWER(ObjectTemplate.name) LIKE' => $searchTerm,
+                'ObjectTemplate.meta-category LIKE' => $searchTerm,
+                'LOWER(ObjectTemplate.description) LIKE' => $searchTerm,
             );
         }
         if ($this->_isRest()) {
@@ -196,11 +188,11 @@ class ObjectTemplatesController extends AppController
             unset($rules['order']);
             $objectTemplates = $this->ObjectTemplate->find('all', $rules);
             return $this->RestResponse->viewData($objectTemplates, $this->response->type());
-        } else {
-            $this->paginate['order'] = array('ObjectTemplate.name' => 'ASC');
-            $objectTemplates = $this->paginate();
-            $this->set('list', $objectTemplates);
         }
+
+        $this->paginate['order'] = array('ObjectTemplate.name' => 'ASC');
+        $objectTemplates = $this->paginate();
+        $this->set('list', $objectTemplates);
         $this->set('passedArgs', json_encode($passedArgs));
         $this->set('passedArgsArray', $passedArgsArray);
     }
@@ -315,4 +307,28 @@ class ObjectTemplatesController extends AppController
         }
         return $this->RestResponse->viewData($template, $this->response->type());
     }
+
+    public function possibleObjectTemplates()
+    {
+        session_abort();
+        $this->request->allowMethod(['post']);
+
+        $attributeTypes = $this->request->data['attributeTypes'];
+        $templates = $this->ObjectTemplate->fetchPossibleTemplatesBasedOnTypes($attributeTypes)['templates'];
+
+        $results = [];
+        foreach ($templates as $template) {
+            $template = $template['ObjectTemplate'];
+            if ($template['compatibility'] === true && empty($template['invalidTypes'])) {
+                $results[] = [
+                    'id' => $template['id'],
+                    'name' => $template['name'],
+                    'description' => $template['description'],
+                    'meta-category' => $template['meta-category'],
+                ];
+            }
+        }
+
+        return $this->RestResponse->viewData($results, 'json');
+    }
 }

app/Controller/ObjectsController.php

@@ -60,33 +60,6 @@ class ObjectsController extends AppController
             $sgs = $this->MispObject->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', false, array_keys($sharing_groups));
             $this->set('sharing_groups', $sgs);
         }
-        $multiple_template_elements = Hash::extract($template['ObjectTemplateElement'], sprintf('{n}[multiple=true]'));
-        $multiple_attribute_allowed = array();
-        foreach ($multiple_template_elements as $template_element) {
-            $relation_type = $template_element['object_relation'] . ':' . $template_element['type'];
-            $multiple_attribute_allowed[$relation_type] = true;
-        }
-        $this->set('multiple_attribute_allowed', $multiple_attribute_allowed);
-        // try to fetch similar objects
-        $cur_attrs = Hash::extract($this->request->data, 'Attribute.{n}.value');
-        $conditions = array(
-            'event_id' => $event_id,
-            'value1' => $cur_attrs,
-            'object_id !=' => '0'
-        );
-        $similar_objects = $this->MispObject->Attribute->find('all', array(
-            'conditions' => $conditions,
-            'recursive' => -1,
-            'fields' => 'object_id, count(object_id) as similarity_amount',
-            'group' => 'object_id',
-            'order' => 'similarity_amount DESC'
-        ));
-        $similar_object_ids = array();
-        $similar_object_similarity_amount = array();
-        foreach ($similar_objects as $obj) {
-            $similar_object_ids[] = $obj['Attribute']['object_id'];
-            $similar_object_similarity_amount[$obj['Attribute']['object_id']] = $obj[0]['similarity_amount'];
-        }
 
         if (isset($this->request->data['Attribute'])) {
             foreach ($this->request->data['Attribute'] as &$attribute) {
@@ -113,33 +86,35 @@ class ObjectsController extends AppController
             'cur_object_tmp_uuid' => $curObjectTmpUuid,
             'data' => $this->request->data
         ));
-        if (!empty($similar_object_ids)) {
-            $this->set('similar_objects_count', count($similar_object_ids));
-            $similar_object_ids = array_slice($similar_object_ids, 0, $similar_objects_display_threshold); // slice to honor the threshold
-            $similar_objects = $this->MispObject->fetchObjects($this->Auth->user(), array(
-                'conditions' => array(
-                    'Object.id' => $similar_object_ids,
-                    'Object.template_uuid' => $template['ObjectTemplate']['uuid']
-                )
-            ));
-            foreach ($similar_objects as $key => $obj) {
-                $similar_objects[$key]['Object']['similarity_amount'] = $similar_object_similarity_amount[$obj['Object']['id']]; // sorting function cannot use external variables
-            }
-            usort($similar_objects, function ($a, $b) { // fetch Object returns object sorted by IDs, force the sort by the similarity amount
-                if ($a['Object']['similarity_amount'] == $b['Object']['similarity_amount']) {
-                    return 0;
+
+        if ($action === 'add') {
+            list($similar_objects_count, $similar_objects, $simple_flattened_attribute, $simple_flattened_attribute_noval) = $this->MispObject->findSimilarObjects(
+                $this->Auth->user(),
+                $event_id,
+                $this->request->data['Attribute'],
+                $template,
+                $similar_objects_display_threshold
+            );
+            if ($similar_objects_count) {
+                $this->set('similar_objects_count', $similar_objects_count);
+                $this->set('similar_objects', $similar_objects);
+                $this->set('similar_objects_display_threshold', $similar_objects_display_threshold);
+                $this->set('simple_flattened_attribute', $simple_flattened_attribute);
+                $this->set('simple_flattened_attribute_noval', $simple_flattened_attribute_noval);
+
+                $multiple_template_elements = Hash::extract($template['ObjectTemplateElement'],'{n}[multiple=true]');
+                $multiple_attribute_allowed = array();
+                foreach ($multiple_template_elements as $template_element) {
+                    $relation_type = $template_element['object_relation'] . ':' . $template_element['type'];
+                    $multiple_attribute_allowed[$relation_type] = true;
                 }
-                return ($a['Object']['similarity_amount'] > $b['Object']['similarity_amount']) ? -1 : 1;
-            });
-            $this->set('similar_objects', $similar_objects);
-            $this->set('similar_object_similarity_amount', $similar_object_similarity_amount);
-            $this->set('similar_objects_display_threshold', $similar_objects_display_threshold);
+                $this->set('multiple_attribute_allowed', $multiple_attribute_allowed);
+            }
         }
     }
 
-
     /**
-   * Create an object using a template
+     * Create an object using a template
      * POSTing will take the input and validate it against the template
      * GETing will return the template
      */
@@ -343,10 +318,16 @@ class ObjectsController extends AppController
         $template = $this->MispObject->prepareTemplate($template);
         $element = array();
         foreach ($template['ObjectTemplateElement'] as $templateElement) {
-            if ($templateElement['object_relation'] == $object_relation) {
+            if ($templateElement['object_relation'] === $object_relation) {
                 $element = $templateElement;
+                break;
             }
         }
+
+        if (empty($element)) {
+            throw new NotFoundException(__("Object template do not contains object relation $object_relation"));
+        }
+
         $distributionData = $this->MispObject->Event->Attribute->fetchDistributionData($this->Auth->user());
         $this->layout = false;
         $this->set('distributionData', $distributionData);
@@ -431,7 +412,7 @@ class ObjectsController extends AppController
             $savedObject = array();
             if (!is_numeric($objectToSave)) {
                 $object_validation_errors = array();
-                foreach($objectToSave as $field => $field_errors) {
+                foreach ($objectToSave as $field => $field_errors) {
                     $object_validation_errors[] = sprintf('%s: %s', $field,  implode(', ', $field_errors));
                 }
                 $error_message = __('Object could not be saved.') . PHP_EOL . implode(PHP_EOL, $object_validation_errors);
@@ -458,12 +439,10 @@ class ObjectsController extends AppController
                     return $this->RestResponse->saveFailResponse('Objects', 'edit', false, $id, $this->response->type());
                 }
             } else {
-                $message = __('Object attributes saved.');
                 if ($this->request->is('ajax')) {
-                    $this->autoRender = false;
                     if (is_numeric($objectToSave)) {
                         $this->MispObject->Event->unpublishEvent($event);
-                        return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => $message)), 'status'=>200, 'type' => 'json'));
+                        return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => __('Object attributes saved.'))), 'status'=>200, 'type' => 'json'));
                     } else {
                         return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'errors' => $error_message)), 'status'=>200, 'type' => 'json'));
                     }
@@ -597,7 +576,7 @@ class ObjectsController extends AppController
         $object = $object[0];
         $result = $object['Object'][$field];
         if ($field === 'distribution') {
-            $result = $this->MispObject->shortDist[$result];
+            $this->set('shortDist', $this->MispObject->Attribute->shortDist);
         }
         $this->set('value', $result);
         $this->set('field', $field);
@@ -634,7 +613,7 @@ class ObjectsController extends AppController
             throw new NotFoundException(__('Invalid object'));
         }
         $this->layout = false;
-        if ($field == 'distribution') {
+        if ($field === 'distribution') {
             $distributionLevels = $this->MispObject->shortDist;
             unset($distributionLevels[4]);
             $this->set('distributionLevels', $distributionLevels);
@@ -732,7 +711,7 @@ class ObjectsController extends AppController
                 'fields' => array('template_uuid', 'template_version', 'id', 'event_id'),
                 'flatten' => 1,
                 'contain' => array(
-                    'Event'
+                    'Event' => ['fields' => ['id', 'user_id', 'org_id', 'orgc_id']]
                 )
             );
             // fetchObjects restrict access based on user
@@ -1137,23 +1116,8 @@ class ObjectsController extends AppController
 
         $selectedAttributes = $this->_jsonDecode($selectedAttributes);
         $res = $this->MispObject->validObjectsFromAttributeTypes($this->Auth->user(), $eventId, $selectedAttributes);
-        $potentialTemplates = $res['templates'];
-        $attributeTypes = $res['types'];
-        usort($potentialTemplates, function($a, $b) {
-            if ($a['ObjectTemplate']['id'] == $b['ObjectTemplate']['id']) {
-                return 0;
-            } else if (is_array($a['ObjectTemplate']['compatibility']) && is_array($b['ObjectTemplate']['compatibility'])) {
-                return count($a['ObjectTemplate']['compatibility']) > count($b['ObjectTemplate']['compatibility']) ? 1 : -1;
-            } else if (is_array($a['ObjectTemplate']['compatibility']) && !is_array($b['ObjectTemplate']['compatibility'])) {
-                return 1;
-            } else if (!is_array($a['ObjectTemplate']['compatibility']) && is_array($b['ObjectTemplate']['compatibility'])) {
-                return -1;
-            } else { // sort based on invalidTypes count
-                return count($a['ObjectTemplate']['invalidTypes']) > count($b['ObjectTemplate']['invalidTypes']) ? 1 : -1;
-            }
-        });
-        $this->set('potential_templates', $potentialTemplates);
-        $this->set('selected_types', $attributeTypes);
+        $this->set('potential_templates', $res['templates']);
+        $this->set('selected_types', $res['types']);
         $this->set('event_id', $eventId);
     }
 
@@ -1230,7 +1194,8 @@ class ObjectsController extends AppController
             if (empty($template)) {
                 throw new NotFoundException(__('Invalid template.'));
             }
-            $conformity_result = $this->MispObject->ObjectTemplate->checkTemplateConformityBasedOnTypes($template, $selected_attributes);
+            $attributeTypes = array_column(array_column($selected_attributes, 'Attribute'), 'type');
+            $conformity_result = $this->MispObject->ObjectTemplate->checkTemplateConformityBasedOnTypes($template, $attributeTypes);
             $skipped_attributes = 0;
             foreach ($selected_attributes as $i => $attribute) {
                 if (in_array($attribute['Attribute']['type'], $conformity_result['invalidTypes'], true)) {
@@ -1253,8 +1218,18 @@ class ObjectsController extends AppController
             ));
 
             foreach ($object_references as $i => $object_reference) {
-                $temp_object = $this->MispObject->find('first', array('id' => $object_reference['ObjectReference']['object_id'], 'recursive' => -1));
-                $temp_attribute = $this->MispObject->Attribute->find('first', array('id' => $object_reference['ObjectReference']['referenced_id'], 'recursive' => -1));
+                $temp_object = $this->MispObject->find('first', [
+                    'conditions' => [
+                        'id' => $object_reference['ObjectReference']['object_id']
+                    ],
+                    'recursive' => -1
+                ]);
+                $temp_attribute = $this->MispObject->Attribute->find('first', [
+                    'conditions' => [
+                        'id' => $object_reference['ObjectReference']['referenced_id'],
+                    ],
+                    'recursive' => -1
+                ]);
                 if (!empty($temp_object) && !empty($temp_attribute)) {
                     $temp_object = $temp_object['Object'];
                     $temp_attribute = $temp_attribute['Attribute'];
@@ -1278,6 +1253,151 @@ class ObjectsController extends AppController
         }
     }
 
+    public function createFromFreetext($eventId)
+    {
+        $this->request->allowMethod(['post']);
+
+        $event = $this->MispObject->Event->find('first', array(
+            'recursive' => -1,
+            'fields' => array('Event.id', 'Event.uuid', 'Event.orgc_id', 'Event.user_id', 'Event.publish_timestamp'),
+            'conditions' => array('Event.id' => $eventId)
+        ));
+        if (empty($event)) {
+            throw new NotFoundException(__('Invalid event.'));
+        }
+        if (!$this->__canModifyEvent($event)) {
+            throw new ForbiddenException(__('You do not have permission to do that.'));
+        }
+
+        $requestData = $this->request->data['Object'];
+        $selectedTemplateId = $requestData['selectedTemplateId'];
+        $template = $this->MispObject->ObjectTemplate->find('first', array(
+            'recursive' => -1,
+            'conditions' => array(
+                'ObjectTemplate.id' => $selectedTemplateId,
+                'ObjectTemplate.active' => true,
+            ),
+            'contain' => ['ObjectTemplateElement'],
+        ));
+        if (empty($template)) {
+            throw new NotFoundException(__('Invalid template.'));
+        }
+
+        if (isset($requestData['selectedObjectRelationMapping'])) {
+            $distribution = $requestData['distribution'];
+            $sharingGroupId = $requestData['sharing_group_id'] ?? 0;
+            $comment = $requestData['comment'];
+            if ($distribution == 4) {
+                $sg = $this->MispObject->SharingGroup->fetchSG($sharingGroupId, $this->Auth->user());
+                if (empty($sg)) {
+                    throw new NotFoundException(__('Invalid sharing group.'));
+                }
+            } else {
+                $sharingGroupId = 0;
+            }
+
+            $attributes = $this->_jsonDecode($requestData['attributes']);
+            $selectedObjectRelationMapping = $this->_jsonDecode($requestData['selectedObjectRelationMapping']);
+
+            // Attach object relation to attributes and fix tag format
+            foreach ($attributes as $k => &$attribute) {
+                $attribute['object_relation'] = $selectedObjectRelationMapping[$k];
+                if (!empty($attribute['tags'])) {
+                    $attribute['Tag'] = [];
+                    foreach (explode(",", $attribute['tags']) as $tagName) {
+                        $attribute['Tag'][] = [
+                            'name' => trim($tagName),
+                        ];
+                    }
+                    unset($attribute['tags']);
+                }
+            }
+
+            $object = [
+               'Object' => [
+                   'event_id' => $eventId,
+                   'distribution' => $distribution,
+                   'sharing_group_id' => $sharingGroupId,
+                   'comment' => $comment,
+                   'Attribute' => $attributes,
+               ],
+            ];
+
+            $object = $this->MispObject->fillObjectDataFromTemplate($object, $template);
+            $result = $this->MispObject->captureObject($object, $eventId, $this->Auth->user(), true, false, $event);
+            if ($result === true) {
+                return $this->RestResponse->saveSuccessResponse('Objects', 'Created from Attributes', $result, 'json');
+            } else {
+                $error = __('Failed to create an Object from Attributes. Error: ') . PHP_EOL . h($result);
+                return $this->RestResponse->saveFailResponse('Objects', 'Created from Attributes', false, $error, 'json');
+            }
+        } else {
+            $attributes = $this->_jsonDecode($requestData['attributes']);
+
+            $processedAttributes = [];
+            foreach ($attributes as $attribute) {
+                if ($attribute['type'] === 'ip-src/ip-dst') {
+                    $types = array('ip-src', 'ip-dst');
+                } elseif ($attribute['type'] === 'ip-src|port/ip-dst|port') {
+                    $types = array('ip-src|port', 'ip-dst|port');
+                } else {
+                    $types = [$attribute['type']];
+                }
+                foreach ($types as $type) {
+                    $attribute['type'] = $type;
+                    $processedAttributes[] = $attribute;
+                }
+            }
+
+            $attributeTypes = array_column($processedAttributes, 'type');
+            $conformityResult = $this->MispObject->ObjectTemplate->checkTemplateConformityBasedOnTypes($template, $attributeTypes);
+
+            if ($conformityResult['valid'] !== true || !empty($conformityResult['invalidTypes'])) {
+                throw new NotFoundException(__('Invalid template.'));
+            }
+
+            $objectRelations = [];
+            foreach ($template['ObjectTemplateElement'] as $templateElement) {
+                $objectRelations[$templateElement['type']][] = $templateElement;
+            }
+
+            // Attach first object_relation according to attribute type that will be considered as default
+            foreach ($processedAttributes as &$attribute) {
+                $attribute['object_relation'] = $objectRelations[$attribute['type']][0]['object_relation'];
+            }
+
+            $distributionData = $this->MispObject->Event->Attribute->fetchDistributionData($this->Auth->user());
+            $this->set('event', $event);
+            $this->set('distributionData', $distributionData);
+            $this->set('distributionLevels', $this->MispObject->Attribute->distributionLevels);
+            $this->set('template', $template);
+            $this->set('objectRelations', $objectRelations);
+            $this->set('attributes', $processedAttributes);
+
+            list($similar_objects_count, $similar_objects, $simple_flattened_attribute, $simple_flattened_attribute_noval) = $this->MispObject->findSimilarObjects(
+                $this->Auth->user(),
+                $eventId,
+                $processedAttributes,
+                $template
+            );
+            if ($similar_objects_count) {
+                $this->set('similar_objects_count', $similar_objects_count);
+                $this->set('similar_objects', $similar_objects);
+                $this->set('similar_objects_display_threshold', 15);
+                $this->set('simple_flattened_attribute', $simple_flattened_attribute);
+                $this->set('simple_flattened_attribute_noval', $simple_flattened_attribute_noval);
+
+                $multiple_template_elements = Hash::extract($template['ObjectTemplateElement'],'{n}[multiple=true]');
+                $multiple_attribute_allowed = array();
+                foreach ($multiple_template_elements as $template_element) {
+                    $relation_type = $template_element['object_relation'] . ':' . $template_element['type'];
+                    $multiple_attribute_allowed[$relation_type] = true;
+                }
+                $this->set('multiple_attribute_allowed', $multiple_attribute_allowed);
+            }
+        }
+    }
+
     private function __objectIdToConditions($id)
     {
         if (is_numeric($id)) {

app/Controller/ServersController.php

@@ -26,7 +26,7 @@ class ServersController extends AppController
                 'fields' => array('RemoteOrg.name', 'RemoteOrg.id'),
             ),
         ),
-        'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events
+        'maxLimit' => 9999,
         'order' => array(
             'Server.priority' => 'ASC'
         ),
@@ -57,6 +57,14 @@ class ServersController extends AppController
         unset($fields['authkey']);
         $fields = array_keys($fields);
 
+        $filters = $this->IndexFilter->harvestParameters(['search']);
+        $conditions = [];
+        if (!empty($filters['search'])) {
+            $strSearch = '%' . trim(strtolower($filters['search'])) . '%';
+            $conditions['OR'][]['LOWER(Server.name) LIKE'] = $strSearch;
+            $conditions['OR'][]['LOWER(Server.url) LIKE'] = $strSearch;
+        }
+
         if ($this->_isRest()) {
             $params = array(
                 'fields' => $fields,
@@ -72,12 +80,14 @@ class ServersController extends AppController
                         'fields' => array('RemoteOrg.id', 'RemoteOrg.name', 'RemoteOrg.uuid', 'RemoteOrg.nationality', 'RemoteOrg.sector', 'RemoteOrg.type'),
                     ),
                 ),
+                'conditions' => $conditions,
             );
             $servers = $this->Server->find('all', $params);
             $servers = $this->Server->attachServerCacheTimestamps($servers);
             return $this->RestResponse->viewData($servers, $this->response->type());
         } else {
             $this->paginate['fields'] = $fields;
+            $this->paginate['conditions'] = $conditions;
             $servers = $this->paginate();
             $servers = $this->Server->attachServerCacheTimestamps($servers);
             $this->set('servers', $servers);
@@ -905,30 +915,52 @@ class ServersController extends AppController
             App::uses('File', 'Utility');
             App::uses('Folder', 'Utility');
             App::uses('FileAccessTool', 'Tools');
+            App::uses('SyncTool', 'Tools');
             if (isset($server['Server'][$subm]['name'])) {
                 if ($this->request->data['Server'][$subm]['size'] != 0) {
                     if (!$this->Server->checkFilename($server['Server'][$subm]['name'])) {
                         throw new Exception(__('Filename not allowed'));
                     }
-                    $file = new File($server['Server'][$subm]['name']);
-                    $ext = $file->ext();
+
+                    if (!is_uploaded_file($server['Server'][$subm]['tmp_name'])) {
+                        throw new Exception(__('File not uploaded correctly'));
+                    }
+
+                    $ext = pathinfo($server['Server'][$subm]['name'], PATHINFO_EXTENSION);
+                    if (!in_array($ext, SyncTool::ALLOWED_CERT_FILE_EXTENSIONS)) {
+                        $this->Flash->error(__('Invalid extension.'));
+                        $this->redirect(array('action' => 'index'));
+                    }
+
                     if (!$server['Server'][$subm]['size'] > 0) {
                         $this->Flash->error(__('Incorrect extension or empty file.'));
                         $this->redirect(array('action' => 'index'));
                     }
 
-                    // read pem file data
-                    $pemData = FileAccessTool::readFromFile($server['Server'][$subm]['tmp_name'], $server['Server'][$subm]['size']);
+                    // read certificate file data
+                    $certData = FileAccessTool::readFromFile($server['Server'][$subm]['tmp_name'], $server['Server'][$subm]['size']);
                 } else {
                     return true;
                 }
             } else {
-                $pemData = base64_decode($server['Server'][$subm]);
+                $ext = 'pem';
+                $certData = base64_decode($server['Server'][$subm]);
             }
+
+            // check if the file is a valid x509 certificate
+            try {
+                $cert = openssl_x509_parse($certData);
+                if (!$cert) {
+                    throw new Exception(__('Invalid certificate.'));
+                }
+            } catch (Exception $e) {
+                $this->Flash->error(__('Invalid certificate.'));
+                $this->redirect(array('action' => 'index'));
+            }
+
             $destpath = APP . "files" . DS . "certs" . DS;
-            $dir = new Folder(APP . "files" . DS . "certs", true);
             $pemfile = new File($destpath . $id . $ins . '.' . $ext);
-            $result = $pemfile->write($pemData);
+            $result = $pemfile->write($certData);
             $s = $this->Server->read(null, $id);
             $s['Server'][$attr] = $s['Server']['id'] . $ins . '.' . $ext;
             if ($result) {
@@ -1075,6 +1107,9 @@ class ServersController extends AppController
             $this->set('correlation_metrics', $correlation_metrics);
         }
         if ($tab === 'files') {
+            if (!empty(Configure::read('Security.disable_instance_file_uploads'))) {
+                throw new MethodNotAllowedException(__('This functionality is disabled.'));
+            }
             $files = $this->Server->grabFiles();
             $this->set('files', $files);
         }
@@ -1624,6 +1659,9 @@ class ServersController extends AppController
         if (!$this->request->is('post')) {
             throw new MethodNotAllowedException();
         }
+        if (!empty(Configure::read('Security.disable_instance_file_uploads'))) {
+            throw new MethodNotAllowedException(__('Feature disabled.'));
+        }
         $validItems = $this->Server->getFileRules();
 
         // Check if there were problems with the file upload
@@ -1685,8 +1723,9 @@ class ServersController extends AppController
         if (!function_exists('getallheaders')) {
             $headers = [];
             foreach ($_SERVER as $name => $value) {
-                if (substr($name, 0, 5) === 'HTTP_') {
-                    $headers[strtolower(str_replace('_', '-', substr($name, 5)))] = $value;
+                $name = strtolower($name);
+                if (substr($name, 0, 5) === 'http_') {
+                    $headers[str_replace('_', '-', substr($name, 5))] = $value;
                 }
             }
         } else {
@@ -1719,6 +1758,7 @@ class ServersController extends AppController
         if (!$server) {
             throw new NotFoundException(__('Invalid server'));
         }
+        @session_write_close(); // close session to allow concurrent requests
         $result = $this->Server->runConnectionTest($server);
         if ($result['status'] == 1) {
             if (isset($result['info']['version']) && preg_match('/^[0-9]+\.+[0-9]+\.[0-9]+$/', $result['info']['version'])) {

app/Controller/ShadowAttributesController.php

@@ -62,8 +62,10 @@ class ShadowAttributesController extends AppController
         // If the old_id is set to anything but 0 then we're dealing with a proposed edit to an existing attribute
         if ($shadow['old_id'] != 0) {
             // Find the live attribute by the shadow attribute's uuid, so we can begin editing it
-            $this->Attribute->contain = 'Event';
-            $activeAttribute = $this->Attribute->findByUuid($shadow['uuid']);
+            $activeAttribute = $this->Attribute->find('first', [
+                'conditions' => ['Attribute.uuid' => $shadow['uuid']],
+                'contain' => ['Event'],
+            ]);
 
             // Send those away that shouldn't be able to edit this
             if (!$this->__canModifyEvent($activeAttribute)) {
@@ -76,7 +78,7 @@ class ShadowAttributesController extends AppController
             }
 
             if (isset($shadow['proposal_to_delete']) && $shadow['proposal_to_delete']) {
-                $this->Attribute->delete($activeAttribute['Attribute']['id']);
+                $this->Attribute->deleteAttribute($activeAttribute['Attribute']['id'], $this->Auth->user(), false);
             } else {
                 // Update the live attribute with the shadow data
                 $fieldsToUpdate = array('value1', 'value2', 'value', 'type', 'category', 'comment', 'to_ids', 'first_seen', 'last_seen');
@@ -597,8 +599,8 @@ class ShadowAttributesController extends AppController
     // if any of these fields is set, it will create a proposal
     public function edit($id = null)
     {
-        $existingAttribute = $this->ShadowAttribute->Event->Attribute->fetchAttributes($this->Auth->user(), array(
-            'contain' => array('Event' => array('fields' => array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.distribution', 'Event.uuid'))),
+        $existingAttribute = $this->ShadowAttribute->Attribute->fetchAttributes($this->Auth->user(), array(
+            'contain' => ['Event' => ['fields' => ['Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.distribution', 'Event.uuid', 'Event.user_id']]],
             'conditions' => $this->__attributeIdToConditions($id),
             'flatten' => 1
         ));
@@ -673,9 +675,9 @@ class ShadowAttributesController extends AppController
                     $sa = $this->ShadowAttribute->find(
                             'first',
                             array(
-                                    'conditions' => array('ShadowAttribute.id' => $this->ShadowAttribute->id),
-                                    'recursive' => -1,
-                                    'fields' => array('id', 'old_id', 'event_id', 'type', 'category', 'value', 'comment','to_ids', 'uuid', 'event_org_id', 'email', 'deleted', 'timestamp', 'first_seen', 'last_seen')
+                                'conditions' => array('ShadowAttribute.id' => $this->ShadowAttribute->id),
+                                'recursive' => -1,
+                                'fields' => array('id', 'old_id', 'event_id', 'type', 'category', 'value', 'comment','to_ids', 'uuid', 'event_org_id', 'email', 'deleted', 'timestamp', 'first_seen', 'last_seen')
                             )
                     );
                     $this->set('ShadowAttribute', $sa['ShadowAttribute']);
@@ -904,7 +906,24 @@ class ShadowAttributesController extends AppController
         }
         $params = array(
             'conditions' => $conditions,
-            'fields' => array('ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.timestamp', 'ShadowAttribute.first_seen', 'ShadowAttribute.last_seen'),
+            'fields' => array(
+                'ShadowAttribute.id',
+                'ShadowAttribute.old_id',
+                'ShadowAttribute.event_id',
+                'ShadowAttribute.type',
+                'ShadowAttribute.category',
+                'ShadowAttribute.uuid',
+                'ShadowAttribute.to_ids',
+                'ShadowAttribute.value',
+                'ShadowAttribute.comment',
+                'ShadowAttribute.org_id',
+                'ShadowAttribute.timestamp',
+                'ShadowAttribute.first_seen',
+                'ShadowAttribute.last_seen',
+                'ShadowAttribute.deleted',
+                'ShadowAttribute.proposal_to_delete',
+                'ShadowAttribute.disable_correlation'
+            ),
             'contain' => array(
                     'Event' => array(
                             'fields' => array('id', 'org_id', 'info', 'orgc_id', 'uuid'),

app/Controller/SightingsController.php

@@ -66,7 +66,8 @@ class SightingsController extends AppController
                 $filters = !empty($this->request->data['filters']) ? $this->request->data['filters'] : false;
             }
             if (!$error) {
-                $result = $this->Sighting->saveSightings($id, $values, $timestamp, $this->Auth->user(), $type, $source, false, true, false, $filters);
+                $publish_sighting = !empty(Configure::read('Sightings_enable_realtime_publish'));
+                $result = $this->Sighting->saveSightings($id, $values, $timestamp, $this->Auth->user(), $type, $source, false, $publish_sighting, false, $filters);
             }
             if (!is_numeric($result)) {
                 $error = $result;

app/Controller/TagsController.php

@@ -60,7 +60,7 @@ class TagsController extends AppController
             $this->paginate['conditions']['AND'][] = ['LOWER(Tag.name) LIKE' => '%' . strtolower($passedArgsArray['searchall']) . '%'];
         }
         foreach (['name', 'filter', 'search'] as $f) {
-            if (!empty($passedArgsArray['name'])) {
+            if (!empty($passedArgsArray[$f])) {
                 $this->paginate['conditions']['AND'][] = ['LOWER(Tag.name)' => strtolower($passedArgsArray[$f])];
             }
         }
@@ -367,6 +367,9 @@ class TagsController extends AppController
         // Remove galaxy tags
         $event = $this->Tag->removeGalaxyClusterTags($user, $event);
 
+        $highlightedTags = $this->Taxonomy->getHighlightedTags($this->Taxonomy->getHighlightedTaxonomies(), $event['EventTag']);
+        $this->set('highlightedTaxonomies', $highlightedTags);
+
         $this->set('tags', $event['EventTag']);
         $this->set('missingTaxonomies', $this->Tag->EventTag->Event->missingTaxonomies($event));
         $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($event['EventTag']);

app/Controller/TaxiiServersController.php

@@ -0,0 +1,231 @@
+<?php
+App::uses('AppController', 'Controller');
+
+class TaxiiServersController extends AppController
+{
+    public $components = array('Session', 'RequestHandler');
+
+
+    public function beforeFilter()
+    {
+        // No need for CSRF tokens for a search
+        if ('getRoot' == $this->request->params['action'] || 'getCollections' == $this->request->params['action']) {
+            $this->Security->csrfCheck = false;
+        }
+        if ($this->request->params['action'] === 'add' || $this->request->params['action'] === 'edit') {
+            $this->Security->unlockedFields = ['api_root', 'collection'];
+        }
+        parent::beforeFilter();
+    }
+
+    public $paginate = array(
+            'limit' => 60,
+            'maxLimit' => 9999
+    );
+
+    public function index()
+    {
+        $params = [
+            'filters' => ['name', 'url', 'uuid'],
+            'quickFilters' => ['name']
+        ];
+        $this->CRUD->index($params);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }
+        $this->set('menuData', array('menuList' => 'sync', 'menuItem' => 'list_taxii'));
+    }
+
+    public function add()
+    {
+        $params = [];
+        $this->CRUD->add($params);
+        if ($this->restResponsePayload) {
+            return $this->restResponsePayload;
+        }
+        $dropdownData = [];
+        $this->set(compact('dropdownData'));
+        $this->set('menuData', array('menuList' => 'sync', 'menuItem' => 'add_taxii'));
+    }
+
+    public function edit($id)
+    {
+        $this->set('menuData', array('menuList' => 'sync', 'menuItem' => 'edit_taxii'));
+        $this->set('id', $id);
+        $params = [];
+        $this->CRUD->edit($id, $params);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }
+        $dropdownData = [];
+        $this->set(compact('dropdownData'));
+        $this->render('add');
+    }
+
+    public function delete($id)
+    {
+        $this->CRUD->delete($id);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }
+    }
+
+    public function view($id)
+    {
+        $this->set('menuData', ['menuList' => 'sync', 'menuItem' => 'view_taxii']);
+        $this->CRUD->view($id);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }
+        $this->set('id', $id);
+    }
+
+    public function push($id)
+    {
+        $this->set('menuData', ['menuList' => 'sync', 'menuItem' => 'push_taxii']);
+        $taxii_server = $this->TaxiiServer->find('first', [
+            'recursive' => -1,
+            'conditions' => ['TaxiiServer.id' => $id]
+        ]);
+        if (empty($taxii_server)) {
+            throw new NotFoundException(__('Invalid Taxii Server ID provided.'));
+        }
+
+        if ($this->request->is('post')) {
+            $result = $this->TaxiiServer->pushRouter($taxii_server['TaxiiServer']['id'],  $this->Auth->user());
+            $message = __('Taxii push initiated.');
+            if ($this->_isRest()) {
+                return $this->RestResponse->saveSuccessResponse('TaxiiServers', 'push', $id, false, $message);
+            } else {
+                $this->Flash->success($message);
+                $this->redirect($this->referer());
+            }
+        } else {
+            $this->set('id', $taxii_server['TaxiiServer']['id']);
+            $this->set('title', __('Push data to TAXII server'));
+            $this->set('question', __('Are you sure you want to Push data as configured in the filters to the TAXII server?'));
+            $this->set('actionName', __('Push'));
+            $this->layout = 'ajax';
+            $this->render('/genericTemplates/confirm');
+        }
+    }
+
+    public function getRoot()
+    {
+        if (empty($this->request->data['baseurl'])) {
+            return $this->RestResponse->saveFailResponse(
+                'TaxiiServers', 'getRoot', null, __('No baseurl set.'), $this->response->type()
+            );
+        } else {
+            $this->request->data['uri'] = '/taxii2/';
+            $result = $this->TaxiiServer->queryInstance(
+                [
+                    'TaxiiServer' => $this->request->data,
+                    'type' => 'get'
+                ]
+            );
+            if (is_array($result)) {
+                $results = [];
+                foreach ($result['api_roots'] as $api_root) {
+                    $api_root = explode('/', trim($api_root, '/'));
+                    $api_root = end($api_root);
+                    $results[$api_root] = $this->request->data['baseurl'] . '/' . $api_root . '/';
+                }
+                return $this->RestResponse->viewData($results, 'json');
+            } else {
+                return $this->RestResponse->saveFailResponse(
+                    'TaxiiServers', 'getRoot', null, $result, $this->response->type()
+                );  
+            }
+        }
+    }
+
+    public function getCollections()
+    {
+        if (empty($this->request->data['baseurl'])) {
+            return $this->RestResponse->saveFailResponse(
+                'TaxiiServers', 'getCollections', null, __('No baseurl set.'), $this->response->type()
+            );
+        }
+        if (empty($this->request->data['api_root'])) {
+            return $this->RestResponse->saveFailResponse(
+                'TaxiiServers', 'getCollections', null, __('No api_root set.'), $this->response->type()
+            );
+        }
+        $this->request->data['uri'] = '/' . $this->request->data['api_root'] . '/collections/';
+        $result = $this->TaxiiServer->queryInstance(
+            [
+                'TaxiiServer' => $this->request->data,
+                'type' => 'get'
+            ]
+        );
+        if (is_array($result)) {
+            $results = [];
+            foreach ($result['collections'] as $collection) {
+                if (!empty($collection['can_write'])) {
+                    $versions = '';
+                    if (!empty($collection['media_types'])) {
+                        if (!is_array(($collection['media_types']))) {
+                            $collection['media_types'] = [$collection['media_types']];
+                        }
+                        $versions = [];
+                        foreach ($collection['media_types'] as $media_type) {
+                            $media_type = explode('=', $media_type);
+                            $media_type = end($media_type);
+                            $versions[$media_type] = true;
+                        }
+                        $versions = implode(', ', array_keys($versions));
+                    }
+                    $text = (empty($versions) ? '' : '[' . $versions . '] ') . $collection['title'];
+                    $results[$collection['id']] = $text;
+                }
+            }
+            return $this->RestResponse->viewData($results, 'json');
+        } else {
+            return $this->RestResponse->saveFailResponse(
+                'TaxiiServers', 'getRoot', null, $result, $this->response->type()
+            );  
+        }
+    }
+
+    public function collectionsIndex($id)
+    {
+        $result = $this->TaxiiServer->getCollections($id);
+        if ($this->_isRest()) {
+            return $this->RestResponse->viewData($result, $this->response->type());
+        } else {
+            App::uses('CustomPaginationTool', 'Tools');
+            $customPagination = new CustomPaginationTool();
+            $customPagination->truncateAndPaginate($result, $this->params, false, true);
+            $this->set('data', $result);
+            $this->set('id', $id);
+            $this->set('menuData', array('menuList' => 'sync', 'menuItem' => 'list_taxii_collections'));
+        }
+
+    }
+
+    public function objectsIndex($id, $collection_id, $next = null)
+    {
+        $result = $this->TaxiiServer->getObjects($id, $collection_id, $next);
+        if ($this->_isRest()) {
+            return $this->RestResponse->viewData($result, $this->response->type());
+        } else {
+            $this->set('data', $result['objects']);
+            $this->set('more', $result['more']);
+            $this->set('next', isset($result['next']) ? $result['next'] : null);
+            $this->set('id', $id);
+            $this->set('collection_id', $collection_id);
+            $this->set('menuData', array('menuList' => 'sync', 'menuItem' => 'list_taxii_collection_objects'));
+        }
+    }
+
+    public function objectView($server_id, $collection_id, $id)
+    {
+        $result = $this->TaxiiServer->getObject($id, $server_id, $collection_id);
+        $result = json_encode($result, JSON_PRETTY_PRINT);
+        $this->layout = false;
+        $this->set('title', h($id));
+        $this->set('json', $result);
+        $this->render('/genericTemplates/display');
+    }
+}

app/Controller/TaxonomiesController.php

@@ -59,7 +59,7 @@ class TaxonomiesController extends AppController
 
     public function view($id)
     {
-        $taxonomy = $this->Taxonomy->getTaxonomy($id, ['full' => $this->_isRest()]);
+        $taxonomy = $this->Taxonomy->getTaxonomy($id, $this->_isRest());
         if (empty($taxonomy)) {
             throw new NotFoundException(__('Taxonomy not found.'));
         }
@@ -498,6 +498,32 @@ class TaxonomiesController extends AppController
         $this->render('ajax/toggle_required');
     }
 
+    public function toggleHighlighted($id)
+    {
+        $taxonomy = $this->Taxonomy->find('first', array(
+            'recursive' => -1,
+            'conditions' => array('Taxonomy.id' => $id)
+        ));
+        if (empty($taxonomy)) {
+            return $this->RestResponse->saveFailResponse('Taxonomy', 'toggleHighlighted', $id, 'Invalid Taxonomy', $this->response->type());
+        }
+        if ($this->request->is('post')) {
+            $taxonomy['Taxonomy']['highlighted'] = $this->request->data['Taxonomy']['highlighted'];
+            $result = $this->Taxonomy->save($taxonomy);
+            if ($result) {
+                return $this->RestResponse->saveSuccessResponse('Taxonomy', 'toggleHighlighted', $id, $this->response->type());
+            } else {
+                return $this->RestResponse->saveFailResponse('Taxonomy', 'toggleHighlighted', $id, $this->validationError, $this->response->type());
+            }
+        }
+
+        $this->set('highlighted', !$taxonomy['Taxonomy']['highlighted']);
+        $this->set('id', $id);
+        $this->autoRender = false;
+        $this->layout = false;
+        $this->render('ajax/toggle_highlighted');
+    }
+
     /**
      * @param string $action
      * @param int $modelId

app/Controller/TemplatesController.php

@@ -18,7 +18,7 @@ class TemplatesController extends AppController
     public function beforeFilter()
     { // TODO REMOVE
         parent::beforeFilter();
-        $this->Security->unlockedActions = array('uploadFile', 'deleteTemporaryFile');
+        $this->Security->unlockedActions = array('uploadFile', 'deleteTemporaryFile', 'saveElementSorting');
     }
 
     public function index()
@@ -188,7 +188,7 @@ class TemplatesController extends AppController
         $this->request->onlyAllow('ajax');
         $orderedElements = $this->request->data;
         foreach ($orderedElements as $key => $e) {
-            $orderedElements[$key] = ltrim($e, 'id_');
+            $orderedElements[$key] = (int)ltrim($e, 'id_');
         }
         $extractedIds = array();
         foreach ($orderedElements as $element) {

app/Controller/UsersController.php

@@ -1,5 +1,5 @@
 <?php
-App::uses('AppController', 'Controller');
+App::uses('AppController', 'Controller', 'OTPHP\TOTP');
 
 /**
  * @property User $User
@@ -29,7 +29,11 @@ class UsersController extends AppController
         parent::beforeFilter();
 
         // what pages are allowed for non-logged-in users
-        $allowedActions = array('login', 'logout', 'getGpgPublicKey');
+        $allowedActions = array('login', 'logout', 'getGpgPublicKey', 'logout401', 'otp');
+        if (!empty(Configure::read('Security.allow_password_forgotten'))) {
+            $allowedActions[] = 'forgot';
+            $allowedActions[] = 'password_reset';
+        }
         if(!empty(Configure::read('Security.email_otp_enabled'))) {
           $allowedActions[] = 'email_otp';
         }
@@ -88,6 +92,7 @@ class UsersController extends AppController
             unset($user['User']['authkey']);
         }
         $user['User']['password'] = '*****';
+        $user['User']['totp'] = '*****';
         $temp = [];
         $objectsToInclude = array('User', 'Role', 'UserSetting', 'Organisation');
         foreach ($objectsToInclude as $objectToInclude) {
@@ -117,8 +122,14 @@ class UsersController extends AppController
         return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => 'Something went wrong, please try again later.')), 'status'=>200, 'type' => 'json'));
     }
 
-    public function unsubscribe($code)
+    public function unsubscribe($code, $type = null)
     {
+        if ($type === null) {
+            $type = 'autoalert';
+        } else if (!in_array($type, ['autoalert', 'notification_daily', 'notification_weekly', 'notification_monthly'], true)) {
+            throw new NotFoundException("Invalid type $type.");
+        }
+
         $user = $this->Auth->user();
 
         if (!hash_equals($this->User->unsubscribeCode($user), rtrim($code, '.'))) {
@@ -126,11 +137,11 @@ class UsersController extends AppController
             $this->redirect(['action' => 'view', 'me']);
         }
 
-        if ($user['autoalert']) {
-            $this->User->updateField($this->Auth->user(), 'autoalert', false);
-            $this->Flash->success(__('Successfully unsubscribed from event alert.'));
+        if ($user[$type]) {
+            $this->User->updateField($user, $type, false);
+            $this->Flash->success(__('Successfully unsubscribed from notification.'));
         } else {
-            $this->Flash->info(__('Already unsubscribed from event alert.'));
+            $this->Flash->info(__('Already unsubscribed from notification.'));
         }
         $this->redirect(['action' => 'view', 'me']);
     }
@@ -255,6 +266,77 @@ class UsersController extends AppController
         $this->set('canFetchPgpKey', $this->__canFetchPgpKey());
     }
 
+    private function __pw_change($user, $source, &$abortPost, $token = false)
+    {
+        if (!isset($this->request->data['User'])) {
+            $this->request->data = array('User' => $this->request->data);
+        }
+        if (Configure::read('Security.require_password_confirmation')) {
+            if (!empty($this->request->data['User']['current_password'])) {
+                $hashed = $this->User->verifyPassword($this->Auth->user('id'), $this->request->data['User']['current_password']);
+                if (!$hashed) {
+                    $message = __('Invalid password. Please enter your current password to continue.');
+                    if ($this->_isRest()) {
+                        return $this->RestResponse->saveFailResponse('Users', $source, false, $message, $this->response->type());
+                    }
+                    $abortPost = true;
+                    $this->Flash->error($message);
+                }
+                unset($this->request->data['User']['current_password']);
+            } else if (!$this->_isRest()) {
+                $message = __('Please enter your current password to continue.');
+                if ($this->_isRest()) {
+                    return $this->RestResponse->saveFailResponse('Users', $source, false, $message, $this->response->type());
+                }
+                $abortPost = true;
+                $this->Flash->info($message);
+            }
+        }
+        $hashed = $this->User->verifyPassword($this->Auth->user('id'), $this->request->data['User']['password']);
+        if ($hashed) {
+            $message = __('Submitted new password cannot be the same as the current one');
+            $abortPost = true;
+        }
+        if (!$abortPost) {
+            // What fields should be saved (allowed to be saved)
+            $user['User']['change_pw'] = 0;
+            $user['User']['password'] = $this->request->data['User']['password'];
+            if ($this->_isRest()) {
+                $user['User']['confirm_password'] = $this->request->data['User']['password'];
+            } else {
+                $user['User']['confirm_password'] = $this->request->data['User']['confirm_password'];
+            }
+            $temp = $user['User']['password'];
+            // Save the data
+            if ($this->User->save($user)) {
+                if ($token) {
+                    $this->User->purgeForgetToken($token);
+                }
+                $message = __('Password Changed.');
+                // log as System if the reset comes from an unauthed user using password_reset tokens
+                $logUser = empty($this->Auth->user()) ? 'SYSTEM' : $this->Auth->user();
+                $this->User->extralog($logUser, $source, null, null, $user);
+                if ($this->_isRest()) {
+                    return $this->RestResponse->saveSuccessResponse('User', $source, false, $this->response->type(), $message);
+                }
+                $this->Flash->success($message);
+                $this->redirect(array('action' => 'view', $user['User']['id']));
+            } else {
+                $message = __('The password could not be updated. Make sure you meet the minimum password length / complexity requirements.');
+                if ($this->_isRest()) {
+                    return $this->RestResponse->saveFailResponse('Users', $source, false, $message, $this->response->type());
+                }
+                $this->Flash->error($message);
+            }
+        } else {
+            if ($this->_isRest()) {
+                return $this->RestResponse->saveFailResponse('Users', $source, false, $message, $this->response->type());
+            } else {
+                $this->Flash->error($message);
+            }
+        }
+    }
+
     public function change_pw()
     {
         $id = $this->Auth->user('id');
@@ -263,69 +345,8 @@ class UsersController extends AppController
             'recursive' => -1
         ));
         if ($this->request->is('post') || $this->request->is('put')) {
-            if (!isset($this->request->data['User'])) {
-                $this->request->data = array('User' => $this->request->data);
-            }
             $abortPost = false;
-            if (Configure::read('Security.require_password_confirmation')) {
-                if (!empty($this->request->data['User']['current_password'])) {
-                    $hashed = $this->User->verifyPassword($this->Auth->user('id'), $this->request->data['User']['current_password']);
-                    if (!$hashed) {
-                        $message = __('Invalid password. Please enter your current password to continue.');
-                        if ($this->_isRest()) {
-                            return $this->RestResponse->saveFailResponse('Users', 'change_pw', false, $message, $this->response->type());
-                        }
-                        $abortPost = true;
-                        $this->Flash->error($message);
-                    }
-                    unset($this->request->data['User']['current_password']);
-                } else if (!$this->_isRest()) {
-                    $message = __('Please enter your current password to continue.');
-                    if ($this->_isRest()) {
-                        return $this->RestResponse->saveFailResponse('Users', 'change_pw', false, $message, $this->response->type());
-                    }
-                    $abortPost = true;
-                    $this->Flash->info($message);
-                }
-            }
-            $hashed = $this->User->verifyPassword($this->Auth->user('id'), $this->request->data['User']['password']);
-            if ($hashed) {
-                $message = __('Submitted new password cannot be the same as the current one');
-                $abortPost = true;
-            }
-            if (!$abortPost) {
-                // What fields should be saved (allowed to be saved)
-                $user['User']['change_pw'] = 0;
-                $user['User']['password'] = $this->request->data['User']['password'];
-                if ($this->_isRest()) {
-                    $user['User']['confirm_password'] = $this->request->data['User']['password'];
-                } else {
-                    $user['User']['confirm_password'] = $this->request->data['User']['confirm_password'];
-                }
-                $temp = $user['User']['password'];
-                // Save the data
-                if ($this->User->save($user)) {
-                    $message = __('Password Changed.');
-                    $this->User->extralog($this->Auth->user(), "change_pw", null, null, $user);
-                    if ($this->_isRest()) {
-                        return $this->RestResponse->saveSuccessResponse('User', 'change_pw', false, $this->response->type(), $message);
-                    }
-                    $this->Flash->success($message);
-                    $this->redirect(array('action' => 'view', $id));
-                } else {
-                    $message = __('The password could not be updated. Make sure you meet the minimum password length / complexity requirements.');
-                    if ($this->_isRest()) {
-                        return $this->RestResponse->saveFailResponse('Users', 'change_pw', false, $message, $this->response->type());
-                    }
-                    $this->Flash->error($message);
-                }
-            } else {
-                if ($this->_isRest()) {
-                    return $this->RestResponse->saveFailResponse('Users', 'change_pw', false, $message, $this->response->type());
-                } else {
-                    $this->Flash->error($message);
-                }
-            }
+            return $this->__pw_change($user, 'change_pw', $abortPost);
         }
         if ($this->_isRest()) {
             return $this->RestResponse->describe('Users', 'change_pw', false, $this->response->type());
@@ -410,6 +431,12 @@ class UsersController extends AppController
                             $this->paginate['conditions']['AND'][] = $test;
                         }
                     }
+                } elseif ("inactive" == $searchTerm) {
+                    if ($v == "1") {
+                        $this->paginate['conditions']['AND'][] = array('User.last_login <' =>  time() - 60*60*24*30);  // older than a month 
+                        $this->paginate['conditions']['AND'][] = array('User.current_login <' =>  time() - 60*60*24*30);  // older than a month 
+                        $this->paginate['conditions']['AND'][] = array('User.last_api_access <' =>  time() - 60*60*24*30);  // older than a month 
+                    }
                 }
                 $passedArgsArray[$searchTerm] = $v;
             }
@@ -487,7 +514,7 @@ class UsersController extends AppController
     public function admin_filterUserIndex()
     {
         $passedArgsArray = array();
-        $booleanFields = array('autoalert', 'contactalert', 'termsaccepted', 'disabled');
+        $booleanFields = array('autoalert', 'contactalert', 'termsaccepted', 'disabled', 'inactive');
         $textFields = array('role', 'email');
         if (empty(Configure::read('Security.advanced_authkeys'))) {
             $textFields[] = 'authkey';
@@ -559,7 +586,7 @@ class UsersController extends AppController
     {
         $user = $this->User->find('first', array(
             'recursive' => -1,
-            'conditions' => $this->__adminFetchConditions($id),
+            'conditions' => $this->__adminFetchConditions($id, $edit=False),
             'contain' => [
                 'UserSetting',
                 'Role',
@@ -582,17 +609,7 @@ class UsersController extends AppController
             unset($user['User']['authkey']);
         }
         if ($this->_isRest()) {
-            $user['User']['password'] = '*****';
-            $temp = array();
-            foreach ($user['UserSetting'] as $v) {
-                $temp[$v['setting']] = $v['value'];
-            }
-            $user['UserSetting'] = $temp;
-            return $this->RestResponse->viewData(array(
-                'User' => $user['User'],
-                'Role' => $user['Role'],
-                'UserSetting' => $user['UserSetting']
-            ), $this->response->type());
+            return $this->RestResponse->viewData($this->__massageUserObject($user), $this->response->type());
         }
         $this->set('user', $user);
 
@@ -848,9 +865,6 @@ class UsersController extends AppController
             // MISP automatically chooses the first available option for the user as the selected setting (usually user)
             // Org admin is downgraded to a user
             // Now we make an exception for the already assigned role, both in the form and the actual edit.
-            if (!empty($userToEdit['Role']['perm_site_admin'])) {
-                throw new NotFoundException(__('Invalid user'));
-            }
             $allowedRole = $userToEdit['User']['role_id'];
             $params = array('conditions' => array(
                     'OR' => array(
@@ -1080,27 +1094,36 @@ class UsersController extends AppController
 
     public function admin_delete($id = null)
     {
-        $this->request->allowMethod(['post', 'delete']);
-
-        $user = $this->User->find('first', array(
-            'conditions' => $this->__adminFetchConditions($id),
-            'recursive' => -1
-        ));
-        if (empty($user)) {
-            throw new NotFoundException(__('Invalid user'));
-        }
-        if ($this->User->delete($id)) {
-            $fieldsDescrStr = 'User (' . $id . '): ' . $user['User']['email'];
-            $this->User->extralog($this->Auth->user(), "delete", $fieldsDescrStr, '');
-            if ($this->_isRest()) {
-                return $this->RestResponse->saveSuccessResponse('User', 'admin_delete', $id, $this->response->type(), 'User deleted.');
-            } else {
-                $this->Flash->success(__('User deleted'));
-                $this->redirect(array('action' => 'index'));
+        if ($this->request->is('post') || $this->request->is('delete')) {
+            $user = $this->User->find('first', array(
+                'conditions' => $this->__adminFetchConditions($id),
+                'recursive' => -1,
+                'contain' => array('Role')
+            ));
+            if (empty($user)) {
+                throw new NotFoundException(__('Invalid user'));
             }
+            if ($this->User->delete($id)) {
+                $fieldsDescrStr = 'User (' . $id . '): ' . $user['User']['email'];
+                $this->User->extralog($this->Auth->user(), "delete", $fieldsDescrStr, '');
+                if ($this->_isRest()) {
+                    return $this->RestResponse->saveSuccessResponse('User', 'admin_delete', $id, $this->response->type(), 'User deleted.');
+                } else {
+                    $this->Flash->success(__('User deleted'));
+                    $this->redirect(array('action' => 'index'));
+                }
+            }
+            $this->Flash->error(__('User was not deleted'));
+            $this->redirect(array('action' => 'index'));
+        } else {
+            $this->set(
+                'question',
+                __('Are you sure you want to delete the user? It is highly recommended to never delete users but to disable them instead.')
+            );
+            $this->set('title', __('Delete user'));
+            $this->set('actionName', 'Delete');
+            $this->render('/genericTemplates/confirm');
         }
-        $this->Flash->error(__('User was not deleted'));
-        $this->redirect(array('action' => 'index'));
     }
 
     public function admin_massToggleField($fieldName, $enabled)
@@ -1115,6 +1138,7 @@ class UsersController extends AppController
                 'conditions' => $this->__adminFetchConditions($ids),
                 'recursive' => -1,
                 'fields' => ['id', $fieldName],
+                'contain' => array('Role')
             ]);
             if (empty($users)) {
                 throw new NotFoundException(__('Invalid users'));
@@ -1167,23 +1191,35 @@ class UsersController extends AppController
         if ($this->request->is(['post', 'put'])) {
             $this->Bruteforce = ClassRegistry::init('Bruteforce');
             if (!empty($this->request->data['User']['email'])) {
-                if ($this->Bruteforce->isBlocklisted($_SERVER['REMOTE_ADDR'], $this->request->data['User']['email'])) {
+                if ($this->Bruteforce->isBlocklisted($this->request->data['User']['email'])) {
                     $expire = Configure::check('SecureAuth.expire') ? Configure::read('SecureAuth.expire') : 300;
                     throw new ForbiddenException('You have reached the maximum number of login attempts. Please wait ' . $expire . ' seconds and try again.');
                 }
             }
-            // Check the length of the user's authkey match old format. This can be removed in future.
-            $userPass = $this->User->find('first', [
+            $unauth_user = $this->User->find('first', [
                 'conditions' => ['User.email' => $this->request->data['User']['email']],
-                'fields' => ['User.password'],
+                'fields' => ['User.password', 'User.totp', 'User.hotp_counter'],
                 'recursive' => -1,
             ]);
-            if (!empty($userPass) && strlen($userPass['User']['password']) === 40) {
-                $oldHash = true;
-                unset($this->Auth->authenticate['Form']['passwordHasher']); // use default password hasher
-                $this->Auth->constructAuthenticate();
+            if ($unauth_user) {
+                // Check the length of the user's authkey match old format. This can be removed in future.
+                $userPass = $unauth_user['User']['password'];
+                if (!empty($userPass) && strlen($userPass) === 40) {
+                    $oldHash = true;
+                    unset($this->Auth->authenticate['Form']['passwordHasher']); // use default password hasher
+                    $this->Auth->constructAuthenticate();
+                }
+                // user has TOTP token, check creds and redirect to TOTP validation
+                if (!empty($unauth_user['User']['totp']) && !$unauth_user['User']['disabled'] && class_exists('\OTPHP\TOTP')) {
+                    $user = $this->Auth->identify($this->request, $this->response);
+                    if ($user && !$user['disabled']) {
+                        $this->Session->write('otp_user', $user);
+                        return $this->redirect('otp');
+                    }
+                }
             }
         }
+        // if instance requires email OTP 
         if ($this->request->is('post') && Configure::read('Security.email_otp_enabled')) {
             $user = $this->Auth->identify($this->request, $this->response);
             if ($user && !$user['disabled']) {
@@ -1213,9 +1249,14 @@ class UsersController extends AppController
             if ($this->request->is('post') || $this->request->is('put')) {
                 $this->Flash->error(__('Invalid username or password, try again'));
                 if (isset($this->request->data['User']['email'])) {
-                    $this->Bruteforce->insert($_SERVER['REMOTE_ADDR'], $this->request->data['User']['email']);
+                    $this->Bruteforce->insert($this->request->data['User']['email']);
                 }
             }
+
+            //
+            // Actions needed for the first access, when the database is not populated yet.
+            // 
+
             // populate the DB with the first role (site admin) if it's empty
             if (!$this->User->Role->hasAny()) {
                 $siteAdmin = array('Role' => array(
@@ -1265,7 +1306,6 @@ class UsersController extends AppController
                 }
                 $org_id = $this->User->Organisation->id;
             }
-
             // populate the DB with the first user if it's empty
             if (!$this->User->hasAny()) {
                 if (!isset($org_id)) {
@@ -1277,7 +1317,6 @@ class UsersController extends AppController
                         $org_id = $firstOrg['Organisation']['id'];
                     }
                 }
-
                 $this->User->runUpdates();
                 $this->User->createInitialUser($org_id);
             }
@@ -1286,25 +1325,25 @@ class UsersController extends AppController
 
     private function _postlogin()
     {
-      $this->User->extralog($this->Auth->user(), "login");
-      $this->User->Behaviors->disable('SysLogLogable.SysLogLogable');
-      $this->User->id = $this->Auth->user('id');
-      $user = $this->User->find('first', array(
-          'conditions' => array(
-              'User.id' => $this->Auth->user('id')
-          ),
-          'recursive' => -1
-      ));
-      unset($user['User']['password']);
-      $this->User->updateLoginTimes($user['User']);
-      $lastUserLogin = $user['User']['last_login'];
-      $this->User->Behaviors->enable('SysLogLogable.SysLogLogable');
-      if ($lastUserLogin) {
-          $readableDatetime = (new DateTime())->setTimestamp($lastUserLogin)->format('D, d M y H:i:s O'); // RFC822
-          $this->Flash->info(__('Welcome! Last login was on %s', $readableDatetime));
-      }
-      // no state changes are ever done via GET requests, so it is safe to return to the original page:
-      $this->redirect($this->Auth->redirectUrl());
+        $this->User->extralog($this->Auth->user(), "login");
+        $this->User->Behaviors->disable('SysLogLogable.SysLogLogable');
+        $this->User->id = $this->Auth->user('id');
+        $user = $this->User->find('first', array(
+            'conditions' => array(
+                'User.id' => $this->Auth->user('id')
+            ),
+            'recursive' => -1
+        ));
+        unset($user['User']['password']);
+        $this->User->updateLoginTimes($user['User']);
+        $lastUserLogin = $user['User']['last_login'];
+        $this->User->Behaviors->enable('SysLogLogable.SysLogLogable');
+        if ($lastUserLogin) {
+            $readableDatetime = (new DateTime())->setTimestamp($lastUserLogin)->format('D, d M y H:i:s O'); // RFC822
+            $this->Flash->info(__('Welcome! Last login was on %s', $readableDatetime));
+        }
+        // no state changes are ever done via GET requests, so it is safe to return to the original page:
+        $this->redirect($this->Auth->redirectUrl());
     }
 
     public function routeafterlogin()
@@ -1344,6 +1383,7 @@ class UsersController extends AppController
         unset($user['User']['password']);
         $user['User']['action'] = 'logout';
         $this->User->save($user['User'], true, array('id'));
+        $this->Session->write('otp_secret', null);
         $this->redirect($this->Auth->logout());
     }
 
@@ -1536,8 +1576,9 @@ class UsersController extends AppController
     public function admin_quickEmail($user_id)
     {
         $user = $this->User->find('first', array(
-            'conditions' => $this->__adminFetchConditions($user_id),
-            'recursive' => -1
+            'conditions' => $this->__adminFetchConditions($user_id, $edit=False),
+            'recursive' => -1,
+            'contain' => array('Role')
         ));
         $error = false;
         if (empty($user)) {
@@ -1701,7 +1742,8 @@ class UsersController extends AppController
     {
         $user = $this->User->find('first', array(
             'conditions' => $this->__adminFetchConditions($id),
-            'recursive' => -1
+            'recursive' => -1,
+            'contain' => array('Role')
         ));
         if (empty($user)) {
             throw new NotFoundException(__('Invalid user'));
@@ -1731,6 +1773,178 @@ class UsersController extends AppController
         }
     }
 
+    public function otp()
+    {
+        $user = $this->Session->read('otp_user');
+        if (empty($user)) {
+            $this->redirect('login');
+        }
+        if ($this->request->is('post') && isset($this->request->data['User']['otp'])) {
+            $this->Bruteforce = ClassRegistry::init('Bruteforce');
+            if ($this->Bruteforce->isBlocklisted($user['email'])) {
+                $expire = Configure::check('SecureAuth.expire') ? Configure::read('SecureAuth.expire') : 300;
+                throw new ForbiddenException('You have reached the maximum number of login attempts. Please wait ' . $expire . ' seconds and try again.');
+            }
+            $secret = $user['totp'];
+            $totp = \OTPHP\TOTP::create($secret);
+            $hotp = \OTPHP\HOTP::create($secret);
+            if ($totp->verify(trim($this->request->data['User']['otp']))) {
+                // OTP is correct, we login the user with CakePHP
+                $this->Auth->login($user);
+                $this->_postlogin();
+            } elseif (isset($user['hotp_counter']) && $hotp->verify(trim($this->request->data['User']['otp']), $user['hotp_counter'])) {
+                // HOTP is correct, update the counter and login
+                $this->User->id = $user['id'];
+                $this->User->saveField('hotp_counter', $user['hotp_counter']+1);
+                $this->Auth->login($user);
+                $this->_postlogin();
+            } else {
+                $this->Flash->error(__("The OTP is incorrect or has expired"));
+                $fieldsDescrStr = 'User (' . $user['id'] . '): ' . $user['email']. ' wrong OTP token';
+                $this->User->extralog($user, "login_fail", $fieldsDescrStr, '');
+                $this->Bruteforce->insert($user['email']);
+            }
+        }
+        // GET Request or wrong OTP, just show the form
+        $this->set('totp', $user['totp']? true : false);
+        $this->set('hotp_counter', $user['hotp_counter']);
+    }
+
+    public function hotp()
+    {
+        if (!class_exists('\OTPHP\HOTP')) {
+            $this->Flash->error(__("The required PHP libraries to support OTP are not installed. Please contact your administrator to address this."));
+            $this->redirect($this->referer());
+        }
+
+        $user = $this->User->find('first', array(
+            'recursive' => -1,
+            'conditions' => array('User.id' => $this->Auth->user('id')),
+            'fields' => array(
+                'totp', 'email', 'id', 'hotp_counter'
+            )
+        ));
+        $hotp = \OTPHP\HOTP::create($user['User']['totp'], $user['User']['hotp_counter']);
+        $hotp_codes = [];
+        for ($i=$user['User']['hotp_counter']; $i < $user['User']['hotp_counter']+50 ; $i++) {
+            $hotp_codes[$i] = $hotp->at($i);
+        }
+        $this->set('hotp_codes', $hotp_codes);
+    }
+
+    public function totp_new()
+    {
+        if (Configure::read('LinOTPAuth.enabled')) {
+            $this->Flash->error(__("LinOTP is enabled for this instance. Built-in TOTP should not be used."));
+            $this->redirect($this->referer());
+        }
+        if (!class_exists('\OTPHP\TOTP') || !class_exists('\BaconQrCode\Writer')) {
+            $this->Flash->error(__("The required PHP libraries to support TOTP are not installed. Please contact your administrator to address this."));
+            $this->redirect($this->referer());
+        }
+        // only allow the users themselves to generate a TOTP secret.
+        // If TOTP is enforced they will be invited to generate it at first login
+        $user = $this->User->find('first', array(
+            'recursive' => -1,
+            'conditions' => array('User.id' => $this->Auth->user('id')),
+            'fields' => array(
+                'totp', 'email', 'id'
+            )
+        ));
+        if (empty($user)) {
+            throw new NotFoundException(__('Invalid user'));
+        }
+        // do not allow this page to be accessed if the current already has a TOTP. Just redirect to the users details page with a Flash->error()
+        if ($user['User']['totp']) { 
+            $this->Flash->error(__("Your account already has a TOTP. Please contact your organisational administrator to change or delete it."));
+            $this->redirect($this->referer());
+        }
+
+        if ($this->request->is('get')) {
+            $totp = \OTPHP\TOTP::create();
+            $secret = $totp->getSecret();
+            $this->Session->write('otp_secret', $secret);  // Store in session, we want to create a new secret each time the totp_new() function is queried via a GET (this will not impede incorrect confirmation attempty)
+        } else {
+            $secret = $this->Session->read('otp_secret');  // Reload secret from session.
+            if ($secret) {
+                $totp = \OTPHP\TOTP::create($secret);
+            } else {
+                $totp = \OTPHP\TOTP::create();
+                $secret = $totp->getSecret();
+                $this->Session->write('otp_secret', $secret); // Store in session, we want to keep reusing the same QR code until the user correctly enters the generated key on their authenticator
+            }
+        }
+        if ($this->request->is('post') && isset($this->request->data['User']['otp'])) {
+            if ($totp->verify(trim($this->request->data['User']['otp']))) {
+                // we know the user can generate TOTP tokens, save the new TOTP to the database
+                $this->User->id = $user['User']['id'];
+                $this->User->saveField('totp', $secret);
+                $this->User->saveField('hotp_counter', 0);
+                $this->_refreshAuth();    
+                $this->Flash->info(__('The OTP is correct and now active for your account.'));
+                $fieldsDescrStr = 'User (' . $user['User']['id'] . '): ' . $user['User']['email']. ' TOTP token created';
+                $this->User->extralog($this->Auth->user(), "update", $fieldsDescrStr, '');
+                // redirect to a page that gives the next 50 HOTP
+                $this->redirect(array('controller' => 'users', 'action'=> 'hotp'));
+            } else {
+                $this->Flash->error(__("The OTP is incorrect or has expired."));
+            }
+        } else {
+            // GET Request, just show the form
+        }
+        // generate QR code with the secret
+        $renderer = new \BaconQrCode\Renderer\ImageRenderer(
+            new \BaconQrCode\Renderer\RendererStyle\RendererStyle(200),
+            new \BaconQrCode\Renderer\Image\SvgImageBackEnd()
+        );
+        $writer = new \BaconQrCode\Writer($renderer);
+        $totp->setLabel($user['User']['email']);
+        if (Configure::read('Security.otp_issuer')) {
+            $totp->setIssuer(Configure::read('Security.otp_issuer'));
+        } else {
+            $totp->setIssuer(Configure::read('MISP.org') . ' MISP');
+        }
+        $qrcode = $writer->writeString($totp->getProvisioningUri());
+        $qrcode = preg_replace('/^.+\n/', '', $qrcode); // ignore first <?xml version line 
+
+        $this->set('qrcode', $qrcode);
+        $this->set('secret', $secret);
+    }
+
+    public function totp_delete($id) {
+        if ($this->request->is('post') || $this->request->is('delete')) {
+            $user = $this->User->find('first', array(
+                'conditions' => $this->__adminFetchConditions($id),
+                'recursive' => -1,
+                'contain' => array('Role')
+            ));
+            if (empty($user)) {
+                throw new NotFoundException(__('Invalid user'));
+            }
+            $this->User->id = $id;
+            if ($this->User->saveField('totp', null)) {
+                $fieldsDescrStr = 'User (' . $id . '): ' . $user['User']['email'] . ' TOTP deleted';
+                $this->User->extralog($this->Auth->user(), "update", $fieldsDescrStr, '');
+                if ($this->_isRest()) {
+                    return $this->RestResponse->saveSuccessResponse('User', 'admin_totp_delete', $id, $this->response->type(), 'User TOTP deleted.');
+                } else {
+                    $this->Flash->success(__('User TOTP deleted'));
+                    $this->redirect('/admin/users/index');
+                }
+            }
+            $this->Flash->error(__('User TOTP was not deleted'));
+            $this->redirect('/admin/users/index');
+        } else {
+            $this->set(
+                'question',
+                __('Are you sure you want to delete the TOTP of the user?.')
+            );
+            $this->set('title', __('Delete user TOTP'));
+            $this->set('actionName', 'Delete');
+            $this->render('/genericTemplates/confirm');
+        }
+    }
+
     public function email_otp()
     {
         $user = $this->Session->read('email_otp_user');
@@ -1750,6 +1964,8 @@ class UsersController extends AppController
                 $this->_postlogin();
             } else {
                 $this->Flash->error(__("The OTP is incorrect or has expired"));
+                $fieldsDescrStr = 'User (' . $user['id'] . '): ' . $user['email']. ' wrong email OTP token';
+                $this->User->extralog($user, "login_fail", $fieldsDescrStr, '');
             }
         } else {
             // GET Request
@@ -1784,7 +2000,7 @@ class UsersController extends AppController
             $body = $this->__replaceEmailVariables($body);
             $body = str_replace('$validity', $validity, $body);
             $body = str_replace('$otp', $otp, $body);
-            $body = str_replace('$ip', $this->__getClientIP(), $body);
+            $body = str_replace('$ip', $this->_remoteIp(), $body);
             $body = str_replace('$username', $user['email'], $body);
 
             // Fetch user that contains also PGP or S/MIME keys for e-mail encryption
@@ -1800,22 +2016,6 @@ class UsersController extends AppController
         }
     }
 
-    /**
-    * Helper function to determine the IP of a client (proxy aware)
-    */
-    private function __getClientIP() {
-      $x_forwarded = filter_input(INPUT_SERVER, 'HTTP_X_FORWARDED_FOR', FILTER_SANITIZE_STRING);
-      $client_ip = filter_input(INPUT_SERVER, 'HTTP_CLIENT_IP', FILTER_SANITIZE_STRING);
-      if (!empty($x_forwarded)) {
-        $x_forwarded = explode(",", $x_forwarded);
-        return $x_forwarded[0];
-      } elseif(!empty($client_ip)){
-        return $client_ip;
-      } else {
-        return filter_input(INPUT_SERVER, 'REMOTE_ADDR', FILTER_SANITIZE_STRING);
-      }
-    }
-
     // shows some statistics about the instance
     public function statistics($page = 'data')
     {
@@ -2831,7 +3031,7 @@ class UsersController extends AppController
      * @return array
      * @throws NotFoundException
      */
-    private function __adminFetchConditions($id)
+    private function __adminFetchConditions($id, $edit = True)
     {
         if (empty($id)) {
             throw new NotFoundException(__('Invalid user'));
@@ -2841,7 +3041,133 @@ class UsersController extends AppController
         $user = $this->Auth->user();
         if (!$user['Role']['perm_site_admin']) {
             $conditions['User.org_id'] = $user['org_id']; // org admin
+            if ($edit) {
+                $conditions['Role.perm_site_admin'] = False;
+            }
         }
         return $conditions;
     }
+
+    public function admin_destroy($id = null)
+    {
+        $conditionFields = ['id', 'email'];
+        $params = $this->IndexFilter->harvestParameters(['id', 'email']);
+        if (!empty($id)) {
+            $params['id'] = $id;
+        }
+        $conditions = [];
+        foreach ($conditionFields as $conditionField) {
+            if (!empty($params[$conditionField])) {
+                $conditions[$conditionField . ' LIKE'] = $params[$conditionField];
+            }
+        }
+        if (!empty($conditions)) {
+            $user_ids = $this->User->find('list', [
+                'recursive' => -1,
+                'fields' => ['email', 'id'],
+                'conditions' => $conditions
+            ]);
+        } else {
+            $user_ids = [__('Every user') => 'all'];
+        }
+        if ($this->request->is('post')) {
+            $redis = RedisTool::init();
+            $kill_before = time();
+            foreach (array_values($user_ids) as $user_id) {
+                $redis->set('misp:session_destroy:' . $user_id, $kill_before);
+            }
+            $message = __(
+                'Session destruction cutoff set to the current timestamp for the given selection (%s). Session(s) will be destroyed on the next user interaction.',
+                implode(', ', array_keys($user_ids))
+            );
+            if ($this->_isRest()) {
+                return $this->RestResponse->saveSuccessResponse('User', 'admin_destroy', false, $this->response->type(), $message);
+            }
+            $this->Flash->success($message);
+            $this->redirect($this->referer());
+        } else {
+            $this->set(
+                'question',
+                __(
+                    'Do you really wish to destroy the session for: %s ? The session destruction will occur when the users try to interact with MISP the next time.',
+                    implode(', ', array_keys($user_ids))
+                )
+            );
+            $this->set('title', __('Destroy sessions'));
+            $this->set('actionName', 'Destroy');
+            $this->render('/genericTemplates/confirm');
+        }
+    }
+    public function logout401() {
+        # You should read the documentation in docs/CONFIG.ApacheSecureAuth.md
+        # before using this endpoint. It is not useful without webserver config
+        # changes.
+        # To use this, set Plugin.CustomAuth_custom_logout to /users/logout401
+        $this->response->statusCode(401);
+    }
+
+    public function forgot()
+    {
+        if (empty(Configure::read('Security.allow_password_forgotten'))) {
+            $this->Flash->error(__('This feature is disabled.'));
+            $this->redirect('/');
+        }
+        if (!empty($this->Auth->user()) && !$this->_isRest()) {
+            $this->Flash->info(__('You are already logged in, no need to ask for a password reset. Log out first.'));
+            $this->redirect('/');
+        }
+        if ($this->request->is('post')) {
+            if (empty($this->request->data['User'])) {
+                $this->request->data = ['User' => $this->request->data];
+            }
+            if (empty($this->request->data['User']['email'])) {
+                throw new MethodNotAllowedException(__('No email provided, cannot generate password reset message.'));
+            }
+            $user = [
+                'id' => 0,
+                'email' => 'SYSTEM',
+                'Organisation' => [
+                    'name' => 'SYSTEM'
+                ]
+            ];
+            $this->loadModel('Log');
+            $this->Log->createLogEntry($user, 'forgot', 'User', 0, 'Password reset requested for: ' . $this->request->data['User']['email']);
+            $this->User->forgotRouter($this->request->data['User']['email'], $this->_remoteIp());
+            $message = __('Password reset request submitted. If a valid user is found, you should receive an e-mail with a temporary reset link momentarily. Please be advised that this link is only valid for 10 minutes.');
+            if ($this->_isRest()) {
+                return $this->RestResponse->saveSuccessResponse('User', 'forgot', false, $this->response->type(), $message);
+            }
+            $this->Flash->info($message);
+            $this->redirect('/');
+        }
+    }
+
+    public function password_reset($token)
+    {
+        if (empty(Configure::read('Security.allow_password_forgotten'))) {
+            $this->Flash->error(__('This feature is disabled.'));
+            $this->redirect('/');
+        }
+        $this->loadModel('Server');
+        $this->set('complexity', !empty(Configure::read('Security.password_policy_complexity')) ? Configure::read('Security.password_policy_complexity') : $this->Server->serverSettings['Security']['password_policy_complexity']['value']);
+        $this->set('length', !empty(Configure::read('Security.password_policy_length')) ? Configure::read('Security.password_policy_length') : $this->Server->serverSettings['Security']['password_policy_length']['value']);
+        if (!empty($this->Auth->user()) && !$this->_isRest()) {
+            $this->redirect('/');
+        }
+        $user = $this->User->fetchForgottenPasswordUser($token);
+        if (empty($user)) {
+            $message = __('Invalid token, or password request token already expired.');
+            if ($this->_isRest()) {
+                throw new MethodNotAllowedException($message);
+            } else {
+                $this->Flash->error($message);
+                $this->redirect('/');
+            }
+        }
+        if ($this->request->is('post') || $this->request->is('put')) {
+            $abortPost = false;
+            return $this->__pw_change(['User' => $user], 'password_reset', $abortPost, $token);
+        }
+    }
+
 }

app/Controller/WorkflowsController.php

@@ -67,6 +67,7 @@ class WorkflowsController extends AppController
             } else {
                 $successMessage = __('Workflow saved.');
                 $savedWorkflow = $result['saved'];
+                $savedWorkflow = $this->Workflow->attachLabelToConnections($savedWorkflow);
                 return $this->__getSuccessResponseBasedOnContext($successMessage, $savedWorkflow, 'edit', false, $redirectTarget);
             }
         } else {
@@ -101,6 +102,9 @@ class WorkflowsController extends AppController
             }
         }
         $this->CRUD->view($id, [
+            'afterFind' => function($workflow) {
+                return $this->Workflow->attachLabelToConnections($workflow);
+            }
         ]);
         if ($this->IndexFilter->isRest()) {
             return $this->restResponsePayload;
@@ -151,6 +155,7 @@ class WorkflowsController extends AppController
         } else {
             $workflow = $this->Workflow->fetchWorkflow($workflow_id);
         }
+        $workflow = $this->Workflow->attachLabelToConnections($workflow, $trigger_id);
         $modules = $this->Workflow->attachNotificationToModules($modules, $workflow);
         $this->loadModel('WorkflowBlueprint');
         $workflowBlueprints = $this->WorkflowBlueprint->find('all');
@@ -285,6 +290,8 @@ class WorkflowsController extends AppController
         if ($this->_isRest()) {
             return $this->RestResponse->viewData($module, $this->response->type());
         }
+        if (!isset($module['Workflow']))
+            $module['Workflow'] = ['counter' => false, 'id' => false];
         $this->set('data', $module);
         $this->set('menuData', ['menuList' => 'workflows', 'menuItem' => 'view_module']);
     }

app/Lib/Dashboard/APIActivityWidget.php

@@ -0,0 +1,137 @@
+<?php
+
+class APIActivityWidget
+{
+    public $title = 'API Activity';
+    public $render = 'SimpleList';
+    public $width = 2;
+    public $height = 2;
+    public $params = [
+        'filter' => 'A list of filters by organisation meta information (sector, type, nationality, id, uuid) to include. (dictionary, prepending values with ! uses them as a negation)',
+        'limit' => 'Limits the number of displayed APIkeys. (-1 will list all) Default: -1',
+        'days' => 'How many days back should the list go - for example, setting 7 will only show contributions in the past 7 days. (integer)',
+        'month' => 'Who contributed most this month? (boolean)',
+        'previous_month' => 'Who contributed most the previous, finished month? (boolean)',
+        'year' => 'Which contributed most this year? (boolean)',
+    ];
+    public $description = 'Basic widget showing some server statistics in regards to MISP.';
+    public $cacheLifetime = 10;
+    public $autoRefreshDelay = null;
+    private $User = null;
+    private $AuthKey = null;
+
+
+    private function getDates($options)
+    {
+        if (!empty($options['days'])) {
+            $begin = new DateTime(date('Y-m-d', strtotime(sprintf("-%s days", $options['days']))));
+        } else if (!empty($options['month'])) {
+            $begin = new DateTime(date('Y-m-d', strtotime('first day of this month 00:00:00', time())));
+        } else if (!empty($options['previous_month'])) {
+            $begin = new DateTime(date('Y-m-d', strtotime('first day of last month 00:00:00', time())));
+            $end = new DateTime(date('Y-m-d', strtotime('last day of last month 23:59:59', time())));
+        } else if (!empty($options['year'])) {
+            $begin = new DateTime(date('Y-m-d', strtotime('first day of this year 00:00:00', time())));
+        } else {
+            $begin = new DateTime(date('Y-m-d', strtotime('-7 days', time())));;
+        }
+
+        $end = isset($end) ? $end : new DateTime();
+        $dates = new DatePeriod(
+            $begin,
+            new DateInterval('P1D'),
+            $end
+        );
+        $results = [];
+        foreach ($dates as $date) {
+            $results[] = $date->format('Ymd');
+        }
+        return $results;
+    }
+
+	public function handler($user, $options = array())
+	{
+        $this->User = ClassRegistry::init('User');
+        $this->AuthKey = ClassRegistry::init('AuthKey');
+        $redis = $this->User->setupRedis();
+        if (!$redis) {
+            throw new NotFoundException(__('No redis connection found.'));
+        }
+
+        $params = ['conditions' => []];
+        $dates = $this->getDates($options);
+        $pipe = $redis->pipeline();
+        foreach ($dates as $date) {
+            $pipe->zrange('misp:authkey_log:' . $date, 0, -1, true);
+        }
+        $temp = $pipe->exec();
+        $raw_results = [];
+        $counts = [];
+        foreach ($dates as $k => $date) {
+            $raw_results[$date] = $temp[$k];
+            if (!empty($temp[$k])) {
+                foreach ($temp[$k] as $key => $count) {
+                    if (isset($counts[$key])) {
+                        $counts[$key] += (int)$count;
+                    } else {
+                        $counts[$key] = (int)$count;
+                    }
+                }
+            }
+        }
+        arsort($counts);
+        $this->AuthKey->Behaviors->load('Containable');
+        $temp_apikeys = array_flip(array_keys($counts));
+        foreach ($temp_apikeys as $apikey => $value) {
+            $temp_apikeys[$apikey] = $this->AuthKey->find('first', [
+                'conditions' => [
+                    'AuthKey.authkey_start' => substr($apikey, 0, 4),
+                    'AuthKey.authkey_end' => substr($apikey, 4)
+                ],
+                'fields' => ['AuthKey.authkey_start', 'AuthKey.authkey_end', 'AuthKey.id', 'User.id', 'User.email'],
+                'recursive' => 1
+            ]);
+        }
+        $results = [];
+        $baseurl = empty(Configure::read('MISP.external_baseurl')) ? h(Configure::read('MISP.baseurl')) : Configure::read('MISP.external_baseurl');
+        foreach ($counts as $key => $junk) {
+            $data = $temp_apikeys[$key];
+            if (!empty($data)) {
+                $results[] = [
+                    'html_title' => sprintf(
+                        '<a href="%s/auth_keys/view/%s">%s</a>',
+                        h($baseurl),
+                        h($data['AuthKey']['id']),
+                        $key
+                    ),
+                    'html' => sprintf(
+                        '%s (<a href="%s/admin/users/view/%s">%s</a>)',
+                        h($counts[$key]),
+                        h($baseurl),
+                        h($data['User']['id']),
+                        h($data['User']['email'])
+                    )
+                ];
+            } else {
+                $results[] = [
+                    'title' => $key,
+                    'html' => sprintf(
+                        '%s (<span class="red" title="%s">%s</span>)',
+                        h($counts[$key]),
+                        __('An unknown key can be caused by the given key having been permanently deleted or falsely mis-identified (for the purposes of this widget) on instances using legacy API key authentication.'),
+                        __('Unknown key')
+                    )
+                ];
+            }
+        }
+        return $results;
+	}
+
+    public function checkPermissions($user)
+    {
+        if (empty($user['Role']['perm_site_admin'])) {
+            return false;
+        }
+        return true;
+    }
+}

app/Lib/Dashboard/AttackWidget.php

@@ -0,0 +1,38 @@
+<?php
+class AttackWidget
+{
+    public $title = 'ATT&CK heatmap';
+    public $render = 'Attack';
+    public $description = 'Retrieve an ATT&CK (or ATT&CK like) heatmap for the current instance.';
+    public $width = 3;
+    public $height = 4;
+    public $params = [
+        'filters' => 'A list of restsearch filters to apply to the heatmap. (dictionary, prepending values with ! uses them as a negation)'
+    ];
+    public $cacheLifetime = 1200;
+    public $autoRefreshDelay = false;
+    private $validFilterKeys = [
+        'filters'
+    ];
+    private $Event = null;
+    public $placeholder =
+'{
+    "filters": {
+        "attackGalaxy": "mitre-attack-pattern",
+        "timestamp": ["2023-01-01", "2023-03-31"],
+        "published": [0,1]
+    }
+}';
+
+    public function handler($user, $options = array())
+    {
+        $this->Event = ClassRegistry::init('Event');
+        $data = null;
+        if (!empty($options['filters'])) {
+            $data = $this->Event->restSearch($user, 'attack', $options['filters']);
+            $data = JsonTool::decode($data->intoString());
+        }
+        return $data;
+    }
+}
+?>

app/Lib/Dashboard/EventEvolutionLineWidget.php

@@ -0,0 +1,136 @@
+<?php
+
+class EventEvolutionLineWidget
+{
+    public $title = 'Evolution of published event count (filterable)';
+    public $render = 'MultiLineChart';
+    public $width = 7;
+    public $height = 6;
+    public $description = 'A linechart of event publishes.';
+    public $cacheLifetime = null;
+    public $autoRefreshDelay = false;
+    public $params = [
+        'filter' => 'A list of filters by organisation meta information (nationality, sector, type, name, uuid) to include. (dictionary, prepending values with ! uses them as a negation)',
+        'start_date' => 'Start date, expressed in Y-m-d format (e.g. 2012-10-01)'
+    ];
+    private $validFilterKeys = [
+        'nationality',
+        'sector',
+        'type',
+        'name',
+        'uuid'
+    ];
+
+    public $placeholder =
+        '{
+    "filter": {
+        "sector": "financial"
+    },
+    "start_date": "2017-01",
+}';
+
+    private $Organisation = null;
+    private $Event = null;
+
+    private function timeConditions($options)
+    {
+        if (!empty($options['start_date'])) {
+            $condition = strtotime($options['start_date']);
+        } else {
+            $condition = strtotime('2012-10-01');
+        }
+        $datetime = new DateTime();
+        $datetime->setTimestamp($condition);
+        return $datetime->format('Y-m-d H:i:s');
+    }
+
+    public function handler($user, $options = array())
+    {
+        $this->Organisation = ClassRegistry::init('Organisation');
+        $this->Event = ClassRegistry::init('Event');
+        $oparams = [
+            'conditions' => [
+                'AND' => ['Organisation.local' => !isset($options['local']) ? 1 : $options['local']]
+            ],
+            'limit' => 10,
+            'recursive' => -1
+        ];
+        $eparams = [];
+        if (!empty($options['filter']) && is_array($options['filter'])) {
+            foreach ($this->validFilterKeys as $filterKey) {
+                if (!empty($options['filter'][$filterKey])) {
+                    if (!is_array($options['filter'][$filterKey])) {
+                        $options['filter'][$filterKey] = [$options['filter'][$filterKey]];
+                    }
+                    $tempConditionBucket = [];
+                    foreach ($options['filter'][$filterKey] as $value) {
+                        if ($value[0] === '!') {
+                            $tempConditionBucket['Organisation.' . $filterKey . ' NOT IN'][] = mb_substr($value, 1);
+                        } else {
+                            $tempConditionBucket['Organisation.' . $filterKey . ' IN'][] = $value;
+                        }
+                    }
+                    if (!empty($tempConditionBucket)) {
+                        $oparams['conditions']['AND'][] = $tempConditionBucket;
+                    }
+                }
+            }
+        }
+        $timeConditions = $this->timeConditions($options);
+        if ($timeConditions) {
+            $eparams['conditions']['AND'][] = ['Event.publish_timestamp >=' => strtotime($timeConditions)];
+        }
+        $org_ids = $this->Organisation->find('list', [
+            'recursive' => -1,
+            'conditions' => $oparams['conditions'],
+            'fields' => ['id']
+        ]);
+        $this->Event->virtualFields = [
+            'published_date' => null
+        ];
+        $raw = $this->Event->find('all', [
+            'recursive' => -1,
+            'conditions' => $eparams['conditions'],
+            'fields' => ['DATE_FORMAT(FROM_UNIXTIME(Event.publish_timestamp), "%Y-%m") AS date', 'count(id) AS count'],
+            'group' => 'MONTH(FROM_UNIXTIME(Event.publish_timestamp)), YEAR(FROM_UNIXTIME(Event.publish_timestamp))'
+            
+        ]);
+
+        usort($raw, [$this, 'sortByCreationDate']);
+        $raw_padded = [];
+        $total = 0;
+        $default_start_date = empty($raw) ? '2012-10-01' : ($raw[0][0]['date'] . '-01');
+        $start = new DateTime(empty($options['start_date']) ? $default_start_date : $options['start_date']);
+        $end = new DateTime(date('Y-m') . '-01');
+        $interval = DateInterval::createFromDateString('1 month');
+        $period = new DatePeriod($start, $interval, $end);
+        foreach ($period as $dt) {
+            $raw_padded[$dt->format('Y-m') . '-01'] = 0;
+        }
+        foreach ($raw as $datapoint) {
+            $raw_padded[$datapoint[0]['date'] . '-01'] = (int)$datapoint[0]['count'];
+        }
+        $total = 0;
+        foreach ($raw_padded as $date => $count) {
+            $total += $count;
+            $raw_padded[$date] = $total;
+        }
+        $data = [];
+        foreach ($raw_padded as $date => $count) {
+            $data['data'][] = [
+              'Events' => (int)$count,
+              'date' => $date
+            ];
+        }
+        return $data;
+    }
+
+    private function sortByCreationDate($a, $b) {
+        if ($a[0]['date'] > $b[0]['date']) { 
+            return 1;
+        } else {
+            return -1;
+        }
+        return 0;
+    }
+}

app/Lib/Dashboard/LoginsWidget.php

@@ -0,0 +1,99 @@
+<?php
+
+class LoginsWidget
+{
+    public $title = 'Logins';
+    public $render = 'SimpleList';
+    public $width = 2;
+    public $height = 2;
+    public $params = [
+        'filter' => 'A list of filters by organisation meta information (sector, type, nationality, id, uuid) to include. (dictionary, prepending values with ! uses them as a negation)',
+        'limit' => 'Limits the number of displayed APIkeys. (-1 will list all) Default: -1',
+        'days' => 'How many days back should the list go - for example, setting 7 will only show contributions in the past 7 days. (integer)',
+        'month' => 'Who contributed most this month? (boolean)',
+        'previous_month' => 'Who contributed most the previous, finished month? (boolean)',
+        'year' => 'Which contributed most this year? (boolean)',
+    ];
+    public $description = 'Basic widget showing some server statistics in regards to MISP.';
+    public $cacheLifetime = 10;
+    public $autoRefreshDelay = null;
+    private $User = null;
+    private $Log = null;
+
+
+    private function getDates($options)
+    {
+        if (!empty($options['days'])) {
+            $begin = date('Y-m-d H:i:s', strtotime(sprintf("-%s days", $options['days'])));
+        } else if (!empty($options['month'])) {
+            $begin = date('Y-m-d H:i:s', strtotime('first day of this month 00:00:00', time()));
+        } else if (!empty($options['previous_month'])) {
+            $begin = date('Y-m-d H:i:s', strtotime('first day of last month 00:00:00', time()));
+            $end = date('Y-m-d H:i:s', strtotime('last day of last month 23:59:59', time()));
+        } else if (!empty($options['year'])) {
+            $begin = date('Y-m-d', strtotime('first day of this year 00:00:00', time()));
+        } else {
+            $begin = date('Y-m-d H:i:s', strtotime('-7 days', time()));
+        }
+        $params = [];
+        if (!empty($end)) {
+            $params['Log.created <='] = $end;
+        }
+        if (!empty($begin)) {
+            $params['Log.created >='] = $begin;
+        }
+        return $params;
+    }
+
+	public function handler($user, $options = array())
+	{
+        $this->User = ClassRegistry::init('User');
+        $this->Log = ClassRegistry::init('Log');
+        $conditions = $this->getDates($options);
+        $conditions['Log.action'] = 'login';
+        $this->Log->Behaviors->load('Containable');
+        $this->Log->bindModel([
+            'belongsTo' => [
+                'User'
+            ]
+        ]);
+        $this->Log->virtualFields['count'] = 0;
+        $this->Log->virtualFields['email'] = '';
+        $logs = $this->Log->find('all', [
+            'recursive' => -1,
+            'conditions' => $conditions,
+            'fields' => ['Log.user_id', 'COUNT(Log.id) AS Log__count', 'User.email AS Log__email'],
+            'contain' => ['User'],
+            'group' => ['Log.user_id']
+        ]);
+        $counts = [];
+        $emails = [];
+        foreach ($logs as $log) {
+            $counts[$log['Log']['user_id']] = $log['Log']['count'];
+            $emails[$log['Log']['user_id']] = $log['Log']['email'];
+        }
+        $results = [];
+        arsort($counts);
+        $baseurl = empty(Configure::read('MISP.external_baseurl')) ? h(Configure::read('MISP.baseurl')) : Configure::read('MISP.external_baseurl');
+        foreach ($counts as $user_id => $count) {
+            $results[] = [
+                'html_title' => sprintf(
+                    '<a href="%s/admin/users/view/%s">%s</a>',
+                    h($baseurl),
+                    h($user_id),
+                    h($emails[$user_id])
+                ),
+                'value' => $count
+            ];
+        }
+        return $results;
+	}
+
+    public function checkPermissions($user)
+    {
+        if (empty($user['Role']['perm_site_admin'])) {
+            return false;
+        }
+        return true;
+    }
+}

app/Lib/Dashboard/NewOrgsWidget.php

@@ -0,0 +1,171 @@
+<?php
+
+class NewOrgsWidget
+{
+    public $title = 'New organisations';
+    public $render = 'Index';
+    public $width = 7;
+    public $height = 6;
+    public $description = 'A list of the latest new member organisations.';
+    private $tableDescription = null;
+    public $cacheLifetime = null;
+    public $autoRefreshDelay = false;
+    public $params = [
+        'limit' => 'Maximum number of joining organisations shown. (integer, defaults to 10 if not set)',
+        'filter' => 'A list of filters by organisation meta information (nationality, sector, type, name, uuid) to include. (dictionary, prepending values with ! uses them as a negation)',
+        'days' => 'How many days back should the list go - for example, setting 7 will only show the organisations that were added in the past 7 days. (integer)',
+        'month' => 'Which organisations have been added this month? (boolean)',
+        'previous_month' => 'Who contributed most the previous, finished month? (boolean)',
+        'year' => 'Which organisations have been added this year? (boolean)',
+        'local' => 'Should the list only show local organisations? (boolean or list of booleans, defaults to 1. To get both sets, use [0,1])',
+        'fields' => 'Which fields should be displayed, by default all are selected. Pass a list with the following options: [id, uuid, name, sector, type, nationality, creation_date]'
+    ];
+    private $validFilterKeys = [
+        'nationality',
+        'sector',
+        'type',
+        'name',
+        'uuid'
+    ];
+
+    public $placeholder =
+        '{
+    "limit": 5,
+    "filter": {
+       "nationality": [
+            "Hungary",
+            "Russia",
+            "North Korea"
+       ]
+    },
+    "month": true
+}';
+
+    private $Organisation = null;
+
+    private function timeConditions($options)
+    {
+        $limit = empty($options['limit']) ? 10 : $options['limit'];
+        if (!empty($options['days'])) {
+            $condition = strtotime(sprintf("-%s days", $options['days']));
+            $this->tableDescription = __('The %d newest organisations created in the past %d days', $limit, (int)$options['days']);
+        } else if (!empty($options['month'])) {
+            $condition = strtotime('first day of this month 00:00:00', time());
+            $this->tableDescription = __('The %d newest organisations created during the current month', $limit);
+        } else if (!empty($options['previous_month'])) {
+            $condition = strtotime('first day of last month 00:00:00', time());
+            $end_condition = strtotime('last day of last month 23:59:59', time());
+            $this->tableDescription = __('The %d newest organisations created during the previous month', $limit);
+        } else if (!empty($options['year'])) {
+            $condition = strtotime('first day of this year 00:00:00', time());
+            $this->tableDescription = __('The %d newest organisations created during the current year', $limit);
+        } else {
+            $this->tableDescription = __('The %d newest organisations created', $limit);
+            return null;
+        }
+        $conditions = [];
+        if (!empty($condition)) {
+            $datetime = new DateTime();
+            $datetime->setTimestamp($condition);
+            $conditions['Organisation.date_created >='] = $datetime->format('Y-m-d H:i:s');
+        }
+        if (!empty($end_condition)) {
+            $datetime = new DateTime();
+            $datetime->setTimestamp($end_condition);
+            $conditions['Organisation.date_created <='] = $datetime->format('Y-m-d H:i:s');
+        }
+        return $conditions;
+    }
+
+    public function handler($user, $options = array())
+    {
+        $this->Organisation = ClassRegistry::init('Organisation');
+        $field_options = [
+            'id' => [
+                'name' => '#',
+                'url' => Configure::read('MISP.baseurl') . '/organisations/view',
+                'element' => 'links',
+                'data_path' => 'Organisation.id',
+                'url_params_data_paths' => 'Organisation.id'
+            ],
+            'date_created' => [
+                'name' => 'Creation date',
+                'data_path' => 'Organisation.date_created'
+            ],
+            'name' => [
+                'name' => 'Name',
+                'data_path' => 'Organisation.name',
+            ],
+            'uuid' => [
+                'name' => 'UUID',
+                'data_path' => 'Organisation.uuid',
+            ],
+            'sector' => [
+                'name' => 'Sector',
+                'data_path' => 'Organisation.sector',
+            ],
+            'nationality' => [
+                'name' => 'Nationality',
+                'data_path' => 'Organisation.nationality',
+            ],
+            'type' => [
+                'name' => 'Type',
+                'data_path' => 'Organisation.type',
+            ]
+        ];
+        $params = [
+            'conditions' => [
+                'AND' => ['Organisation.local' => !isset($options['local']) ? 1 : $options['local']]
+            ],
+            'limit' => 10,
+            'recursive' => -1
+        ];
+        if (!empty($options['filter']) && is_array($options['filter'])) {
+            foreach ($this->validFilterKeys as $filterKey) {
+                if (!empty($options['filter'][$filterKey])) {
+                    if (!is_array($options['filter'][$filterKey])) {
+                        $options['filter'][$filterKey] = [$options['filter'][$filterKey]];
+                    }
+                    $tempConditionBucket = [];
+                    foreach ($options['filter'][$filterKey] as $value) {
+                        if ($value[0] === '!') {
+                            $tempConditionBucket['Organisation.' . $filterKey . ' NOT IN'][] = mb_substr($value, 1);
+                        } else {
+                            $tempConditionBucket['Organisation.' . $filterKey . ' IN'][] = $value;
+                        }
+                    }
+                    if (!empty($tempConditionBucket)) {
+                        $params['conditions']['AND'][] = $tempConditionBucket;
+                    }
+                }
+            }
+        }
+        $timeConditions = $this->timeConditions($options);
+        if ($timeConditions) {
+            $params['conditions']['AND'][] = ['Organisation.date_created >=' => $timeConditions];
+        }
+        if (isset($options['fields'])) {
+            $fields = [];
+            foreach ($options['fields'] as $field) {
+                if (isset($field_options[$field])) {
+                    $fields[$field] = $field_options[$field];
+                }
+            }
+        } else {
+            $fields = $field_options;
+        }
+        $data = $this->Organisation->find('all', [
+            'recursive' => -1,
+            'conditions' => $params['conditions'],
+            'limit' => isset($options['limit']) ? (int)$options['limit'] : 10,
+            'fields' => array_keys($fields),
+            'order' => 'Organisation.date_created DESC'
+        ]);
+
+        return [
+            'data' => $data,
+            'fields' => $fields,
+            'description' => $this->tableDescription
+        ];
+    }
+}

app/Lib/Dashboard/NewUsersWidget.php

@@ -0,0 +1,187 @@
+<?php
+
+class NewUsersWidget
+{
+    public $title = 'New users';
+    public $render = 'Index';
+    public $width = 7;
+    public $height = 6;
+    public $description = 'A list of the latest new users.';
+    private $tableDescription = null;
+    public $cacheLifetime = null;
+    public $autoRefreshDelay = false;
+    public $params = [
+        'limit' => 'Maximum number of joining users shown. (integer, defaults to 10 if not set)',
+        'filter' => 'A list of filters for the organisations (nationality, sector, type, name, uuid) to include. (dictionary, prepending values with ! uses them as a negation)',
+        'days' => 'How many days back should the list go - for example, setting 7 will only show the organisations that were added in the past 7 days. (integer)',
+        'month' => 'Which organisations have been added this month? (boolean)',
+        'previous_month' => 'Who contributed most the previous, finished month? (boolean)',
+        'year' => 'Which organisations have been added this year? (boolean)',
+        'fields' => 'Which fields should be displayed, by default all are selected. Pass a list with the following options: [id, email, Organisation.name, Role.name, date_created]'
+    ];
+    private $validFilterKeys = [
+        'id',
+        'email',
+        'Organisation.name',
+        'Role.name',
+        'date_created'
+    ];
+
+    public $placeholder =
+        '{
+    "limit": 10,
+    "filter": {
+       "Organisation.name": [
+            "!FSB",
+            "!GRU",
+            "!Kaspersky"
+       ],
+       "email": [
+            "!andras.iklody@circl.lu"
+       ],
+       "Role.name": [
+            "Publisher",
+            "User"
+       ]
+    },
+    "year": true
+}';
+
+    private $User = null;
+
+    private function timeConditions($options)
+    {
+        $limit = empty($options['limit']) ? 10 : $options['limit'];
+        if (!empty($options['days'])) {
+            $condition = strtotime(sprintf("-%s days", $options['days']));
+            $this->tableDescription = __('The %d newest users created in the past %d days', $limit, (int)$options['days']);
+        } else if (!empty($options['month'])) {
+            $condition = strtotime('first day of this month 00:00:00', time());
+            $this->tableDescription = __('The %d newest users created during the current month', $limit);
+        } else if (!empty($options['previous_month'])) {
+            $condition = strtotime('first day of last month 00:00:00', time());
+            $end_condition = strtotime('last day of last month 23:59:59', time());
+            $this->tableDescription = __('The %d newest organisations created during the previous month', $limit);
+        } else if (!empty($options['year'])) {
+            $condition = strtotime('first day of this year 00:00:00', time());
+            $this->tableDescription = __('The %d newest users created during the current year', $limit);
+        } else {
+            $this->tableDescription = __('The %d newest users created', $limit);
+            return null;
+        }
+        $conditions = [];
+        if (!empty($condition)) {
+            $datetime = new DateTime();
+            $datetime->setTimestamp($condition);
+            $conditions['Organisation.date_created >='] = $datetime->format('Y-m-d H:i:s');
+        }
+        if (!empty($end_condition)) {
+            $datetime = new DateTime();
+            $datetime->setTimestamp($end_condition);
+            $conditions['Organisation.date_created <='] = $datetime->format('Y-m-d H:i:s');
+        }
+        return $conditions;
+    }
+
+    public function handler($user, $options = array())
+    {
+        $this->User = ClassRegistry::init('User');
+        $field_options = [
+            'id' => [
+                'name' => '#',
+                'url' => empty($user['Role']['perm_site_admin']) ? null : Configure::read('MISP.baseurl') . '/admin/users/view',
+                'element' => 'links',
+                'data_path' => 'User.id',
+                'url_params_data_paths' => 'User.id'
+            ],
+            'date_created' => [
+                'name' => 'Creation date',
+                'data_path' => 'User.date_created'
+            ],
+            'email' => [
+                'name' => 'E-mail',
+                'data_path' => 'User.email',
+            ],
+            'Organisation.name' => [
+                'name' => 'Organisation',
+                'data_path' => 'Organisation.name',
+            ],
+            'Role.name' => [
+                'name' => 'Role',
+                'data_path' => 'Role.name',
+            ]
+        ];
+        $params = [
+            'conditions' => [],
+            'limit' => 10,
+            'recursive' => -1
+        ];
+        if (!empty($options['filter']) && is_array($options['filter'])) {
+            foreach ($this->validFilterKeys as $filterKey) {
+                if (!empty($options['filter'][$filterKey])) {
+                    if (!is_array($options['filter'][$filterKey])) {
+                        $options['filter'][$filterKey] = [$options['filter'][$filterKey]];
+                    }
+                    $tempConditionBucket = [];
+                    foreach ($options['filter'][$filterKey] as $value) {
+                        $filterName = strpos($filterKey, '.') ? $filterKey : 'User.' . $filterKey;
+                        if ($value[0] === '!') {
+                            $tempConditionBucket[$filterName . ' NOT IN'][] = mb_substr($value, 1);
+                        } else {
+                            $tempConditionBucket[$filterName . ' IN'][] = $value;
+                        }
+                    }
+                    if (!empty($tempConditionBucket)) {
+                        $params['conditions']['AND'][] = $tempConditionBucket;
+                    }
+                }
+            }
+        }
+        $timeConditions = $this->timeConditions($options);
+        if ($timeConditions) {
+            $params['conditions']['AND'][] = $timeConditions;
+        }
+        if (isset($options['fields'])) {
+            $fields = [];
+            foreach ($options['fields'] as $field) {
+                if (isset($field_options[$field])) {
+                    $fields[$field] = $field_options[$field];
+                }
+            }
+        } else {
+            $fields = $field_options;
+        }
+
+        // redact e-mails for non site admins unless specifically allowed
+        if (
+            empty($user['Role']['perm_site_admin']) &&
+            !Configure::read('Security.disclose_user_emails') &&
+            isset($fields['email'])
+        ) {
+                unset($fields['email']);
+        }
+        $data = $this->User->find('all', [
+            'recursive' => -1,
+            'contain' => ['Organisation.name', 'Role.name'],
+            'conditions' => $params['conditions'],
+            'limit' => isset($options['limit']) ? $options['limit'] : 10,
+            'fields' => array_keys($fields),
+            'order' => 'User.date_created DESC'
+        ]);
+
+        foreach ($data as &$u) {
+            if (empty($u['User']['date_created'])) {
+                continue;
+            }
+            $tempDate = new DateTime();
+            $tempDate->setTimestamp($u['User']['date_created']);
+            $u['User']['date_created'] = $tempDate->format('Y-m-d H:i:s');
+        }
+
+        return [
+            'data' => $data,
+            'fields' => $fields,
+            'description' => $this->tableDescription
+        ];
+    }
+}

app/Lib/Dashboard/OrgContributionToplistWidget.php

@@ -0,0 +1,122 @@
+<?php
+class OrgContributionToplistWidget
+{
+    public $title = 'Contributor Top List (Orgs)';
+    public $render = 'BarChart';
+    public $description = 'The top contributors (orgs) in a selected time frame.';
+    public $width = 3;
+    public $height = 4;
+    public $params = [
+        'days' => 'How many days back should the list go - for example, setting 7 will only show contributions in the past 7 days. (integer)',
+        'month' => 'Who contributed most this month? (boolean)',
+        'previous_month' => 'Who contributed most the previous, finished month? (boolean)',
+        'year' => 'Which contributed most this year? (boolean)',
+        'filter' => 'A list of filters by organisation meta information (nationality, sector, type, name, uuid, local (- expects a boolean or a list of boolean values)) to include. (dictionary, prepending values with ! uses them as a negation)',
+        'limit' => 'Limits the number of displayed tags. Default: 10'
+    ];
+    public $cacheLifetime = null;
+    public $autoRefreshDelay = false;
+    private $validFilterKeys = [
+        'nationality',
+        'sector',
+        'type',
+        'name',
+        'uuid'
+    ];
+    public $placeholder =
+'{
+    "days": "7d",
+    "threshold": 15,
+    "filter": {
+        "sector": "Financial"
+    }
+}';
+    private $Org = null;
+    private $Event = null;
+
+
+    private function timeConditions($options)
+    {
+        $limit = empty($options['limit']) ? 10 : $options['limit'];
+        if (!empty($options['days'])) {
+            $condition = strtotime(sprintf("-%s days", $options['days']));
+        } else if (!empty($options['month'])) {
+            $condition = strtotime('first day of this month 00:00:00', time());
+        } else if (!empty($options['previous_month'])) {
+            $condition = strtotime('first day of previous month 00:00:00', time());
+            $end_condition = strtotime('last day of last month 23:59:59', time());
+        } else if (!empty($options['year'])) {
+            $condition = strtotime('first day of this year 00:00:00', time());
+        } else {
+            return null;
+        }
+        $conditions = [];
+        if (!empty($condition)) {
+            $datetime = new DateTime();
+            $datetime->setTimestamp($condition);
+            $conditions['Event.timestamp >='] = $datetime->format('Y-m-d H:i:s');
+        }
+        if (!empty($end_condition)) {
+            $datetime = new DateTime();
+            $datetime->setTimestamp($end_condition);
+            $conditions['Event.timestamp <='] = $datetime->format('Y-m-d H:i:s');
+        }
+        return $conditions;
+    }
+
+
+    public function handler($user, $options = array())
+    {
+        $params = ['conditions' => []];
+        $timeConditions = $this->timeConditions($options);
+        if ($timeConditions) {
+            $params['conditions']['AND'][] = $timeConditions;
+        }
+        if (!empty($options['filter']) && is_array($options['filter'])) {
+            foreach ($this->validFilterKeys as $filterKey) {
+                if (!empty($options['filter'][$filterKey])) {
+                    if (!is_array($options['filter'][$filterKey])) {
+                        $options['filter'][$filterKey] = [$options['filter'][$filterKey]];
+                    }
+                    $tempConditionBucket = [];
+                    foreach ($options['filter'][$filterKey] as $value) {
+                        if ($value[0] === '!') {
+                            $tempConditionBucket['Organisation.' . $filterKey . ' NOT IN'][] = mb_substr($value, 1);
+                        } else {
+                            $tempConditionBucket['Organisation.' . $filterKey . ' IN'][] = $value;
+                        }
+                    }
+                    if (!empty($tempConditionBucket)) {
+                        $params['conditions']['AND'][] = $tempConditionBucket;
+                    }
+                }
+            }
+        }
+        if (isset($options['filter']['local'])) {
+            $params['conditions']['AND']['local'] = $options['filter']['local'];
+        }
+
+        $this->Org = ClassRegistry::init('Organisation');
+        $org_ids = $this->Org->find('list', [
+            'fields' => ['Organisation.id', 'Organisation.name'],
+            'conditions' => $params['conditions']
+        ]);
+        $conditions = ['Event.orgc_id IN' => array_keys($org_ids)];
+        $this->Event = ClassRegistry::init('Event');
+        $this->Event->virtualFields['frequency'] = 0;
+        $orgs = $this->Event->find('all', [
+            'recursive' => -1,
+            'fields' => ['orgc_id', 'count(Event.orgc_id) as Event__frequency'],
+            'group' => ['orgc_id'],
+            'conditions' => $conditions,
+            'order' => 'count(Event.orgc_id) desc',
+            'limit' => empty($options['limit']) ? 10 : $options['limit']
+        ]);
+        $results = [];
+        foreach($orgs as $org) {
+            $results[$org_ids[$org['Event']['orgc_id']]] = $org['Event']['frequency'];
+        }
+        return ['data' => $results];
+    }
+}
+?>

app/Lib/Dashboard/OrgEventsWidget.php

@@ -0,0 +1,121 @@
+<?php
+/**
+* Org Events widget which reportes the number of events created monthly by each local organizations
+*
+*/
+class OrgEventsWidget
+{
+    public $title = 'Org Events';
+    public $render = 'MultiLineChart';
+    public $width = 8;
+    public $height = 6;
+    public $description = 'A graph to show the monthly number of events per organisation';
+    public $cacheLifetime = 10;
+    public $autoRefreshDelay = false;
+    public $params = array (
+        'blocklist_orgs' => 'A list of organisation names to filter out',
+        'months' => 'Number of past months to consider for the graph',
+        'logarithmic' => 'Visualize data on logarithmic scale'
+    );
+
+    public $placeholder =
+'{
+    "blocklist_orgs": ["Orgs to filter"],
+    "months": "6",
+    "logarithmic": "true"
+}';
+
+
+
+
+
+    /*
+    * Target_month must be from 1 to 12
+    * Target year must be 4 digits
+    */
+    private function org_events_count($user, $org, $target_month, $target_year) {
+        $events_count = 0;
+
+        $start_date = $target_year.'-'.$target_month.'-01';
+        if($target_month == 12) {
+            $end_date = ($target_year+1).'-01-01';
+        } else {
+            $end_date = $target_year.'-'.($target_month+1).'-01';
+        }
+        $conditions = array('Event.orgc_id' => $org['Organisation']['id'], 'Event.date >=' => $start_date, 'Event.date <' => $end_date);
+
+        //This is required to enforce the ACL (not pull directly from the DB)
+        $eventIds = $this->Event->fetchSimpleEventIds($user, array('conditions' => $conditions));
+
+        if(!empty($eventIds)) {
+            $params = array('Event.id' => $eventIds);
+            $events = $this->Event->find('all', array('conditions' => array('AND' => $params)));
+            foreach($events as $event) {
+                $events_count+= 1;
+            }
+        }
+        return $events_count;
+    }
+
+    private function filter_ghost_orgs(&$data, $orgs){
+        foreach ($data['data'] as &$item) {
+            foreach(array_keys($orgs) as $org_name) {
+                unset($item[$org_name]);
+            }
+        }
+    }
+
+    public function handler($user, $options = array())
+    {
+        $this->Log = ClassRegistry::init('Log');
+        $this->Org = ClassRegistry::init('Organisation');
+        $this->Event = ClassRegistry::init('Event');
+        $orgs = $this->Org->find('all', array( 'conditions' => array('Organisation.local' => 1)));
+        $current_month = date('n');
+        $current_year = date('Y');
+        $limit = 6; // months
+        if(!empty($options['months'])) {
+            $limit = (int) ($options['months']);
+        }
+        $offset = 0;
+        $ghost_orgs = array(); // track orgs without any contribution
+        // We start by putting all orgs_id in there:
+        foreach($orgs as $org) {
+            // We check for blocklisted orgs
+            if(!empty($options['blocklist_orgs']) && in_array($org['Organisation']['name'], $options['blocklist_orgs'])) {
+                unset($orgs[$offset]);
+            } else {
+                $ghost_orgs[$org['Organisation']['name']] = true;
+            }
+            $offset++;
+        }
+        $data = array();
+        $data['data'] = array();
+        for ($i=0; $i < $limit; $i++) {
+            $target_month = $current_month - $i;
+            $target_year = $current_year;
+            if ($target_month < 1) {
+                $target_month += 12;
+                $target_year -= 1;
+            }
+            $item = array();
+            $item ['date'] = $target_year.'-'.$target_month.'-01';
+            foreach($orgs as $org) {
+                    $count = $this->org_events_count($user, $org, $target_month, $target_year);
+                    if($options['logarithmic'] === "true" || $options['logarithmic'] === "1") {
+                        $item[$org['Organisation']['name']] = (int) round(log($count, 1.1));  // taking the logarithmic view
+                    } else if(empty($options['logarithmic']) || $options['logarithmic'] === "true" || $options['logarithmic'] === "1"){
+                        $item[$org['Organisation']['name']] = $count;
+                    }
+                    // if a positive score is detected at least once it's enough to be
+                    // considered for the graph
+                    if($count > 0) {
+                        unset($ghost_orgs[$org['Organisation']['name']]);
+                    }
+                }
+            $data['data'][] = $item;
+        }
+        $this->filter_ghost_orgs($data, $ghost_orgs);
+        return $data;
+    }
+}

app/Lib/Dashboard/OrgEvolutionLineWidget.php

@@ -0,0 +1,126 @@
+<?php
+
+class OrgEvolutionLineWidget
+{
+    public $title = 'Evolution of orgs count (filterable)';
+    public $render = 'MultiLineChart';
+    public $width = 7;
+    public $height = 6;
+    public $description = 'A linechart of organisations joining.';
+    private $tableDescription = null;
+    public $cacheLifetime = null;
+    public $autoRefreshDelay = false;
+    public $params = [
+        'filter' => 'A list of filters by organisation meta information (nationality, sector, type, name, uuid) to include. (dictionary, prepending values with ! uses them as a negation)',
+        'start_date' => 'Start date, expressed in Y-m-d format (e.g. 2012-10-01)',
+        'local' => 'Should the list only show local organisations? (boolean or list of booleans, defaults to 1. To get both sets, use [0,1])'
+    ];
+    private $validFilterKeys = [
+        'nationality',
+        'sector',
+        'type',
+        'name',
+        'uuid'
+    ];
+
+    public $placeholder =
+        '{
+    "filter": {
+        "sector": "financial"
+    },
+    "start_date": "2017-01",
+}';
+
+    private $Organisation = null;
+
+    private function timeConditions($options)
+    {
+        if (!empty($options['start_date'])) {
+            $condition = strtotime($options['start_date']);
+        } else {
+            $condition = strtotime('2012-10-01');
+        }
+        $datetime = new DateTime();
+        $datetime->setTimestamp($condition);
+        return $datetime->format('Y-m-d H:i:s');
+    }
+
+    public function handler($user, $options = array())
+    {
+        $this->Organisation = ClassRegistry::init('Organisation');
+        $params = [
+            'conditions' => [
+                'AND' => ['Organisation.local' => !isset($options['local']) ? 1 : $options['local']]
+            ],
+            'limit' => 10,
+            'recursive' => -1
+        ];
+        if (!empty($options['filter']) && is_array($options['filter'])) {
+            foreach ($this->validFilterKeys as $filterKey) {
+                if (!empty($options['filter'][$filterKey])) {
+                    if (!is_array($options['filter'][$filterKey])) {
+                        $options['filter'][$filterKey] = [$options['filter'][$filterKey]];
+                    }
+                    $tempConditionBucket = [];
+                    foreach ($options['filter'][$filterKey] as $value) {
+                        if ($value[0] === '!') {
+                            $tempConditionBucket['Organisation.' . $filterKey . ' NOT IN'][] = mb_substr($value, 1);
+                        } else {
+                            $tempConditionBucket['Organisation.' . $filterKey . ' IN'][] = $value;
+                        }
+                    }
+                    if (!empty($tempConditionBucket)) {
+                        $params['conditions']['AND'][] = $tempConditionBucket;
+                    }
+                }
+            }
+        }
+        $timeConditions = $this->timeConditions($options);
+        if ($timeConditions) {
+            $params['conditions']['AND'][] = ['Organisation.date_created >=' => $timeConditions];
+        }
+        $raw = $this->Organisation->find('all', [
+            'recursive' => -1,
+            'conditions' => $params['conditions'],
+            'fields' => ['DATE_FORMAT(date_created, "%Y-%m") AS date', 'count(id) AS count'],
+            'group' => 'MONTH(date_created), YEAR(date_created)'
+            
+        ]);
+        usort($raw, [$this, 'sortByCreationDate']);
+        $raw_padded = [];
+        $total = 0;
+        $default_start_date = empty($raw) ? '2012-10-01' : ($raw[0][0]['date'] . '-01');
+        $start = new DateTime(empty($options['start_date']) ? $default_start_date : $options['start_date']);
+        $end = new DateTime(date('Y-m') . '-01');
+        $interval = DateInterval::createFromDateString('1 month');
+        $period = new DatePeriod($start, $interval, $end);
+        foreach ($period as $dt) {
+            $raw_padded[$dt->format('Y-m') . '-01'] = 0;
+        }
+        foreach ($raw as $datapoint) {
+            $raw_padded[$datapoint[0]['date'] . '-01'] = (int)$datapoint[0]['count'];
+        }
+        $total = 0;
+        foreach ($raw_padded as $date => $count) {
+            $total += $count;
+            $raw_padded[$date] = $total;
+        }
+        $data = [];
+        foreach ($raw_padded as $date => $count) {
+            $data['data'][] = [
+              'Organisations' => (int)$count,
+              'date' => $date
+            ];
+        }
+        return $data;
+    }
+
+    private function sortByCreationDate($a, $b) {
+        if ($a[0]['date'] > $b[0]['date']) { 
+            return 1;
+        } else {
+            return -1;
+        }
+        return 0;
+    }
+}

app/Lib/Dashboard/OrganisationListWidget.php

@@ -0,0 +1,81 @@
+<?php
+class OrganisationListWidget
+{
+    public $title = 'Organisation list';
+    public $render = 'BarChart';
+    public $description = 'The countries represented via organisations on the current instance.';
+    public $width = 3;
+    public $height = 4;
+    public $params = [
+        'filter' => 'A list of filters by organisation meta information (sector, type, local (- expects a boolean or a list of boolean values)) to include. (dictionary, prepending values with ! uses them as a negation)',
+        'limit' => 'Limits the number of displayed tags. Default: 10'
+    ];
+    public $cacheLifetime = null;
+    public $autoRefreshDelay = false;
+    private $validFilterKeys = [
+        'sector',
+        'type',
+        'local'
+    ];
+    public $placeholder =
+'{
+    "filter": {
+        "type": "Member",
+        "local": [0,1]
+    }
+}';
+    private $Organisation = null;
+
+    public $countryCodes = [];
+
+    public function handler($user, $options = array())
+    {
+        App::uses('WidgetToolkit', 'Lib/Dashboard/Tools');
+        $WidgetToolkit = new WidgetToolkit();
+        $this->countryCodes = $WidgetToolkit->getCountryCodeMapping();
+        $params = [
+            'conditions' => [
+                'Nationality !=' => ''
+            ]
+        ];
+        if (!empty($options['filter']) && is_array($options['filter'])) {
+            foreach ($this->validFilterKeys as $filterKey) {
+                if (!empty($options['filter'][$filterKey])) {
+                    if (!is_array($options['filter'][$filterKey])) {
+                        $options['filter'][$filterKey] = [$options['filter'][$filterKey]];
+                    }
+                    $tempConditionBucket = [];
+                    foreach ($options['filter'][$filterKey] as $value) {
+                        if ($value[0] === '!') {
+                            $tempConditionBucket['Organisation.' . $filterKey . ' NOT IN'][] = mb_substr($value, 1);
+                        } else {
+                            $tempConditionBucket['Organisation.' . $filterKey . ' IN'][] = $value;
+                        }
+                    }
+                    if (!empty($tempConditionBucket)) {
+                        $params['conditions']['AND'][] = $tempConditionBucket;
+                    }
+                }
+            }
+        }
+        $this->Organisation = ClassRegistry::init('Organisation');
+        $orgs = $this->Organisation->find('all', [
+            'recursive' => -1,
+            'fields' => ['Organisation.nationality', 'COUNT(Organisation.nationality) AS frequency'],
+            'conditions' => $params['conditions'],
+            'group' => ['Organisation.nationality']
+        ]);
+        $results = [];
+        foreach($orgs as $org) {
+            $country = $org['Organisation']['nationality'];
+            $count = $org['0']['frequency'];
+            if (isset($this->countryCodes[$country])) {
+                $countryCode = $this->countryCodes[$country];
+                $results[$countryCode] = $count;
+            }
+        }
+        arsort($results);
+        return ['data' => $results];
+    }
+}
+?>

app/Lib/Dashboard/OrganisationMapWidget.php

@@ -0,0 +1,80 @@
+<?php
+class OrganisationMapWidget
+{
+    public $title = 'Organisation world map';
+    public $render = 'WorldMap';
+    public $description = 'The countries represented via organisations on the current instance.';
+    public $width = 3;
+    public $height = 4;
+    public $params = [
+        'filter' => 'A list of filters by organisation meta information (sector, type, local (- expects a boolean or a list of boolean values)) to include. (dictionary, prepending values with ! uses them as a negation)',
+        'limit' => 'Limits the number of displayed tags. Default: 10'
+    ];
+    public $cacheLifetime = null;
+    public $autoRefreshDelay = false;
+    private $validFilterKeys = [
+        'sector',
+        'type',
+        'local'
+    ];
+    public $placeholder =
+'{
+    "filter": {
+        "type": "Member",
+        "local": [0,1]
+    }
+}';
+    private $Organisation = null;
+
+    public $countryCodes = [];
+
+    public function handler($user, $options = array())
+    {
+        App::uses('WidgetToolkit', 'Lib/Dashboard/Tools');
+        $WidgetToolkit = new WidgetToolkit();
+        $this->countryCodes = $WidgetToolkit->getCountryCodeMapping();
+        $params = [
+            'conditions' => [
+                'Nationality !=' => ''
+            ]
+        ];
+        if (!empty($options['filter']) && is_array($options['filter'])) {
+            foreach ($this->validFilterKeys as $filterKey) {
+                if (!empty($options['filter'][$filterKey])) {
+                    if (!is_array($options['filter'][$filterKey])) {
+                        $options['filter'][$filterKey] = [$options['filter'][$filterKey]];
+                    }
+                    $tempConditionBucket = [];
+                    foreach ($options['filter'][$filterKey] as $value) {
+                        if ($value[0] === '!') {
+                            $tempConditionBucket['Organisation.' . $filterKey . ' NOT IN'][] = mb_substr($value, 1);
+                        } else {
+                            $tempConditionBucket['Organisation.' . $filterKey . ' IN'][] = $value;
+                        }
+                    }
+                    if (!empty($tempConditionBucket)) {
+                        $params['conditions']['AND'][] = $tempConditionBucket;
+                    }
+                }
+            }
+        }
+        $this->Organisation = ClassRegistry::init('Organisation');
+        $orgs = $this->Organisation->find('all', [
+            'recursive' => -1,
+            'fields' => ['Organisation.nationality', 'COUNT(Organisation.nationality) AS frequency'],
+            'conditions' => $params['conditions'],
+            'group' => ['Organisation.nationality']
+        ]);
+        $results = ['data' => [], 'scope' => 'Organisations'];
+        foreach($orgs as $org) {
+            $country = $org['Organisation']['nationality'];
+            $count = $org['0']['frequency'];
+            if (isset($this->countryCodes[$country])) {
+                $countryCode = $this->countryCodes[$country];
+                $results['data'][$countryCode] = $count;
+            }
+        }
+        return $results;
+    }
+}
+?>

app/Lib/Dashboard/RecentSightingsWidget.php

@@ -35,21 +35,21 @@ class RecentSightingsWidget
         $data = array();
         $count = 0;
 
-        foreach (JsonTool::decode($Sighting->restSearch($user, 'json', $filters)->intoString())->{'response'} as $el) {
-            $sighting = $el->{'Sighting'};
-            $event = $sighting->{'Event'};
-            $attribute = $sighting->{'Attribute'};
+        foreach (JsonTool::decode($Sighting->restSearch($user, 'json', $filters)->intoString())['response'] as $el) {
+            $sighting = $el['Sighting'];
+            $event = $sighting['Event'];
+            $attribute = $sighting['Attribute'];
 
-            if ($sighting->{'type'} == 0) $type = "Sighting";
-            elseif ($sighting->{'type'} == 1) $type = "False positive";
+            if ($sighting['type'] == 0) $type = "Sighting";
+            elseif ($sighting['type'] == 1) $type = "False positive";
             else $type = "Expiration";
 
-            $output = $attribute->{'value'} . " (id: " . $attribute->{'id'} . ") in " . $event->{'info'} . " (id: " . $event->{'id'} . ")";
+            $output = $attribute['value'] . " (id: " . $attribute['id'] . ") in " . $event['info'] . " (id: " . $event['id'] . ")";
             $data[] = array( 'title' => $type, 'value' => $output, 
                                 'html' => sprintf(
                                     ' (Event <a href="%s%s">%s</a>)',
-                                    Configure::read('MISP.baseurl') . '/events/view/', $event->{'id'},
-                                    $event->{'id'}
+                                    Configure::read('MISP.baseurl') . '/events/view/', $event['id'],
+                                    $event['id']
                                 )
                         );
             ++$count;

app/Lib/Dashboard/Tools/WidgetToolkit.php

@@ -0,0 +1,189 @@
+<?php
+
+class WidgetToolkit
+{
+
+    public function getCountryCodeMapping(): array
+    {
+        return [
+            'Afghanistan' => 'AF',
+            'Albania' => 'AL',
+            'Algeria' => 'DZ',
+            'Angola' => 'AO',
+            'Argentina' => 'AR',
+            'Armenia' => 'AM',
+            'Australia' => 'AU',
+            'Austria' => 'AT',
+            'Azerbaijan' => 'AZ',
+            'Bahamas' => 'BS',
+            'Bangladesh' => 'BD',
+            'Belarus' => 'BY',
+            'Belgium' => 'BE',
+            'Belize' => 'BZ',
+            'Benin' => 'BJ',
+            'Bhutan' => 'BT',
+            'Bolivia' => 'BO',
+            'Bosnia and Herz.' => 'BA',
+            'Botswana' => 'BW',
+            'Brazil' => 'BR',
+            'Brunei' => 'BN',
+            'Bulgaria' => 'BG',
+            'Burkina Faso' => 'BF',
+            'Burundi' => 'BI',
+            'Cambodia' => 'KH',
+            'Cameroon' => 'CM',
+            'Canada' => 'CA',
+            'Central African Rep.' => 'CF',
+            'Chad' => 'TD',
+            'Chile' => 'CL',
+            'China' => 'CN',
+            'Colombia' => 'CO',
+            'Congo' => 'CG',
+            'Costa Rica' => 'CR',
+            'Croatia' => 'HR',
+            'Cuba' => 'CU',
+            'Cyprus' => 'CY',
+            'Czech Rep.' => 'CZ',
+            'Czech Republic' => 'CZ',
+            'Côte d\'Ivoire' => 'CI',
+            'Dem. Rep. Congo' => 'CD',
+            'Dem. Rep. Korea' => 'KP',
+            'Denmark' => 'DK',
+            'Djibouti' => 'DJ',
+            'Dominican Rep.' => 'DO',
+            'Ecuador' => 'EC',
+            'Egypt' => 'EG',
+            'El Salvador' => 'SV',
+            'Eq. Guinea' => 'GQ',
+            'Eritrea' => 'ER',
+            'Estonia' => 'EE',
+            'Ethiopia' => 'ET',
+            'Falkland Is.' => 'FK',
+            'Fiji' => 'FJ',
+            'Finland' => 'FI',
+            'Fr. S. Antarctic Lands' => 'TF',
+            'France' => 'FR',
+            'Gabon' => 'GA',
+            'Gambia' => 'GM',
+            'Georgia' => 'GE',
+            'Germany' => 'DE',
+            'Ghana' => 'GH',
+            'Greece' => 'GR',
+            'Greenland' => 'GL',
+            'Guatemala' => 'GT',
+            'Guinea' => 'GN',
+            'Guinea-Bissau' => 'GW',
+            'Guyana' => 'GY',
+            'Haiti' => 'HT',
+            'Honduras' => 'HN',
+            'Hungary' => 'HU',
+            'Iceland' => 'IS',
+            'India' => 'IN',
+            'Indonesia' => 'ID',
+            'Iran' => 'IR',
+            'Iraq' => 'IQ',
+            'Ireland' => 'IE',
+            'Ireland {Republic}' => 'IE',
+            'Israel' => 'IL',
+            'Italy' => 'IT',
+            'Jamaica' => 'JM',
+            'Japan' => 'JP',
+            'Jordan' => 'JO',
+            'Kazakhstan' => 'KZ',
+            'Kenya' => 'KE',
+            'Korea' => 'KR',
+            'Kuwait' => 'KW',
+            'Kyrgyzstan' => 'KG',
+            'Lao PDR' => 'LA',
+            'Latvia' => 'LV',
+            'Lebanon' => 'LB',
+            'Lesotho' => 'LS',
+            'Liberia' => 'LR',
+            'Libya' => 'LY',
+            'Lithuania' => 'LT',
+            'Luxembourg' => 'LU',
+            'Macedonia' => 'MK',
+            'Madagascar' => 'MG',
+            'Mainland China' => 'CN',
+            'Malawi' => 'MW',
+            'Malaysia' => 'MY',
+            'Mali' => 'ML',
+            'Malta' => 'MT',
+            'Mauritania' => 'MR',
+            'Mexico' => 'MX',
+            'Moldova' => 'MD',
+            'Mongolia' => 'MN',
+            'Montenegro' => 'ME',
+            'Morocco' => 'MA',
+            'Mozamb' => 'MZ',
+            'Myanmar' => 'MM',
+            'Namibia' => 'NA',
+            'Nepal' => 'NP',
+            'Netherlands' => 'NL',
+            'New Caledonia' => 'NC',
+            'New Zealand' => 'NZ',
+            'Nicaragua' => 'NI',
+            'Niger' => 'NE',
+            'Nigeria' => 'NG',
+            'Norway' => 'NO',
+            'Oman' => 'OM',
+            'Pakistan' => 'PK',
+            'Palestine' => 'PS',
+            'Panama' => 'PA',
+            'Papua New Guinea' => 'PG',
+            'Paraguay' => 'PY',
+            'Peru' => 'PE',
+            'Philippines' => 'PH',
+            'Poland' => 'PL',
+            'Portugal' => 'PT',
+            'Puerto Rico' => 'PR',
+            'Qatar' => 'QA',
+            'Romania' => 'RO',
+            'Russia' => 'RU',
+            'Russian Federation' => 'RU',
+            'Rwanda' => 'RW',
+            'S. Sudan' => 'SS',
+            'Saudi Arabia' => 'SA',
+            'Senegal' => 'SN',
+            'Serbia' => 'RS',
+            'Sierra Leone' => 'SL',
+            'Slovakia' => 'SK',
+            'Slovenia' => 'SI',
+            'Solomon Is.' => 'SB',
+            'Somalia' => 'SO',
+            'South Africa' => 'ZA',
+            'Spain' => 'ES',
+            'Sri Lanka' => 'LK',
+            'Sudan' => 'SD',
+            'Suriname' => 'SR',
+            'Swaziland' => 'SZ',
+            'Sweden' => 'SE',
+            'Switzerland' => 'CH',
+            'Syria' => 'SY',
+            'Taiwan' => 'TW',
+            'Tajikistan' => 'TJ',
+            'Tanzania' => 'TZ',
+            'Thailand' => 'TH',
+            'Timor-Leste' => 'TL',
+            'Togo' => 'TG',
+            'Trinidad and Tobago' => 'TT',
+            'Tunisia' => 'TN',
+            'Turkey' => 'TR',
+            'Turkmenistan' => 'TM',
+            'Uganda' => 'UG',
+            'Ukraine' => 'UA',
+            'United Arab Emirates' => 'AE',
+            'United Kingdom' => 'GB',
+            'United States' => 'US',
+            'Uruguay' => 'UY',
+            'Uzbekistan' => 'UZ',
+            'Vanuatu' => 'VU',
+            'Venezuela' => 'VE',
+            'Vietnam' => 'VN',
+            'W. Sahara' => 'EH',
+            'Yemen' => 'YE',
+            'Zambia' => 'ZM',
+            'Zimbabwe' => 'ZW'
+        ];
+    }
+}

app/Lib/Dashboard/TrendingAttributesWidget.php

@@ -0,0 +1,137 @@
+<?php
+
+class TrendingAttributesWidget
+{
+    public $title = 'Trending Attribute values';
+    public $render = 'BarChart';
+    public $width = 3;
+    public $height = 4;
+    public $params = array(
+        'time_window' => 'The time window, going back in seconds, that should be included. (allows for filtering by days - example: 5d. -1 Will fetch all historic data)',
+        'exclude' => 'List of values to exclude - for example "8.8.8.8".',
+        'threshold' => 'Limits the number of displayed attribute values. Default: 10',
+        'type' => 'List of Attribute types to include',
+        'category' => 'List of Attribute categories to exclude',
+        'to_ids' => 'A list of to_ids settings accepted for the data displayed ([0], [1], [0,1])',
+        'org_filter' => 'List of organisation filters to exclude events by, based on organisation meta-data (Organisation.sector, Organisation.type, Organisation.nationality). Pre-pending a value with a "!" negates it.'
+    );
+    private $validOrgFilters = [
+        'sector',
+        'type',
+        'national',
+        'uuid',
+        'local'
+    ];
+    public $placeholder =
+    '{
+    "time_window": "7d",
+    "threshold": 15,
+    "org_filter": {
+        "sector": ["Financial"]
+    }
+}';
+    public $description = 'Widget showing the trending tags over the past x seconds, along with the possibility to include/exclude tags.';
+    public $cacheLifetime = 3;
+
+    private function getOrgList($options)
+    {
+        $organisationModel = ClassRegistry::init('Organisation');
+        if (!empty($options['org_filter']) && is_array($options['org_filter'])) {
+            foreach ($this->validOrgFilters as $filterKey) {
+                if (isset($options['org_filter'][$filterKey])) {
+                    if ($filterKey === 'local') {
+                        $tempConditionBucket['Organisation.local'] = $options['org_filter']['local'];
+                    } else {
+                        if (!is_array($options['org_filter'][$filterKey])) {
+                            $options['org_filter'][$filterKey] = [$options['org_filter'][$filterKey]];
+                        }
+                        $tempConditionBucket = [];
+                        foreach ($options['org_filter'][$filterKey] as $value) {
+                            if ($value[0] === '!') {
+                                $tempConditionBucket['Organisation.' . $filterKey . ' NOT IN'][] = mb_substr($value, 1);
+                            } else {
+                                $tempConditionBucket['Organisation.' . $filterKey . ' IN'][] = $value;
+                            }
+                        }
+                    }
+                    if (!empty($tempConditionBucket)) {
+                        $orgConditions[] = $tempConditionBucket;
+                    }
+                }
+            }
+            return $organisationModel->find('column', [
+                'recursive' => -1,
+                'conditions' => $orgConditions,
+                'fields' => ['Organisation.id']
+            ]);
+        }
+    }
+
+	public function handler($user, $options = array())
+	{
+	    /** @var Event $eventModel */
+        $attributeModel = ClassRegistry::init('Attribute');
+        $threshold = empty($options['threshold']) ? 10 : $options['threshold'];
+        if (is_string($options['time_window']) && substr($options['time_window'], -1) === 'd') {
+            $time_window = ((int)substr($options['time_window'], 0, -1)) * 24 * 60 * 60;
+        } else {
+            $time_window = empty($options['time_window']) ? (7 * 24 * 60 * 60) : (int)$options['time_window'];
+        }
+        $conditions = $time_window === -1 ? [] : ['Attribute.timestamp >=' => time() - $time_window];
+        $conditions['Attribute.deleted'] = 0;
+        $conditionsToParse = ['type', 'category', 'to_ids'];
+        foreach ($conditionsToParse as $parsedCondition) {
+            if (!empty($options[$parsedCondition])) {
+                $conditions['Attribute.' . $parsedCondition] = $options[$parsedCondition];
+            }
+        }
+        if (!empty($options['exclude'])) {
+            $conditions['Attribute.value1 NOT IN'] = $options['exclude'];
+        }
+        if (!empty($options['org_filter'])) {
+            $conditions['Event.orgc_id IN'] = $this->getOrgList($options);
+            if (empty($conditions['Event.orgc_id IN'])) {
+                $conditions['Event.orgc_id IN'] = [-1];
+            }
+        }
+        $attributeModel->virtualFields['frequency'] = 0;
+        if (!empty($user['Role']['perm_site_admin'])) {
+            $values = $attributeModel->find('all', [
+                'recursive' => -1,
+                'fields' => ['Attribute.value1', 'count(Attribute.value1) as Attribute__frequency'],
+                'group' => ['Attribute.value1'],
+                'conditions' => $conditions,
+                'contain' => ['Event.orgc_id'],
+                'order' => 'count(Attribute.value1) desc',
+                'limit' => empty($options['threshold']) ? 10 : $options['threshold']
+            ]);
+        } else {
+            $conditions['AND'][] = [
+                'OR' => [
+                    'Event.orgc_id' => $user['org_id'],
+
+                ]
+            ];
+            $values = $attributeModel->find('all', [
+                'recursive' => -1,
+                'fields' => ['Attribute.value1', 'count(Attribute.value1) as Attribute__frequency', 'Attribute.distribution', 'Attribute.sharing_group_id'],
+                'group' => 'Attribute.value1',
+                'contain' => [
+                    'Event.org_id',
+                    'Event.distribution',
+                    'Event.sharing_group_id',
+                    'Object.distribution',
+                    'Object.sharing_group_id'
+                ],
+                'conditions' => $conditions,
+                'order' => 'count(Attribute.value1) desc',
+                'limit' => empty($options['threshold']) ? 10 : $options['threshold']
+            ]);
+        }
+        $data = [];
+        foreach ($values as $value) {
+            $data[$value['Attribute']['value1']] = $value['Attribute']['frequency'];
+        }
+        return ['data' => $data];
+	}
+}

app/Lib/Dashboard/TrendingTagsWidget.php

@@ -7,56 +7,115 @@ class TrendingTagsWidget
     public $width = 3;
     public $height = 4;
     public $params = array(
-        'time_window' => 'The time window, going back in seconds, that should be included.',
+        'time_window' => 'The time window, going back in seconds, that should be included. (allows for filtering by days - example: 5d. -1 Will fetch all historic data)',
         'exclude' => 'List of substrings to exclude tags by - for example "sofacy" would exclude any tag containing sofacy.',
         'include' => 'List of substrings to include tags by - for example "sofacy" would include any tag containing sofacy.',
-        'threshold' => 'Limits the number of displayed tags. Default: 10'
+        'threshold' => 'Limits the number of displayed tags. Default: 10',
+        'filter_event_tags' => 'Filters to be applied on event tags',
+        'over_time' => 'Toggle the trending to be over time',
     );
     public $placeholder =
-'{
-    "time_window": "86400",
+    '{
+    "time_window": "7d",
     "threshold": 15,
     "exclude": ["tlp:", "pap:"],
-    "include": ["misp-galaxy:", "my-internal-taxonomy"]
+    "include": ["misp-galaxy:", "my-internal-taxonomy"],
+    "filter_event_tags": ["misp-galaxy:threat-actor="APT 29"],
 }';
     public $description = 'Widget showing the trending tags over the past x seconds, along with the possibility to include/exclude tags.';
-    public $cacheLifetime = 600;
+    public $cacheLifetime = 3;
 
 	public function handler($user, $options = array())
 	{
 	    /** @var Event $eventModel */
         $eventModel = ClassRegistry::init('Event');
         $threshold = empty($options['threshold']) ? 10 : $options['threshold'];
-        $params = [
-            'timestamp' => time() - (empty($options['time_window']) ? 8640000 : $options['time_window']),
-        ];
+        if (is_string($options['time_window']) && substr($options['time_window'], -1) === 'd') {
+            $time_window = ((int)substr($options['time_window'], 0, -1)) * 24 * 60 * 60;
+        } else {
+            $time_window = empty($options['time_window']) ? (7 * 24 * 60 * 60) : (int)$options['time_window'];
+        }
+        $params = $time_window === -1 ? [] : ['timestamp' => time() - $time_window];
+
+        if (!empty($options['filter_event_tags'])) {
+            $params['event_tags'] = $options['filter_event_tags'];
+        }
         $eventIds = $eventModel->filterEventIds($user, $params);
 
-        $tags = [];
         $tagColours = [];
-        if (!empty($eventIds)) {
-            $eventTags = $eventModel->EventTag->find('all', [
-                'conditions' => ['EventTag.event_id' => $eventIds],
-                'contain' => ['Tag' => ['fields' => ['name', 'colour']]],
-                'recursive' => -1,
-                'fields' => ['id'],
-            ]);
+        $allTags = [];
+        $this->render = $this->getRenderer($options);
+        if (!empty($options['over_time'])) {
 
-            foreach ($eventTags as $eventTag) {
-                $tagName = $eventTag['Tag']['name'];
-                if (isset($tags[$tagName])) {
-                    $tags[$tagName]++;
-                } else if ($this->checkTag($options, $tagName)) {
-                    $tags[$tagName] = 1;
-                    $tagColours[$tagName] = $eventTag['Tag']['colour'];
+            $tagOvertime = [];
+            if (!empty($eventIds)) {
+                $events = $eventModel->fetchEvent($user, [
+                    'eventid' => $eventIds,
+                    'order' => 'Event.timestamp',
+                    'metadata' => 1
+                ]);
+
+                foreach ($events as $event) {
+                    $timestamp = $event['Event']['timestamp'];
+                    $timestamp = strftime('%Y-%m-%d', $timestamp);
+                    foreach ($event['EventTag'] as $tag) {
+                        $tagName = $tag['Tag']['name'];
+                        if (isset($tagOvertime[$timestamp][$tagName])) {
+                            $tagOvertime[$timestamp][$tagName]++;
+                        } else if ($this->checkTag($options, $tagName)) {
+                            $tagOvertime[$timestamp][$tagName] = 1;
+                            $tagColours[$tagName] = $tag['Tag']['colour'];
+                            $allTags[$tagName] = $tagName;
+                        }
+                    }
                 }
             }
 
-            arsort($tags);
+            $data = [];
+            $data['data'] = [];
+            foreach($tagOvertime as $date => $tagCount) {
+                $item = [];
+                $item['date'] = $date;
+                foreach ($allTags as $tagName) {
+                    if (!empty($tagCount[$tagName])) {
+                        $item[$tagName] = $tagCount[$tagName];
+                    } else {
+                        $item[$tagName] = 0;
+                    }
+                }
+                $data['data'][] = $item;
+            }
+            uasort($data['data'], function ($a, $b) {
+                return ($a['date'] < $b['date']) ? -1 : 1;
+            });
+            $data['data'] = array_values($data['data']);
+            return $data;
+        } else {
+            $tags = [];
+            if (!empty($eventIds)) {
+                $eventTags = $eventModel->EventTag->find('all', [
+                    'conditions' => ['EventTag.event_id' => $eventIds],
+                    'contain' => ['Tag' => ['fields' => ['name', 'colour']]],
+                    'recursive' => -1,
+                    'fields' => ['id'],
+                ]);
+    
+                foreach ($eventTags as $eventTag) {
+                    $tagName = $eventTag['Tag']['name'];
+                    if (isset($tags[$tagName])) {
+                        $tags[$tagName]++;
+                    } else if ($this->checkTag($options, $tagName)) {
+                        $tags[$tagName] = 1;
+                        $tagColours[$tagName] = $eventTag['Tag']['colour'];
+                    }
+                }
+    
+                arsort($tags);
+                $data['data'] = array_slice($tags, 0, $threshold);
+                $data['colours'] = $tagColours;
+            }
+    
         }
-
-        $data['data'] = array_slice($tags, 0, $threshold);
-        $data['colours'] = $tagColours;
         return $data;
 	}
 
@@ -80,4 +139,9 @@ class TrendingTagsWidget
             return true;
         }
     }
+
+    public function getRenderer(array $options)
+    {
+        return !empty($options['over_time']) ? 'MultiLineChart' : 'BarChart';
+    }
 }

app/Lib/Dashboard/TresholdSightingsWidget.php

@@ -31,21 +31,21 @@ class TresholdSightingsWidget
 
         $data = array();
         $sightings_score = array();
-        $restSearch = JsonTool::decode($Sighting->restSearch($user, 'json', $filters)->intoString())->{'response'};
+        $restSearch = JsonTool::decode($Sighting->restSearch($user, 'json', $filters)->intoString())['response'];
 
         foreach ($restSearch as $el) {
-            $sighting = $el->{'Sighting'};
-            $attribute = $sighting->{'Attribute'};
-            $event = $sighting->{'Event'};
+            $sighting = $el['Sighting'];
+            $attribute = $sighting['Attribute'];
+            $event = $sighting['Event'];
 
-            if (!array_key_exists($attribute->{'id'}, $sightings_score)) $sightings_score[$attribute->{'id'}] = array( 'value' => $attribute->{'value'},
+            if (!array_key_exists($attribute['id'], $sightings_score)) $sightings_score[$attribute['id']] = array( 'value' => $attribute['value'],
                                                                                                                 'score' => 0,
-                                                                                                                'event_title' => $event->{'info'},
-                                                                                                                'event_id' => $event->{'id'});
+                                                                                                                'event_title' => $event['info'],
+                                                                                                                'event_id' => $event['id']);
             # Sighting
-            if ($sighting->{'type'} == 0) $sightings_score[$attribute->{'id'}]['score'] = $sightings_score[$attribute->{'id'}]['score'] - 1;
+            if ($sighting['type'] == 0) $sightings_score[$attribute['id']]['score'] = $sightings_score[$attribute['id']]['score'] - 1;
             # False Positive
-            elseif ($sighting->{'type'} == 1) $sightings_score[$attribute->{'id'}]['score'] = $sightings_score[$attribute->{'id'}]['score'] + 1; 
+            elseif ($sighting['type'] == 1) $sightings_score[$attribute['id']]['score'] = $sightings_score[$attribute['id']]['score'] + 1; 
         }
 
         foreach ($sightings_score as $attribute_id => $s) {

app/Lib/Dashboard/UsageDataWidget.php

@@ -5,40 +5,87 @@ class UsageDataWidget
     public $render = 'SimpleList';
     public $width = 2;
     public $height = 5;
-    public $params = array();
     public $description = 'Shows usage data / statistics.';
     public $cacheLifetime = false;
-    public $autoRefreshDelay = 3;
+    public $autoRefreshDelay = false;
+    public $params = [
+        'filter' => 'A list of filters by organisation meta information (nationality, sector, type, name, uuid) to include. (dictionary, prepending values with ! uses them as a negation)',
+    ];
+    private $User = null;
+    private $Event = null;
+    private $Correlation = null;
+    private $Thread = null;
+    private $AuthKey = null;
+    private $redis = null;
+    
+    private $validFilterKeys = [
+        'nationality',
+        'sector',
+        'type',
+        'name',
+        'uuid'
+    ];
 
-    public function handler($user, $options = array()){
+    private $validFields = [
+        'Events',
+        'Attributes',
+        'Attributes / event',
+        'Correlations',
+        'Active proposals',
+        'Users',
+        'Users with PGP keys',
+        'Organisations',
+        'Local organisations',
+        'Event creator orgs',
+        'Average users / org',
+        'Discussion threads',
+        'Discussion posts'
+    ];
+
+    public function handler($user, $options = array()) {
         $this->User = ClassRegistry::init('User');
-
-        $orgsCount = $this->User->Organisation->find('count');
-        $localOrgsParams['conditions']['Organisation.local'] = 1;
-        $localOrgsCount = $this->User->Organisation->find('count', $localOrgsParams);
-
-        $thisMonth = strtotime('first day of this month');
+        $this->redis = $this->User->setupRedis();
+        if (!$this->redis) {
+            throw new NotFoundException(__('No redis connection found.'));
+        }
         $this->Event = ClassRegistry::init('Event');
-        $eventsCount = $this->Event->find('count', array('recursive' => -1));
-        $eventsCountMonth = $this->Event->find('count', array('conditions' => array('Event.timestamp >' => $thisMonth), 'recursive' => -1));
-
-        $this->Attribute = ClassRegistry::init('Attribute');
-        $attributesCount = $this->Attribute->find('count', array('conditions' => array('Attribute.deleted' => 0), 'recursive' => -1));
-        $attributesCountMonth = $this->Attribute->find('count', array('conditions' => array('Attribute.timestamp >' => $thisMonth, 'Attribute.deleted' => 0), 'recursive' => -1));
-        $attributesPerEvent = round($attributesCount / $eventsCount);
-
-        $this->Correlation = ClassRegistry::init('Correlation');
-        $correlationsCount = $this->Correlation->find('count', array('recursive' => -1)) / 2;
-
-        $proposalsCount = $this->Event->ShadowAttribute->find('count', array('recursive' => -1, 'conditions' => array('deleted' => 0)));
-
-        $usersCount = $this->User->find('count', array('recursive' => -1));
-        $usersCountPgp = $this->User->find('count', array('recursive' => -1, 'conditions' => array('User.gpgkey !=' => '')));
-        $usersCountPgpPercentage = round(100* ($usersCountPgp / $usersCount), 1);
-        $contributingOrgsCount = $this->Event->find('count', array('recursive' => -1, 'group' => array('Event.orgc_id')));
-        $averageUsersPerOrg = round($usersCount / $localOrgsCount, 1);
-
         $this->Thread = ClassRegistry::init('Thread');
+        $this->Correlation = ClassRegistry::init('Correlation');
+        $thisMonth = strtotime('first day of this month');
+        $orgConditions = [];
+        $orgIdList = null;
+        if (!empty($options['filter']) && is_array($options['filter'])) {
+            foreach ($this->validFilterKeys as $filterKey) {
+                if (!empty($options['filter'][$filterKey])) {
+                    if (!is_array($options['filter'][$filterKey])) {
+                        $options['filter'][$filterKey] = [$options['filter'][$filterKey]];
+                    }
+                    $tempConditionBucket = [];
+                    foreach ($options['filter'][$filterKey] as $value) {
+                        if ($value[0] === '!') {
+                            $tempConditionBucket['Organisation.' . $filterKey . ' NOT IN'][] = mb_substr($value, 1);
+                        } else {
+                            $tempConditionBucket['Organisation.' . $filterKey . ' IN'][] = $value;
+                        }
+                    }
+                    if (!empty($tempConditionBucket)) {
+                        $orgConditions[] = $tempConditionBucket;
+                    }
+                }
+            }
+            $orgIdList = $this->User->Organisation->find('column', [
+                'recursive' => -1,
+                'conditions' => $orgConditions,
+                'fields' => ['Organisation.id']
+            ]);
+        }
+        $eventsCount = $this->getEventsCount($orgConditions, $orgIdList, $thisMonth);
+        $attributesCount = $this->getAttributesCount($orgConditions, $orgIdList, $thisMonth);
+        $usersCount = $this->getUsersCount($orgConditions, $orgIdList, $thisMonth);
+        $usersCountPgp = $this->getUsersCountPgp($orgConditions, $orgIdList, $thisMonth);
+        $localOrgsCount = $this->getLocalOrgsCount($orgConditions, $orgIdList, $thisMonth);
+
+
         $threadCount = $this->Thread->find('count', array('conditions' => array('Thread.post_count >' => 0), 'recursive' => -1));
         $threadCountMonth = $this->Thread->find('count', array('conditions' => array('Thread.date_created >' => date("Y-m-d H:i:s", $thisMonth), 'Thread.post_count >' => 0), 'recursive' => -1));
 
@@ -47,21 +94,69 @@ class UsageDataWidget
 
         //Monhtly data is not added to the widget at the moment, could optionally add these later and give user choice?
 
-        $statistics = array(
-            array('title' => 'Events', 'value' => $eventsCount),
-            array('title' => 'Attributes', 'value' => $attributesCount),
-            array('title' => 'Attributes / event', 'value' => $attributesPerEvent),
-            array('title' => 'Correlations', 'value' => $correlationsCount),
-            array('title' => 'Active proposals', 'value' => $proposalsCount),
-            array('title' => 'Users', 'value' => $usersCount),
-            array('title' => 'Users with PGP keys', 'value' => $usersCountPgp . ' (' . $usersCountPgpPercentage . '%)'),
-            array('title' => 'Organisations', 'value' => $orgsCount),
-            array('title' => 'Local organisations', 'value' => $localOrgsCount),
-            array('title' => 'Event creator orgs', 'value' => $contributingOrgsCount),
-            array('title' => 'Average users / org', 'value' => $averageUsersPerOrg),
-            array('title' => 'Discussions threads', 'value' => $threadCount),
-            array('title' => 'Discussion posts', 'value' => $postCount)
-        );
+        $statistics = [
+            'Events' => [
+                'title' => 'Events',
+                'value' => $eventsCount,
+                'change' => $this->getEventsCountMonth($orgConditions, $orgIdList, $thisMonth)
+            ],
+            'Attributes' => [
+                'title' => 'Attributes',
+                'value' => $attributesCount,
+                'change' => $this->getAttributesCountMonth($orgConditions, $orgIdList, $thisMonth)
+            ],
+            'Attributes / event' => [
+                'title' => 'Attributes / event',
+                'value' => $eventsCount ? round($attributesCount / $eventsCount) : 0
+            ],
+            'Correlations' => [
+                'title' => 'Correlations',
+                'value' => $this->getCorrelationsCount($orgConditions, $orgIdList, $thisMonth)
+            ],
+            'Active proposals' => [
+                'title' => 'Active proposals',
+                'value' => $this->getProposalsCount($orgConditions, $orgIdList, $thisMonth)
+            ],
+            'Users' => [
+                'title' => 'Users',
+                'value' => $usersCount,
+                'change' => $this->getUsersCountMonth($orgConditions, $orgIdList, $thisMonth)
+            ],
+            'Users with PGP keys' => [
+                'title' => 'Users with PGP keys',
+                'value' => sprintf(
+                    '%s (%s %%)',
+                    $usersCountPgp,
+                    $usersCount ? round(100* ($usersCountPgp / $usersCount), 1) : 0
+                )
+            ],
+            'Organisations' => [
+                'title' => 'Organisations',
+                'value' => $this->getOrgsCount($orgConditions, $orgIdList, $thisMonth),
+                'change' => $this->getOrgsCountMonth($orgConditions, $orgIdList, $thisMonth)
+            ],
+            'Local organisations' => [
+                'title' => 'Local organisations',
+                'value' => $localOrgsCount,
+                'change' => $this->getLocalOrgsCountMonth($orgConditions, $orgIdList, $thisMonth)
+            ],
+            'Event creator orgs' => [
+                'title' => 'Event creator orgs', 'value' => $this->getContributingOrgsCount($orgConditions, $orgIdList, $thisMonth)
+            ],
+            'Average users / org' => [
+                'title' => 'Average users / org', 'value' => round($usersCount / $localOrgsCount, 1)
+            ],
+            'Discussion threads' => [
+                'title' => 'Discussions threads',
+                'value' => $this->getThreadsCount($orgConditions, $orgIdList, $thisMonth),
+                'change' => $this->getThreadsCountMonth($orgConditions, $orgIdList, $thisMonth)
+            ],
+            'Discussion posts' => [
+                'title' => 'Discussion posts',
+                'value' => $this->getPostsCount($orgConditions, $orgIdList, $thisMonth),
+                'change' => $this->getPostsCountMonth($orgConditions, $orgIdList, $thisMonth)
+            ]
+        ];
         if(!empty(Configure::read('Security.advanced_authkeys'))){
             $this->AuthKey = ClassRegistry::init('AuthKey');
             $authkeysCount = $this->AuthKey->find('count', array('recursive' => -1));
@@ -70,6 +165,239 @@ class UsageDataWidget
         return $statistics;
     }
 
+    private function getEventsCount($orgConditions, $orgIdList, $thisMonth)
+    {
+        $conditions = [];
+        if (!empty($orgIdList)) {
+            $conditions['AND'][] = ['Event.orgc_id IN' => $orgIdList];
+        }
+        return $this->Event->find('count', [
+            'recursive' => -1,
+            'conditions' => $conditions
+        ]);
+    }
+
+    private function getCorrelationsCount($orgConditions, $orgIdList, $thisMonth)
+    {
+        $conditions = [];
+        if (!empty($orgIdList)) {
+            $conditions['AND']['OR'][] = ['Correlation.org_id IN' => $orgIdList];
+            $conditions['AND']['OR'][] = ['Correlation.1_org_id IN' => $orgIdList];
+        }
+        return $this->Correlation->find('count', [
+            'recursive' => -1,
+            'conditions' => $conditions
+        ]);
+    }
+
+    private function getEventsCountMonth($orgConditions, $orgIdList, $thisMonth)
+    {
+        $conditions = ['Event.timestamp >' => $thisMonth];
+        if (!empty($orgIdList)) {
+            $conditions['AND'][] = ['Event.orgc_id IN' => $orgIdList];
+        }
+        return $this->Event->find('count', [
+            'conditions' => $conditions,
+            'recursive' => -1
+        ]);
+    }
+
+    private function getAttributesCount($orgConditions, $orgIdList, $thisMonth)
+    {
+        $conditions = ['Attribute.deleted' => 0];
+        if (!empty($orgIdList)) {
+            $conditions['AND'][] = ['Event.orgc_id IN' => $orgIdList];
+        }
+        $hash = hash('sha256', json_encode($orgIdList));
+        $count = $this->redis->get('misp:dashboard:attribute_count:' . $hash);
+        if (empty($count)) {
+            $count = $this->Event->Attribute->find('count', [
+                'conditions' => $conditions,
+                'contain' => ['Event'],
+                'recursive' => -1
+            ]);
+            $this->redis->setEx('misp:dashboard:attribute_count:' . $hash, 3600, $count);
+        }
+        return $count;
+    }
+
+    private function getAttributesCountMonth($orgConditions, $orgIdList, $thisMonth)
+    {
+        $conditions = ['Attribute.timestamp >' => $thisMonth, 'Attribute.deleted' => 0];
+        if (!empty($orgIdList)) {
+            $conditions['AND'][] = ['Event.orgc_id IN' => $orgIdList];
+        }
+        return $this->Event->Attribute->find('count', [
+            'conditions' => $conditions,
+            'contain' => 'Event.orgc_id',
+            'recursive' => -1
+        ]);
+    }
+
+    private function getOrgsCount($orgConditions, $orgIdList, $thisMonth)
+    {
+        return $this->User->Organisation->find('count', [
+            'conditions' => [
+                'AND' => $orgConditions
+            ]
+        ]);
+    }
+
+    private function getOrgsCountMonth($orgConditions, $orgIdList, $thisMonth)
+    {
+        $datetime = new DateTime();
+        $datetime->setTimestamp($thisMonth);
+        $thisMonth = $datetime->format('Y-m-d H:i:s');
+        return $this->User->Organisation->find('count', [
+            'conditions' => [
+                'AND' => $orgConditions,
+                'Organisation.date_created >' => $thisMonth
+            ]
+        ]);
+    }
+
+    private function getLocalOrgsCount($orgConditions, $orgIdList, $thisMonth)
+    {
+        return $this->User->Organisation->find('count', [
+            'conditions' => [
+                'Organisation.local' => 1,
+                'AND' => $orgConditions
+            ]
+        ]);
+    }
+
+    private function getLocalOrgsCountMonth($orgConditions, $orgIdList, $thisMonth)
+    {
+        $datetime = new DateTime();
+        $datetime->setTimestamp($thisMonth);
+        $thisMonth = $datetime->format('Y-m-d H:i:s');
+        return $this->User->Organisation->find('count', [
+            'conditions' => [
+                'Organisation.local' => 1,
+                'AND' => $orgConditions,
+                'Organisation.date_created >' => $thisMonth
+            ]
+        ]);
+    }
+
+    private function getProposalsCount($orgConditions, $orgIdList, $thisMonth)
+    {
+        $conditions = ['deleted' => 0];
+        if (!empty($orgIdList)) {
+            $conditions['ShadowAttribute.org_id IN'] = $orgIdList;
+        }
+        return $this->Event->ShadowAttribute->find('count', [
+            'recursive' => -1,
+            'conditions' => $conditions
+        ]);
+    }
+
+    private function getUsersCount($orgConditions, $orgIdList, $thisMonth)
+    {
+        $conditions = [];
+        if (!empty($orgIdList)) {
+            $conditions['User.org_id IN'] = $orgIdList;
+        }
+        return $this->User->find('count', [
+            'recursive' => -1,
+            'conditions' => $conditions
+        ]);
+    }
+
+    private function getUsersCountMonth($orgConditions, $orgIdList, $thisMonth)
+    {
+        $conditions = ['User.date_created >' => $thisMonth];
+        if (!empty($orgIdList)) {
+            $conditions['User.org_id IN'] = $orgIdList;
+        }
+        return $this->User->find('count', [
+            'recursive' => -1,
+            'conditions' => $conditions
+        ]);
+    }
+
+    private function getUsersCountPgp($orgConditions, $orgIdList, $thisMonth)
+    {
+        $conditions = ['User.gpgkey !=' => ''];
+        if (!empty($orgIdList)) {
+            $conditions['User.org_id IN'] = $orgIdList;
+        }
+        return $this->User->find('count', [
+            'recursive' => -1,
+            'conditions' => $conditions
+        ]);
+    }
+
+    private function getContributingOrgsCount($orgConditions, $orgIdList, $thisMonth)
+    {
+        $conditions = [];
+        if ($orgConditions) {
+            $conditions['AND'][] = ['Event.orgc_id IN' => $orgIdList];
+        }
+        return $this->Event->find('count', [
+            'recursive' => -1,
+            'group' => ['Event.orgc_id'],
+            'conditions' => $conditions
+        ]);
+    }
+
+    private function getThreadsCount($orgConditions, $orgIdList, $thisMonth)
+    {
+        $conditions = ['Thread.post_count >' => 0];
+        if ($orgConditions) {
+            $conditions['AND'][] = ['Thread.org_id IN' => $orgIdList];
+        }
+        return $this->Thread->find('count', [
+            'conditions' => $conditions,
+            'recursive' => -1
+        ]);
+    }
+
+    private function getThreadsCountMonth($orgConditions, $orgIdList, $thisMonth)
+    {
+        $conditions = [
+            'Thread.post_count >' => 0,
+            'Thread.date_created >=' => $thisMonth
+        ];
+        if ($orgConditions) {
+            $conditions['AND'][] = ['Thread.org_id IN' => $orgIdList];
+        }
+        return $this->Thread->find('count', [
+            'conditions' => $conditions,
+            'recursive' => -1
+        ]);
+    }
+
+    private function getPostsCount($orgConditions, $orgIdList, $thisMonth)
+    {
+        $conditions = [];
+        if ($orgConditions) {
+            $conditions['AND'][] = ['User.org_id IN' => $orgIdList];
+        }
+        return $this->Thread->Post->find('count', [
+            'conditions' => $conditions,
+            'contain' => ['User.org_id'],
+            'recursive' => -1
+        ]);
+    }
+
+    private function getPostsCountMonth($orgConditions, $orgIdList, $thisMonth)
+    {
+        $conditions = [
+            'Post.date_created >=' => $thisMonth
+        ];
+        if ($orgConditions) {
+            $conditions['AND'][] = ['User.org_id IN' => $orgIdList];
+        }
+        return $this->Thread->Post->find('count', [
+            'conditions' => $conditions,
+            'contain' => ['User.org_id'],
+            'recursive' => -1
+        ]);
+    }
+
+    
+/* There is nothing sensitive in here.
     public function checkPermissions($user)
     {
         if (empty($user['Role']['perm_site_admin'])) {
@@ -77,4 +405,5 @@ class UsageDataWidget
         }
         return true;
     }
+*/
 }

app/Lib/Dashboard/UserContributionToplistWidget.php

@@ -0,0 +1,138 @@
+<?php
+class UserContributionToplistWidget
+{
+    public $title = 'Contributor Top List (Users)';
+    public $render = 'BarChart';
+    public $description = 'The top contributors (users) in a selected time frame.';
+    public $width = 3;
+    public $height = 4;
+    public $params = [
+        'days' => 'How many days back should the list go - for example, setting 7 will only show contributions in the past 7 days. (integer)',
+        'month' => 'Who contributed most this month? (boolean)',
+        'previous_month' => 'Who contributed most the previous, finished month? (boolean)',
+        'year' => 'Which contributed most this year? (boolean)',
+        'filter' => 'A list of filters by organisation meta information (nationality, sector, type, name, uuid, local (- expects a boolean or a list of boolean values)) to include. (dictionary, prepending values with ! uses them as a negation)',
+        'limit' => 'Limits the number of displayed tags. Default: 10'
+    ];
+    public $cacheLifetime = null;
+    public $autoRefreshDelay = false;
+    private $validFilterKeys = [
+        'nationality',
+        'sector',
+        'type',
+        'name',
+        'uuid'
+    ];
+    public $placeholder =
+'{
+    "days": "7d",
+    "threshold": 15,
+    "filter": {
+        "sector": "Financial"
+    }
+}';
+    private $Org = null;
+    private $Event = null;
+
+
+    private function timeConditions($options)
+    {
+        $limit = empty($options['limit']) ? 10 : $options['limit'];
+        if (!empty($options['days'])) {
+            $condition = strtotime(sprintf("-%s days", $options['days']));
+        } else if (!empty($options['month'])) {
+            $condition = strtotime('first day of this month 00:00:00', time());
+        } else if (!empty($options['previous_month'])) {
+            $condition = strtotime('first day of previous month 00:00:00', time());
+            $end_condition = strtotime('last day of last month 23:59:59', time());
+        } else if (!empty($options['year'])) {
+            $condition = strtotime('first day of this year 00:00:00', time());
+        } else {
+            return null;
+        }
+        $conditions = [];
+        if (!empty($condition)) {
+            $datetime = new DateTime();
+            $datetime->setTimestamp($condition);
+            $conditions['Event.timestamp >='] = $datetime->format('Y-m-d H:i:s');
+        }
+        if (!empty($end_condition)) {
+            $datetime = new DateTime();
+            $datetime->setTimestamp($end_condition);
+            $conditions['Event.timestamp <='] = $datetime->format('Y-m-d H:i:s');
+        }
+        return $conditions;
+    }
+
+
+    public function handler($user, $options = array())
+    {
+        $params = ['conditions' => []];
+        $timeConditions = $this->timeConditions($options);
+        if ($timeConditions) {
+            $params['conditions']['AND'][] = $timeConditions;
+        }
+        if (!empty($options['filter']) && is_array($options['filter'])) {
+            foreach ($this->validFilterKeys as $filterKey) {
+                if (!empty($options['filter'][$filterKey])) {
+                    if (!is_array($options['filter'][$filterKey])) {
+                        $options['filter'][$filterKey] = [$options['filter'][$filterKey]];
+                    }
+                    $tempConditionBucket = [];
+                    foreach ($options['filter'][$filterKey] as $value) {
+                        if ($value[0] === '!') {
+                            $tempConditionBucket['Organisation.' . $filterKey . ' NOT IN'][] = mb_substr($value, 1);
+                        } else {
+                            $tempConditionBucket['Organisation.' . $filterKey . ' IN'][] = $value;
+                        }
+                    }
+                    if (!empty($tempConditionBucket)) {
+                        $params['conditions']['AND'][] = $tempConditionBucket;
+                    }
+                }
+            }
+        }
+        if (isset($options['filter']['local'])) {
+            $params['conditions']['AND']['local'] = $options['filter']['local'];
+        }
+
+        $this->Org = ClassRegistry::init('Organisation');
+        $org_ids = $this->Org->find('list', [
+            'fields' => ['Organisation.id', 'Organisation.name'],
+            'conditions' => $params['conditions']
+        ]);
+        $userConditions = [];
+        if (!empty($org_ids)) {
+            $userConditions =  ['User.org_id IN' => array_keys($org_ids)];
+        }
+        $user_ids = $this->Org->User->find('list', [
+            'fields' => ['User.id', 'User.email'],
+            'conditions' => $userConditions
+        ]);
+        $conditions = empty($user_ids) ? [] : ['Event.user_id IN' => array_keys($user_ids)];
+        $this->Event = ClassRegistry::init('Event');
+        $this->Event->virtualFields['frequency'] = 0;
+        $users = $this->Event->find('all', [
+            'recursive' => -1,
+            'fields' => ['user_id', 'count(Event.user_id) as Event__frequency'],
+            'group' => ['user_id'],
+            'conditions' => $conditions,
+            'order' => 'count(Event.user_id) desc',
+            'limit' => empty($options['limit']) ? 10 : $options['limit']
+        ]);
+        $results = [];
+        foreach($users as $user) {
+            $results[$user_ids[$user['Event']['user_id']]] = $user['Event']['frequency'];
+        }
+        return ['data' => $results];
+    }
+
+    public function checkPermissions($user)
+    {
+        if (empty(Configure::read('Security.disclose_user_emails')) && empty($user['Role']['perm_site_admin'])) {
+            return false;
+        }
+        return true;
+    }
+}
+?>

app/Lib/Dashboard/WhoamiWidget.php

@@ -28,7 +28,7 @@ class WhoamiWidget
             array('title' => 'Email', 'value' => $user['email']),
             array('title' => 'Role', 'value' => $user['Role']['name']),
             array('title' => 'Organisation', 'value' => $user['Organisation']['name']),
-            array('title' => 'IP', 'value' => empty($_SERVER['HTTP_X_FORWARDED_FOR']) ? $_SERVER['REMOTE_ADDR'] : $_SERVER['HTTP_X_FORWARDED_FOR']),
+            array('title' => 'IP', 'value' => $this->Log->_remoteIp()),
             array('title' => 'Last logins', 'value' => $entries)
         );
 	}

app/Lib/Export/Stix2Export.php

@@ -4,7 +4,7 @@ App::uses('StixExport', 'Export');
 class Stix2Export extends StixExport
 {
     protected $__attributes_limit = 15000;
-    protected $__default_version = '2.0';
+    protected $__default_version = '2.1';
     protected $__sane_versions = array('2.0', '2.1');
 
     protected function __initiate_framing_params()

app/Lib/Export/StixExport.php

@@ -83,6 +83,12 @@ abstract class StixExport
         if ($this->__empty_file) {
             $this->__tmp_file->close();
             $this->__tmp_file->delete();
+            if (empty($this->__filenames)) {
+                $framing = $this->getFraming();
+                $tmpFile = new TmpFileTool();
+                $tmpFile->write($framing['header'] . $framing['footer']);
+                return $tmpFile;
+            }
         } else {
             if (!empty($this->__event_galaxies)) {
                 $this->__write_event_galaxies();

app/Lib/Tools/AttachmentTool.php

@@ -157,7 +157,7 @@ class AttachmentTool
             $filepath = $this->attachmentDir() . DS . $path;
             $file = new File($filepath);
             if (!is_file($file->path)) {
-                throw new NotFoundException("File '$filepath' does not exists.");
+                throw new NotFoundException("File '$filepath' does not exist.");
             }
         }
 

app/Lib/Tools/AttributeValidationTool.php

@@ -202,7 +202,7 @@ class AttributeValidationTool
                     $value = substr($value, 2); // remove 'AS'
                 }
                 if (strpos($value, '.') !== false) { // maybe value is in asdot notation
-                    $parts = explode('.', $value);
+                    $parts = explode('.', $value, 2);
                     if (self::isPositiveInteger($parts[0]) && self::isPositiveInteger($parts[1])) {
                         return $parts[0] * 65536 + $parts[1];
                     }
@@ -224,7 +224,6 @@ class AttributeValidationTool
         switch ($type) {
             case 'md5':
             case 'imphash':
-            case 'telfhash':
             case 'sha1':
             case 'sha224':
             case 'sha256':
@@ -255,6 +254,11 @@ class AttributeValidationTool
                     return true;
                 }
                 return __('Checksum has an invalid length or format (expected: at least 35 hexadecimal characters, optionally starting with t1 instead of hexadecimal characters). Please double check the value or select type "other".');
+            case 'telfhash':
+                if (self::isTelfhashValid($value)) {
+                    return true;
+                }
+                return __('Checksum has an invalid length or format (expected: %s or %s hexadecimal characters). Please double check the value or select type "other".', 70, 72);
             case 'pehash':
                 if (self::isHashValid('pehash', $value)) {
                     return true;
@@ -412,12 +416,12 @@ class AttributeValidationTool
                 }
                 return __('Email address has an invalid format. Please double check the value or select type "other".');
             case 'vulnerability':
-                if (preg_match("#^(CVE-)[0-9]{4}(-)[0-9]{4,}$#", $value)) {
+                if (preg_match("#^CVE-[0-9]{4}-[0-9]{4,}$#", $value)) {
                     return true;
                 }
                 return __('Invalid format. Expected: CVE-xxxx-xxxx...');
             case 'weakness':
-                if (preg_match("#^(CWE-)[0-9]{1,}$#", $value)) {
+                if (preg_match("#^CWE-[0-9]+$#", $value)) {
                     return true;
                 }
                 return __('Invalid format. Expected: CWE-x...');
@@ -526,6 +530,7 @@ class AttributeValidationTool
             case 'favicon-mmh3':
             case 'chrome-extension-id':
             case 'mobile-application-id':
+            case 'azure-application-id':
             case 'named pipe':
                 if (strpos($value, "\n") !== false) {
                     return __('Value must not contain new line character.');
@@ -582,6 +587,28 @@ class AttributeValidationTool
         throw new InvalidArgumentException("Unknown type $type.");
     }
 
+    /**
+     * This method will generate all valid types for given value.
+     * @param array $types Typos to check
+     * @param array $compositeTypes Composite types
+     * @param string $value Values to check
+     * @return array
+     */
+    public static function validTypesForValue(array $types, array $compositeTypes, $value)
+    {
+        $possibleTypes = [];
+        foreach ($types as $type) {
+            if (in_array($type, $compositeTypes, true) && substr_count($value, '|') !== 1) {
+                continue; // value is not in composite format
+            }
+            $modifiedValue = AttributeValidationTool::modifyBeforeValidation($type, $value);
+            if (AttributeValidationTool::validate($type, $modifiedValue) === true) {
+                $possibleTypes[] = $type;
+            }
+        }
+        return $possibleTypes;
+    }
+
     /**
      * @param string $value
      * @return bool
@@ -612,6 +639,15 @@ class AttributeValidationTool
         return strlen($value) > 35 && ctype_xdigit($value);
     }
 
+    /**
+     * @param string $value
+     * @return bool
+     */
+    private static function isTelfhashValid($value)
+    {
+        return strlen($value) == 70 || strlen($value) == 72;
+    }
+
 
     /**
      * @param string $type

app/Lib/Tools/BackgroundJobsTool.php

@@ -161,7 +161,7 @@ class BackgroundJobsTool
             }
             RedisTool::unlink($this->RedisConnection, self::DATA_CONTENT_PREFIX . ':' . $uuid);
             return $data;
-        } else if ($path[0] === '/') { // deprecated storage location when not full path is provided
+        } else if ($path[0] !== '/') { // deprecated storage location when not full path is provided
             $path = APP . 'tmp/cache/ingest' . DS . $path;
         }
         return JsonTool::decode(FileAccessTool::readAndDelete($path));

app/Lib/Tools/CustomPaginationTool.php

@@ -27,6 +27,8 @@ class CustomPaginationTool
                 $params['options'][$v] = $options[$v];
             }
         }
+        $params['page'] = is_numeric($params['page']) ? $params['page'] : 1;
+        $params['limit'] = is_numeric($params['limit']) ? $params['limit'] : 60;
         $maxPage = floor($params['count'] / $params['limit']);
         if ($params['count'] % $params['limit'] != 0) {
             $maxPage += 1;

app/Lib/Tools/EventTimelineTool.php

@@ -57,6 +57,12 @@
                 $event['Attribute'] = array();
             }
 
+            if (!empty($fullevent[0]['Sighting'])) {
+                $event['Sighting'] = $fullevent[0]['Sighting'];
+            } else {
+                $event['Sighting'] = array();
+            }
+
             return $event;
         }
 
@@ -81,6 +87,11 @@
                 $attribute = array();
             }
 
+            $sightingsAttributeMap = [];
+            foreach ($event['Sighting'] as $sighting) {
+                $sightingsAttributeMap[$sighting['attribute_id']][] = $sighting['date_sighting'];
+            }
+
             // extract links and node type
             foreach ($attribute as $attr) {
                 $toPush = array(
@@ -93,6 +104,7 @@
                     'first_seen' => $attr['first_seen'],
                     'last_seen' => $attr['last_seen'],
                     'attribute_type' => $attr['type'],
+                    'date_sighting' => $sightingsAttributeMap[$attr['id']] ?? [],
                     'is_image' => $this->__eventModel->Attribute->isImage($attr),
                 );
                 $this->__json['items'][] = $toPush;
@@ -134,6 +146,7 @@
                         'group' => 'object_attribute',
                         'timestamp' => $obj_attr['timestamp'],
                         'attribute_type' => $obj_attr['type'],
+                        'date_sighting' => $sightingsAttributeMap[$attr['id']] ?? [],
                         'is_image' => $this->__eventModel->Attribute->isImage($obj_attr),
                     );
                     $toPush_obj['Attribute'][] = $toPush_attr;

app/Lib/Tools/FileAccessTool.php

@@ -107,8 +107,14 @@ class FileAccessTool
         }
 
         if (file_put_contents($file, $content, LOCK_EX | (!empty($append) ? FILE_APPEND : 0)) === false) {
-            $freeSpace = disk_free_space($dir);
-            throw new Exception("An error has occurred while attempt to write to file `$file`. Maybe not enough space? ($freeSpace bytes left)");
+            if (file_exists($file) && !is_writable($file)) {
+                $errorMessage = 'File is not writeable.';
+            } else {
+                $freeSpace = disk_free_space($dir);
+                $errorMessage = "Maybe not enough space? ($freeSpace bytes left)";
+            }
+
+            throw new Exception("An error has occurred while attempt to write to file `$file`. $errorMessage");
         }
     }
 

app/Lib/Tools/RedisTool.php

@@ -1,7 +1,7 @@
 <?php
 class RedisTool
 {
-    const COMPRESS_MIN_LENGTH = 200,
+    const COMPRESS_MIN_LENGTH = 256,
         BROTLI_HEADER = "\xce\xb2\xcf\x81",
         ZSTD_HEADER = "\x28\xb5\x2f\xfd";
 
@@ -26,12 +26,18 @@ class RedisTool
         }
 
         $host = Configure::read('MISP.redis_host') ?: '127.0.0.1';
-        $port = Configure::read('MISP.redis_port') ?: 6379;
+        $socket = false;
+        if ($host[0] === '/') {
+            $socket = $host;
+        } else {
+            $port = Configure::read('MISP.redis_port') ?: 6379;
+        }
         $database = Configure::read('MISP.redis_database') ?: 13;
         $pass = Configure::read('MISP.redis_password');
 
         $redis = new Redis();
-        if (!$redis->connect($host, (int) $port)) {
+        $connection = empty($socket) ? $redis->connect($host, (int) $port) : $redis->connect($host);
+        if (!$connection) {
             throw new Exception("Could not connect to Redis: {$redis->getLastError()}");
         }
         if (!empty($pass)) {

app/Lib/Tools/SecurityAudit.php

@@ -138,7 +138,7 @@ class SecurityAudit
         if (!Configure::read('MISP.log_new_audit')) {
             $output['Logging'][] = [
                 'hint',
-                __('New audit log stores more information, like used authkey ID or request ID that can help when analysing or correlating audit logs.'),
+                __('New audit log stores more information, like used authkey ID or request ID that can help when analysing or correlating audit logs. Set `MISP.log_new_audit` to `true` to enable.'),
             ];
         }
 

app/Lib/Tools/SyncTool.php

@@ -2,6 +2,9 @@
 
 class SyncTool
 {
+
+    const ALLOWED_CERT_FILE_EXTENSIONS = ['pem', 'crt'];
+
     /**
      * Take a server as parameter and return a HttpSocket object using the ssl options defined in the server settings
      * @param array|null $server
@@ -15,10 +18,10 @@ class SyncTool
         $params = ['compress' => true];
         if (!empty($server)) {
             if (!empty($server[$model]['cert_file'])) {
-                $params['ssl_cafile'] = APP . "files" . DS . "certs" . DS . $server[$model]['id'] . '.pem';
+                $params['ssl_cafile'] = APP . "files" . DS . "certs" . DS . $server[$model]['cert_file'];
             }
             if (!empty($server[$model]['client_cert_file'])) {
-                $params['ssl_local_cert'] = APP . "files" . DS . "certs" . DS . $server[$model]['id'] . '_client.pem';
+                $params['ssl_local_cert'] = APP . "files" . DS . "certs" . DS . $server[$model]['client_cert_file'];
             }
             if (!empty($server[$model]['self_signed'])) {
                 $params['ssl_allow_self_signed'] = true;

app/Lib/Tools/WorkflowFormatConverterTool.php

@@ -44,6 +44,17 @@ class WorkflowFormatConverterTool
     {
         $converted = [];
         $converted = JSONConverterTool::convert($event, false, true);
+        $eventTags = !empty($converted['Event']['Tag']) ? $converted['Event']['Tag'] : [];
+        if (!empty($converted['Event']['Attribute'])) {
+            foreach ($converted['Event']['Attribute'] as $i => $attribute) {
+                $converted['Event']['Attribute'][$i] = self::__propagateTagToAttributes($attribute, $eventTags);
+            }
+        }
+        if (!empty($converted['Event']['Object'])) {
+            foreach ($converted['Event']['Object'] as $i => $object) {
+                $converted['Event']['Object'][$i] = self::__propagateTagToObjectAttributes($object, $eventTags);
+            }
+        }
         return $converted;
     }
 
@@ -101,6 +112,33 @@ class WorkflowFormatConverterTool
         return $converted;
     }
 
+    private static function __propagateTagToAttributes(array $attribute, array $eventTags): array
+    {
+        $allTags = [];
+        if (!empty($eventTags)) {
+            foreach ($eventTags as $eventTag) {
+                $eventTag['inherited'] = true;
+                $allTags[] = $eventTag;
+            }
+        }
+        if (!empty($attribute['Tag'])) {
+            foreach ($attribute['Tag'] as $tag) {
+                $tag['inherited'] = false;
+                $allTags[] = $tag;
+            }
+        }
+        $attribute['_allTags'] = $allTags;
+        return $attribute;
+    }
+
+    private static function __propagateTagToObjectAttributes(array $object, array $eventTags): array
+    {
+        foreach ($object['Attribute'] as $i => $attribute) {
+            $object['Attribute'][$i] = self::__propagateTagToAttributes($attribute, $eventTags);
+        }
+        return $object;
+    }
+
     private static function __encapsulateEntityWithEvent(array $data): array
     {
         $eventModel = ClassRegistry::init('Event');
@@ -112,12 +150,11 @@ class WorkflowFormatConverterTool
         if (empty($event)) {
             return [];
         }
-        $event = self::__convertEvent($event);
-        $event = $event['Event'];
         reset($data);
         $entityType = key($data);
-        $event[$entityType][] = $data[$entityType];
-        return ['Event' => $event];
+        $event['Event'][$entityType][] = $data[$entityType];
+        $event = self::__convertEvent($event);
+        return $event;
     }
 
     private static function __includeFlattenedAttributes(array $event): array

app/Lib/Tools/WorkflowGraphTool.php

@@ -6,16 +6,18 @@ class GraphUtil
 {
     public function __construct($graphData)
     {
-        $this->graph = $graphData;
+        $this->graph = array_filter($graphData, function($i) {
+            return $i != '_frames';
+        }, ARRAY_FILTER_USE_KEY);
         $this->numberNodes = count($this->graph);
-        $this->edgeList = $this->_buildEdgeList($graphData);
+        $this->edgeList = $this->_buildEdgeList($this->graph);
         $this->properties = [];
     }
 
     private function _buildEdgeList($graphData): array
     {
         $list = [];
-        foreach ($graphData as $node) {
+        foreach ($graphData as $i => $node) {
             $list[(int)$node['id']] = [];
             foreach (($node['outputs'] ?? []) as $output_id => $outputs) {
                 foreach ($outputs as $connections) {
@@ -162,6 +164,12 @@ class GraphWalker
         } else if ($node['data']['id'] == 'concurrent-task') {
             $this->_evaluateConcurrentTask($node, $roamingData, $outputs['output_1']);
             return ['output_1' => []];
+        } else if ($node['data']['id'] == 'generic-filter-data') {
+            $this->_evaluateFilterAddLogic($node, $roamingData, $outputs['output_1']);
+            return ['output_1' => $outputs['output_1']];
+        } else if ($node['data']['id'] == 'generic-filter-reset') {
+            $this->_evaluateFilterRemoveLogic($node, $roamingData, $outputs['output_1']);
+            return ['output_1' => $outputs['output_1']];
         } else {
             $useFirstOutput = $this->_evaluateCustomLogicCondition($node, $roamingData);
             return $useFirstOutput ? ['output_1' => $outputs['output_1']] : ['output_2' => $outputs['output_2']];
@@ -175,6 +183,18 @@ class GraphWalker
         return $result;
     }
 
+    private function _evaluateFilterAddLogic($node, WorkflowRoamingData $roamingData): bool
+    {
+        $result = $this->WorkflowModel->executeNode($node, $roamingData);
+        return $result;
+    }
+
+    private function _evaluateFilterRemoveLogic($node, WorkflowRoamingData $roamingData): bool
+    {
+        $result = $this->WorkflowModel->executeNode($node, $roamingData);
+        return $result;
+    }
+
     private function _evaluateCustomLogicCondition($node, WorkflowRoamingData $roamingData): bool
     {
         $result = $this->WorkflowModel->executeNode($node, $roamingData);
@@ -254,6 +274,7 @@ class WorkflowRoamingData
     private $data;
     private $workflow;
     private $current_node;
+    private $workflowModel;
 
     public function __construct(array $workflow_user, array $data, array $workflow, int $current_node)
     {
@@ -270,9 +291,50 @@ class WorkflowRoamingData
 
     public function getData(): array
     {
+        if (!empty($this->getEnabledFilters())) {
+            return $this->filterDataIfNeeded();
+        }
         return $this->data;
     }
 
+    public function filterDataIfNeeded(): array
+    {
+        $filteredData = $this->data;
+        $filters = $this->getEnabledFilters();
+        foreach ($filters as $filteringLabel => $filteringOptions) {
+            $filteredData = $this->applyFilter($filteredData, $filteringOptions);
+        }
+        return $filteredData;
+    }
+
+    private function applyFilter(array $data, array $filteringOptions): array
+    {
+        if (substr($filteringOptions['selector'], -4) === '.{n}') {
+            $filteringOptions['selector'] = substr($filteringOptions['selector'], 0, -4);
+        }
+        $baseModule = $this->getFilteringModule();
+        $extracted = $baseModule->extractData($data, $filteringOptions['selector']);
+        if ($extracted === false) {
+            $filteredData = false;
+        }
+        $filteredData = $baseModule->getItemsMatchingCondition($extracted, $filteringOptions['value'], $filteringOptions['operator'], $filteringOptions['path']);
+        $newData = Hash::remove($data, $filteringOptions['selector']);
+        $newData = Hash::insert($data, $filteringOptions['selector'], $filteredData);
+        return $newData;
+    }
+
+    private function getFilteringModule()
+    {
+        $this->workflowModel = ClassRegistry::init('Workflow');
+        $moduleClass = $this->workflowModel->getModuleClassByType('logic', 'generic-filter-data');
+        return $moduleClass;
+    }
+
+    public function getEnabledFilters(): array
+    {
+        return !empty($this->data['enabledFilters']) ? $this->data['enabledFilters'] : [];
+    }
+
     public function getWorkflow(): array
     {
         return $this->workflow;
@@ -296,6 +358,20 @@ class WorkflowRoamingData
 
 class WorkflowGraphTool
 {
+
+    /**
+     * cleanGraphData Remove frame nodes from the graph data
+     *
+     * @param  array $graphData
+     * @return array
+     */
+    public static function cleanGraphData(array $graphData): array
+    {
+        return array_filter($graphData, function($i) {
+            return $i != '_frames';
+        }, ARRAY_FILTER_USE_KEY);
+    }
+
     /**
      * extractTriggerFromWorkflow Return the trigger id (or full module) that are specified in the workflow
      *
@@ -322,8 +398,9 @@ class WorkflowGraphTool
      */
     public static function extractTriggersFromWorkflow(array $graphData, bool $fullNode = false): array
     {
+        $graphData = self::cleanGraphData($graphData);
         $triggers = [];
-        foreach ($graphData as $node) {
+        foreach ($graphData as $i => $node) {
             if ($node['data']['module_type'] == 'trigger') {
                 if (!empty($fullNode)) {
                     $triggers[] = $node;
@@ -344,8 +421,9 @@ class WorkflowGraphTool
      */
     public static function extractConcurrentTasksFromWorkflow(array $graphData, bool $fullNode = false): array
     {
+        $graphData = self::cleanGraphData($graphData);
         $nodes = [];
-        foreach ($graphData as $node) {
+        foreach ($graphData as $i => $node) {
             if ($node['data']['module_type'] == 'logic' && $node['data']['id'] == 'concurrent-task') {
                 if (!empty($fullNode)) {
                     $nodes[] = $node;
@@ -357,6 +435,52 @@ class WorkflowGraphTool
         return $nodes;
     }
 
+    /**
+     * extractFilterNodesFromWorkflow Return the list of generic-filter-data's id (or full module) that are included in the workflow
+     *
+     * @param  array $workflow
+     * @param  bool $fullNode
+     * @return array
+     */
+    public static function extractFilterNodesFromWorkflow(array $graphData, bool $fullNode = false): array
+    {
+        $graphData = self::cleanGraphData($graphData);
+        $nodes = [];
+        foreach ($graphData as $i => $node) {
+            if ($node['data']['module_type'] == 'logic' && $node['data']['id'] == 'generic-filter-data') {
+                if (!empty($fullNode)) {
+                    $nodes[] = $node;
+                } else {
+                    $nodes[] = $node['data']['id'];
+                }
+            }
+        }
+        return $nodes;
+    }
+
+    /**
+     * extractResetFilterFromWorkflow Return the list of generic-filter-reset's id (or full module) that are included in the workflow
+     *
+     * @param  array $workflow
+     * @param  bool $fullNode
+     * @return array
+     */
+    public static function extractResetFilterFromWorkflow(array $graphData, bool $fullNode = false): array
+    {
+        $graphData = self::cleanGraphData($graphData);
+        $nodes = [];
+        foreach ($graphData as $i => $node) {
+            if ($node['data']['module_type'] == 'logic' && $node['data']['id'] == 'generic-filter-reset') {
+                if (!empty($fullNode)) {
+                    $nodes[] = $node;
+                } else {
+                    $nodes[] = $node['data']['id'];
+                }
+            }
+        }
+        return $nodes;
+    }
+
     /**
      * isAcyclic Return if the graph contains a cycle
      *

app/Locale/ara/LC_MESSAGES/default.po

@@ -725,7 +725,7 @@ msgid "Invalid Sharing Group or not authorised."
 msgstr ""
 
 #: Controller/AttributesController.php:316;1772
-msgid "Attribute does not exists or you do not have the permission to download this attribute."
+msgid "Attribute does not exist or you do not have the permission to download this attribute."
 msgstr ""
 
 #: Controller/AttributesController.php:334

app/Locale/cake_resque.pot

@@ -578,7 +578,7 @@ msgid "Workers number [%s] is not valid. Please enter a valid number"
 msgstr ""
 
 #: Plugin/CakeResque/Console/Command/CakeResqueShell.php:1282
-msgid "User [%s] does not exists. Please enter a valid system user"
+msgid "User [%s] does not exist. Please enter a valid system user"
 msgstr ""
 
 #: Plugin/CakeResque/Console/Command/CakeResqueShell.php:1304

app/Locale/cze/LC_MESSAGES/default.po

@@ -723,7 +723,7 @@ msgid "Invalid Sharing Group or not authorised."
 msgstr "Neplatná Skupina sdílení nebo není autorizovaná."
 
 #: Controller/AttributesController.php:316;1772
-msgid "Attribute does not exists or you do not have the permission to download this attribute."
+msgid "Attribute does not exist or you do not have the permission to download this attribute."
 msgstr ""
 
 #: Controller/AttributesController.php:334

app/Locale/dan/LC_MESSAGES/default.po

@@ -720,7 +720,7 @@ msgid "Invalid Sharing Group or not authorised."
 msgstr "Ugyldig Delingsgruppe, eller ingen godkendelse."
 
 #: Controller/AttributesController.php:316;1772
-msgid "Attribute does not exists or you do not have the permission to download this attribute."
+msgid "Attribute does not exist or you do not have the permission to download this attribute."
 msgstr ""
 
 #: Controller/AttributesController.php:334

app/Locale/default.pot

@@ -848,7 +848,7 @@ msgid "Add attribute"
 msgstr ""
 
 #: Controller/AttributesController.php:287;1725
-msgid "Attribute does not exists or you do not have the permission to download this attribute."
+msgid "Attribute does not exist or you do not have the permission to download this attribute."
 msgstr ""
 
 #: Controller/AttributesController.php:305

app/Locale/deu/LC_MESSAGES/default.po

@@ -720,7 +720,7 @@ msgid "Invalid Sharing Group or not authorised."
 msgstr "Ungültige Freigabegruppe oder nicht berechtigt."
 
 #: Controller/AttributesController.php:316;1772
-msgid "Attribute does not exists or you do not have the permission to download this attribute."
+msgid "Attribute does not exist or you do not have the permission to download this attribute."
 msgstr ""
 
 #: Controller/AttributesController.php:334

app/Locale/fra/LC_MESSAGES/default.po

@@ -723,7 +723,7 @@ msgid "Invalid Sharing Group or not authorised."
 msgstr "Groupe de partage invalide ou non autorisé."
 
 #: Controller/AttributesController.php:316;1772
-msgid "Attribute does not exists or you do not have the permission to download this attribute."
+msgid "Attribute does not exist or you do not have the permission to download this attribute."
 msgstr ""
 
 #: Controller/AttributesController.php:334

app/Locale/hun/LC_MESSAGES/default.po

@@ -720,7 +720,7 @@ msgid "Invalid Sharing Group or not authorised."
 msgstr ""
 
 #: Controller/AttributesController.php:316;1772
-msgid "Attribute does not exists or you do not have the permission to download this attribute."
+msgid "Attribute does not exist or you do not have the permission to download this attribute."
 msgstr ""
 
 #: Controller/AttributesController.php:334