Merge branch '2.4' of github.com:VincenzoCaputo/MISP into add-enrichment-module-logs

.github/ISSUE_TEMPLATE/bug-form.yml

@@ -9,18 +9,18 @@ body:
         Please read the [FAQ](https://www.circl.lu/doc/misp/faq/) before opening an issue.
 
         If you would like to report a bug, please fill the template bellow:
-  - type: textarea
-    id: expected-behavior
-    attributes:
-      label: Expected behavior
-    validations:
-      required: true
   - type: textarea
     id: actual-behavior
     attributes:
       label: Actual behavior
     validations:
       required: true
+  - type: textarea
+    id: expected-behavior
+    attributes:
+      label: Expected behavior
+    validations:
+      required: true
   - type: textarea
     id: steps-to-reproduce
     attributes:

.github/ISSUE_TEMPLATE/feature-request-form.yml

@@ -1,7 +1,7 @@
 name: Feature request
 description: Suggest an idea for this project
 title: "Feature Request: "
-labels: ["feature request", "needs triage"]
+labels: ["T: feature request", "needs triage"]
 body:
   - type: textarea
     id: motif

.github/workflows/main.yml

@@ -26,7 +26,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: 'recursive'
 
@@ -218,7 +218,7 @@ jobs:
               sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.python_bin" "$GITHUB_WORKSPACE/venv/bin/python"'
               . ./venv/bin/activate
               export PYTHONPATH=$PYTHONPATH:./app/files/scripts
-              pip install ./PyMISP[fileobjects,email] ./app/files/scripts/python-stix ./app/files/scripts/cti-python-stix2 pyzmq redis plyara
+              pip install ./PyMISP[fileobjects,email] ./app/files/scripts/python-stix ./app/files/scripts/cti-python-stix2 pyzmq redis plyara pytest
               deactivate
 
       - name: Test if apache is working
@@ -243,6 +243,13 @@ jobs:
               ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/
               sudo -u www-data ./app/Vendor/bin/phpunit app/Test/
 
+      - name: Clone test files
+        uses: actions/checkout@v4
+        with:
+          repository: viper-framework/viper-test-files
+          path: PyMISP/tests/viper-test-files
+
+
       - name: Run tests
         run: |
               pushd tests
@@ -253,8 +260,9 @@ jobs:
 
               . ./venv/bin/activate
               pushd PyMISP
-              python tests/testlive_comprehensive.py
-              python tests/test_mispevent.py
+              cp tests/keys.py .
+              python -m pytest -v --durations=0 tests/test_mispevent.py
+              python -m pytest -v --durations=0 tests/testlive_comprehensive.py
               popd
               python tests/testlive_security.py -v
               python tests/testlive_sync.py

.travis.yml

@@ -27,8 +27,9 @@ install:
     - sudo apt-get -y update
     # Install haveged, because Travis lacks entropy.
     - sudo apt-get -y install haveged python3 python3-venv python3-pip python3-dev python3-nose python3-redis python3-lxml python3-dateutil python3-msgpack libxml2-dev libzmq3-dev zlib1g-dev apache2 curl php-mysql php-dev php-cli libapache2-mod-php libfuzzy-dev php-mbstring libonig4 php-json php-xml php-opcache php-readline php-redis php-gnupg php-gd
-    - sudo pip3 install --upgrade pip setuptools requests pyzmq
+    - sudo pip3 install --upgrade pip setuptools requests
     - sudo pip3 install --upgrade -r requirements.txt
+    - sudo pip3 install --upgrade -r requirements-dev.txt
     - pip3 install --user poetry
     - phpenv rehash
     - sudo mkdir $HOME/.composer ; sudo chown $USER:www-data $HOME/.composer

CODINGSTYLE.md

@@ -6,11 +6,11 @@ Maintaining proper coding style is very important for any large software project
 - It allows others (as well as the future you!) to easily understand fragments of code and what they were supposed to do, and thus makes it easier to later extend them with newer functionality or bug fixes
 - It allows others to easily review the code and catch bugs
 - It provides for an aesthetically pleasing experience when one reads the code
- 
+
 ## General typographic conventions
 - Maintain a maximum line length of 80 characters. Even though today’s monitors often are very wide and it’s often not a problem to have 120 characters displayed in an editor, maintaining shorter line lengths improves readability. It also allows others to have two parallel windows open, side by side, each with different parts of the source code.
 - Naming conventions:
-    - `ClassName`, 
+    - `ClassName`,
     - `someVariable`, `someFunction`, `someArgument`
 - Maintain a decent amount of horizontal spacing, e.g. add a space after `if` or before `{` in PHP, Python, JavaScript, and similar in other languages. Whether and where to also use spaces within expressions, such as `(y*4+8)` vs. `(y * 4 + 8)` is left to the developer’s judgment. Do not put spaces immediately after or before the brackets in expressions, so avoid constructs like this: `if ( condition )` and use ones like this: `if (condition)` instead.
 - Use descriptive names for variables and functions. At a time when most editors have auto-completion features, there is no excuse for using short variable names.
@@ -25,7 +25,7 @@ Maintaining proper coding style is very important for any large software project
 ## File naming conventions
 - Never use spaces within file names
 - **PHP:** Write file names in title case ,e.g. `AttachmentTool.php`
-- **Python:** Write file names with small letters, use a dash to separate words, rather than underscores, e.g. `load_warninglists.py`
+- **Python:** Write file names with small letters, use an underscore to separate words, rather than dashes, e.g. `load_warninglists.py`
 - **JavaScript:** Write file names with small letters, use dashes to separate words, rather than underscores, e.g. `bootstrap-colorpicker.js`
 
 ## General programming style guidelines
@@ -41,7 +41,7 @@ Maintaining proper coding style is very important for any large software project
         return style;
       }
 - In production code, there should be little to no commented or disabled code fragments. Do not use comments to disable code fragments, unless you need to. But generally, there is little excuse to keep old, unused code fragments in the code. Instead, use the functionality provided by the source code management system, such as git. For example, create a special branch for storing the old, unused code – this way you will always be able to merge this code into upstream in the future.
-- Try not to hardcode values in the code. 
+- Try not to hardcode values in the code.
 
 ## Commit message guidelines
 

CONTRIBUTING.md

@@ -2,7 +2,7 @@
 
 MISP project is a large free software project composed of multiple sub-projects which are contributed by different contributors who are generally active users of the MISP project. MISP project fully supports the [Contributor Covenant Code of Conduct](https://github.com/MISP/MISP/blob/2.4/code_of_conduct.md) to foster an open and dynamic environment for contributing and the exchange in the threat intelligence and information exchange field.
 
-The [MISP roadmap](/roadmap.md) is mostly based on the user communities (e.g. private communities, CSIRTs communities, security researchers, ISACs - Information Sharing and Analysis Center, security providers, governmental or military organisations) relying on MISP to perform their duties of information sharing and analysis. 
+The [MISP roadmap](/ROADMAP.md) is mostly based on the user communities (e.g. private communities, CSIRTs communities, security researchers, ISACs - Information Sharing and Analysis Center, security providers, governmental or military organisations) relying on MISP to perform their duties of information sharing and analysis.
 
 Participating in the MISP project is easy and everyone can contribute following their ability. 
 Get familiar with [how we use GitHub at MISP Project](/GITWORKFLOW.md), then read on for details on some ways you can contribute:

INSTALL/INSTALL.sh

@@ -699,8 +699,9 @@ kaliOnTheR0ckz () {
   overlay=$(df -kh |grep overlay; echo $?) # if 1 overlay NOT present
 
   if [[ ${totalRoot} -lt 3059034 ]]; then
-    echo "(If?) You run Kali in LiveCD mode and we need more overlay disk space."
-    echo "This is defined by the total memory, you have: ${totalMem}kB which is not enough."
+    echo "(If?) You run Kali in LiveCD mode, you need more overlay disk space."
+    echo "This is defined by the total memory setting in you VM config."
+    echo "You currently have: ${totalMem}kB which is not enough."
     echo "6-8Gb should be fine. (need >3Gb overlayFS)"
     exit 1
   fi
@@ -793,6 +794,14 @@ installRNG () {
 kaliUpgrade () {
   debug "Running various Kali upgrade tasks"
   checkAptLock
+  # Fix Missing keys early
+  sudo wget -O /etc/apt/trusted.gpg.d/php.gpg https://packages.sury.org/php/apt.gpg
+  # /!\ The following is a very ugly dependency hack to make php7.4 work on Kali
+  wget http://archive.ubuntu.com/ubuntu/pool/main/o/openssl/libssl1.1_1.1.1f-1ubuntu2_amd64.deb
+  sudo dpkg -i libssl1.1_1.1.1f-1ubuntu2_amd64.deb
+  wget http://ftp.debian.org/debian/pool/main/i/icu/libicu67_67.1-7_amd64.deb
+  sudo dpkg -i libicu67_67.1-7_amd64.deb
+  # EOH End-Of-Hack
   sudo DEBIAN_FRONTEND=noninteractive apt update
   sudo DEBIAN_FRONTEND=noninteractive apt install --only-upgrade bash libc6 -y
   sudo DEBIAN_FRONTEND=noninteractive apt autoremove -y
@@ -3596,17 +3605,19 @@ x86_64-rhel-8
 x86_64-fedora-33
 x86_64-fedora-34
 x86_64-fedora-35
+x86_64-debian-12
 x86_64-debian-stretch
 x86_64-debian-buster
 x86_64-ubuntu-bionic
 x86_64-ubuntu-focal
 x86_64-ubuntu-hirsute
 x86_64-ubuntu-jammy
-x86_64-kali-2021.4
 x86_64-kali-2022.1
 x86_64-kali-2022.2
 x86_64-kali-2022.3
 x86_64-kali-2022.4
+x86_64-kali-2023.1
+x86_64-kali-2023.2
 armv6l-raspbian-stretch
 armv7l-raspbian-stretch
 armv7l-raspbian-buster

INSTALL/INSTALL.sh.sfv

@@ -1,5 +1,5 @@
-; Generated by RHash v1.4.2 on 2023-07-01 at 17:15.04
+; Generated by RHash v1.4.4 on 2023-10-12 at 13:42.08
 ; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/
 ;
-;       160686  17:15.04 2023-07-01 INSTALL.sh
-INSTALL.sh 9576C31EC5BD942E1C9B12413E6408E4623252F7 78B708FE1FC6B39BE081B9F05C6AA5E1478F8762CAF5A8A7671A12EBA4D3C1C5 27991471FB5788F42AF3BBF86FC80A95341AA17AE9487016EEC94961A48437172702EB8E2D6CB300387E87D9E8E0E3E5 C1C21FD491AEFD662C87C3EF62837D769E63E9CF2446B9BD607CCEF8AFD72528824A8F408C6892FD51109390104010EF90DA7F4828950A8671D2986A6B8E216F
+;       161244  13:42.07 2023-10-12 INSTALL.sh
+INSTALL.sh 8624B1D834A4C958B16DCE32CEAE88C3D1DC15D7 D7B9E370C85B53BBF7B0F81F5C263B6AB2E534F59B40A6499277F39407FF194A EF5B68E9D0D634C2CADD4FD9B1E5C2A93DBD938F488417F67A2B9C87E8867CB0200293A150CDAEDA369E7FDC476EEC2B 1445710924BC029647CC5AA0EBFFD0A3B6DDAF39D26B5A0E951FFFEEF621677C59470F250ECFB6059C9E200951F98D4F21646F152D6F8931438531F9516A8748

INSTALL/INSTALL.sh.sha1

@@ -1 +1 @@
-9576c31ec5bd942e1c9b12413e6408e4623252f7  INSTALL.sh
+8624b1d834a4c958b16dce32ceae88c3d1dc15d7  INSTALL.sh

INSTALL/INSTALL.sh.sha256

@@ -1 +1 @@
-78b708fe1fc6b39be081b9f05c6aa5e1478f8762caf5a8a7671a12eba4d3c1c5  INSTALL.sh
+d7b9e370c85b53bbf7b0f81f5c263b6ab2e534f59b40a6499277f39407ff194a  INSTALL.sh

INSTALL/INSTALL.sh.sha384

@@ -1 +1 @@
-27991471fb5788f42af3bbf86fc80a95341aa17ae9487016eec94961a48437172702eb8e2d6cb300387e87d9e8e0e3e5  INSTALL.sh
+ef5b68e9d0d634c2cadd4fd9b1e5c2a93dbd938f488417f67a2b9c87e8867cb0200293a150cdaeda369e7fdc476eec2b  INSTALL.sh

INSTALL/INSTALL.sh.sha512

@@ -1 +1 @@
-c1c21fd491aefd662c87c3ef62837d769e63e9cf2446b9bd607ccef8afd72528824a8f408c6892fd51109390104010ef90da7f4828950a8671d2986a6b8e216f  INSTALL.sh
+1445710924bc029647cc5aa0ebffd0a3b6ddaf39d26b5a0e951fffeef621677c59470f250ecfb6059c9e200951f98d4f21646f152d6f8931438531f9516a8748  INSTALL.sh

INSTALL/INSTALL.tpl.sh

@@ -844,17 +844,19 @@ x86_64-rhel-8
 x86_64-fedora-33
 x86_64-fedora-34
 x86_64-fedora-35
+x86_64-debian-12
 x86_64-debian-stretch
 x86_64-debian-buster
 x86_64-ubuntu-bionic
 x86_64-ubuntu-focal
 x86_64-ubuntu-hirsute
 x86_64-ubuntu-jammy
-x86_64-kali-2021.4
 x86_64-kali-2022.1
 x86_64-kali-2022.2
 x86_64-kali-2022.3
 x86_64-kali-2022.4
+x86_64-kali-2023.1
+x86_64-kali-2023.2
 armv6l-raspbian-stretch
 armv7l-raspbian-stretch
 armv7l-raspbian-buster

INSTALL/old/INSTALL.ubuntu1604.txt

@@ -1 +0,0 @@
-../../docs/archive/INSTALL.ubuntu1604.md

PyMISP

@@ -1 +1 @@
-Subproject commit 7d1d8b6f38f210b28934a206f9c1470542e9da7e
+Subproject commit 2dff8fb9b8d8a5214e5a364da40ca6afca56f09d

VERSION.json

@@ -1 +1 @@
-{"major":2, "minor":4, "hotfix":172}
+{"major":2, "minor":4, "hotfix":177}

app/Config/config.default.php

@@ -106,7 +106,7 @@ $config = array(
         'user_email_notification_ban_time_threshold' => 120,
         'user_email_notification_ban_amount_threshold' => 10,
         'user_email_notification_ban_refresh_on_retry' => true,
-        'warning_for_all'                => true,
+        'warning_for_all'                => false,
         'enable_synchronisation_filtering_on_type' => false,
     ),
     'GnuPG' => array(

app/Console/Command/AdminShell.php

@@ -112,13 +112,31 @@ class AdminShell extends AppShell
         return $parser;
     }
 
-    public function jobGenerateCorrelation()
+    public function jobForgot()
     {
         if (empty($this->args[0])) {
-            die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Generate correlation'] . PHP_EOL);
+            die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Forgot'] . PHP_EOL);
+        }
+
+        $email = $this->args[0];
+        $ip = empty($this->args[1]) ? null : $this->args[1];
+        $jobId = empty($this->args[2]) ? null : $this->args[2];
+        $this->User->forgot($email, $ip, $jobId);
+    }
+
+    public function jobGenerateCorrelation()
+    {
+        $jobId = $this->args[0] ?? null;
+        if (empty($jobId)) {
+            $jobId = $this->Job->createJob(
+                'SYSTEM',
+                Job::WORKER_DEFAULT,
+                'generate correlation',
+                'All attributes',
+                'Job created.'
+            );
         }
 
-        $jobId = $this->args[0];
         $this->Correlation->generateCorrelation($jobId);
     }
 

app/Console/Command/DevShell.php

@@ -1,7 +1,11 @@
 <?php
+
+App::uses('ComponentCollection', 'Controller');
+App::uses('RestSearchComponent', 'Controller/Component');
+
 class DevShell extends AppShell {
 
-    public $uses = [];
+    public $uses = ['Attribute', 'Event', 'Object', 'GalaxyCluster', 'Sighting'];
 
     public function cleanFeedDefault() {
         $this->out(__('Massaging the feed metadata file.'));
@@ -46,4 +50,57 @@ class DevShell extends AppShell {
             }
         }
     }
+
+    public function generateSearchParams()
+    {
+        $fetchFunctionName = [
+            'Attribute' => 'fetchAttributes',
+            'Event' => 'fetchEvents',
+            'Object' => 'fetchObjects',
+            'Sighting' => 'fetchSightings',
+            'GalaxyCluster' => 'fetchGalaxyClusters'
+        ];
+        $collection = new ComponentCollection();
+        $this->RestSearchComponent = $collection->load('RestSearch');
+        $paramArray = $this->RestSearchComponent->paramArray;
+        foreach ($paramArray as $scope => $params) {
+            if (!empty($this->$scope->possibleOptions)) {
+                $paramArray[$scope] = array_values(array_unique(array_merge($paramArray[$scope], $this->$scope->possibleOptions)));
+            } else {
+                $fileName = $scope === 'Object' ? 'MispObject' : $scope;
+                $code = file_get_contents(APP . 'Model/' . $fileName . '.php');
+                $code = explode("\n", $code);
+                $start = false;
+                $end = false;
+                $analyzedBlock = [];
+                foreach ($code as $lineNumber => $line) {
+                    if (strpos($line, 'public function ' . $fetchFunctionName[$scope] . '(') !== false) {
+                        $start = $lineNumber;
+                    }
+                    if ($start) {
+                        if ($lineNumber !== $start && strpos($line, 'public function') !== false) {
+                            $end = $lineNumber - 1;
+                            break;
+                        }
+                        $analyzedBlock[] = $line;
+                    }
+                }
+                $analyzedBlock = implode("\n", $analyzedBlock);
+                $foundParams = [];
+                preg_match_all('/\$options\[\'([^\']+)/i', $analyzedBlock, $foundParams);
+                $foundParams = $foundParams[1];
+                foreach ($foundParams as $k => $v) {
+                    if (in_array(strtolower($v), ['contain', 'fields', 'conditions', 'order', 'joins', 'group', 'limit', 'page', 'recursive', 'callbacks'])) {
+                        unset($foundParams[$k]);
+                    }
+                }
+                $paramArray[$scope] = array_values(array_unique(array_merge($paramArray[$scope], $foundParams)));
+            }
+        }
+        foreach ($paramArray as $scope => $fields) {
+            echo "'" . $scope ."' => [" . PHP_EOL . "    '";
+            echo implode("'," . PHP_EOL . "    '", $fields) . "'" . PHP_EOL;
+            echo "]," . PHP_EOL;
+        }
+    }
 }

app/Console/Command/EventShell.php

@@ -91,12 +91,14 @@ class EventShell extends AppShell
         }
 
         $isXml = $extension === 'xml';
-        $takeOwnership = $this->param('take_ownership');
-        $publish = $this->param('publish');
+        $takeOwnership = $this->params['take-ownership'];
+        $publish = $this->params['publish'];
         $results = $this->Event->addMISPExportFile($user, $content, $isXml, $takeOwnership, $publish);
 
         foreach ($results as $result) {
             if (is_numeric($result['result'])) {
+                $this->out("Event `{$result['info']}` already exists at ({$result['result']}).");
+            } else if ($result['result'] === true) {
                 $this->out("Event #{$result['id']}: {$result['info']} imported.");
             } else {
                 $this->out("Could not import event because of validation errors: " . json_encode($result['validationIssues']));
@@ -405,7 +407,17 @@ class EventShell extends AppShell
         $jobId = $this->args[2];
         $userId = $this->args[3];
         $user = $this->getUser($userId);
-        $job = $this->Job->read(null, $jobId);
+        $job = $this->Job->find('first', [
+            'recursive' => -1,
+            'conditions' => [
+                'Job.id' => $jobId
+            ]
+        ]);
+        if (empty($job)) {
+            $log = ClassRegistry::init('Log');
+            $log->createLogEntry($user, 'publish', 'Event', $id, 'Event (' . $id . '): could not be published - valid job not found.', '');
+            return true;
+        }
         $this->Event->Behaviors->unload('SysLogLogable.SysLogLogable');
         $result = $this->Event->publish($id, $passAlong);
         $job['Job']['progress'] = 100;

app/Console/Command/TrainingShell.php

@@ -5,9 +5,13 @@
  * arg0 = email
  * arg1 = new password
  */
+
+App::uses('File', 'Utility');
+
+
 class TrainingShell extends AppShell {
 
-    public $uses = array('User', 'Organisation', 'Server');
+    public $uses = array('User', 'Organisation', 'Server', 'AuthKey');
 
     private $__currentUrl = false;
     private $__currentAuthKey = false;
@@ -115,6 +119,126 @@ class TrainingShell extends AppShell {
         $this->__printReport('Setup complete. Please find the modifications below:' . PHP_EOL . PHP_EOL);
     }
 
+    public function createOrganisationsFromConfig()
+    {
+        $rawConfig = file_get_contents(APP . 'Console/Command/config_orgs.json');
+        $config = json_decode($rawConfig, true);
+        $createdOrgs = [];
+        foreach ($config as $org) {
+            $filepath = APP . 'Console/Command/' . $org['Organisation']['logo_path'];
+            $file = new File($filepath, false);
+            if ($file->exists()) {
+                $org['Organisation']['logo'] = [
+                    'name' => $file->name(),
+                    'type' => $file->mime(),
+                    'tmp_name' => $filepath,
+                    'error' => 0,
+                    'size' => $file->size(),
+                ];
+            }
+            $file->close();
+            $date = date('Y-m-d H:i:s');
+            $org['Organisation']['date_created'] = $date;
+            $org['Organisation']['date_modified'] = $date;
+            $this->Organisation->create();
+            $this->Organisation->save($org);
+            $filename = $this->Organisation->id . '.' . ($file->ext() === 'svg' ? 'svg' : 'png');
+            $file->copy(APP . 'webroot/img/orgs/' . $filename);
+            $createdOrg = $this->Organisation->find('first', ['conditions' => ['id' => $this->Organisation->id]]);
+            $createdOrgs[$createdOrg['Organisation']['uuid']] = $createdOrg['Organisation'];
+        }
+        return $createdOrgs;
+    }
+
+    public function createUsersFromConfig($createdOrgs)
+    {
+        $rawConfig = file_get_contents(APP . 'Console/Command/config_users.json');
+        $config = json_decode($rawConfig, true);
+        $createdUsers = [];
+        foreach ($config as $user) {
+            if (!empty($user['org_uuid'])) {
+                $user['org_id'] = $createdOrgs[$user['org_uuid']]['id'];
+            }
+            $existingUser = $this->User->find('first', [
+                'recursive' => -1,
+                'conditions' => ['User.email' => $user['email']],
+            ]);
+            if (empty($existingUser)) {
+                $this->User->create();
+            } else {
+                $user['id'] = $existingUser['User']['id'];
+            }
+            $this->User->save($user);
+            $createdUser = $this->User->find('first', ['id' => $this->User->id]);
+            $createdUsers[] = $createdUser;
+        }
+        return $createdUsers;
+    }
+
+    public function setSettingsFromConfig($createdOrgs)
+    {
+        $rawConfig = file_get_contents(APP . 'Console/Command/config_settings.json');
+        $config = json_decode($rawConfig, true);
+        $cli_user = ['id' => 0, 'email' => 'SYSTEM', 'Organisation' => ['name' => 'SYSTEM']];
+        foreach ($config as $setting_name => $value) {
+            if ($setting_name == 'MISP.host_org_id') {
+                $value = $createdOrgs[$value]['id'];
+            }
+            $setting = $this->Server->getSettingData($setting_name);
+            if (empty($setting)) {
+                $this->error(__('Setting change rejected.'));
+            }
+            $result = $this->Server->serverSettingsEditValue($cli_user, $setting, $value, true);
+            if (empty($result)) {
+                $this->error(__('Setting change rejected.'));
+            }
+        }
+    }
+
+    public function createRemoteServersFromConfig($createdOrgs, $createdUsers)
+    {
+        $rawConfig = file_get_contents(APP . 'Console/Command/config_syncs.json');
+        $config = json_decode($rawConfig, true);
+        $createdServers = [];
+        foreach ($config as $sync) {
+            $sync['org_id'] = $createdOrgs[$sync['org_uuid']]['id'];
+            $sync['remote_org_id'] = $createdOrgs[$sync['remote_org_uuid']]['id'];
+            $this->Server->create();
+            $this->Server->save($sync);
+            $createdServer = $this->User->find('first', ['id' => $this->User->id]);
+            $createdServers[] = $createdServer;
+        }
+        return $createdServers;
+    }
+
+    public function createAllFromConfig()
+    {
+        $createdOrgs = $this->createOrganisationsFromConfig();
+        $createdUsers = $this->createUsersFromConfig($createdOrgs);
+        $this->setSettingsFromConfig($createdOrgs);
+        $this->createRemoteServersFromConfig($createdOrgs, $createdUsers);
+    }
+
+    public function WipeAllSyncs()
+    {
+        $this->Server->deleteAll(['Server.id !=' => 0]);
+    }
+
+    public function WipeAllUsers()
+    {
+        $this->User->deleteAll(['User.email !=' => 'admin@admin.test']);
+    }
+
+    public function WipeAllOrgs()
+    {
+        $this->Organisation->deleteAll(['Organisation.name !=' => 'ORGNAME']);
+    }
+
+    public function WipeAllAuthkeys()
+    {
+        $this->AuthKey->deleteAll(['AuthKey.id !=' => 0]);
+    }
+
     private function __createOrgFromBlueprint($id)
     {
         $org = str_replace('$ID', $id, $this->__config['org_blueprint']);

app/Console/Command/UserShell.php

@@ -104,6 +104,18 @@ class UserShell extends AppShell
                 ],
             ],
         ]);
+        $parser->addSubcommand('require_password_change_for_old_passwords', [
+            'help' => __('Trigger forced password change on next login for users with an old (older than x days) password.'),
+            'parser' => [
+                'arguments' => [
+                    'days' => ['help' => __('Amount of days after which a password is considered "old" and needs to be changed.'), 'required' => true]
+                ],
+            ]
+        ]);
+
+        $parser->addSubcommand('expire_authkeys_without_ip_allowlist', [
+            'help' => __('Expire all active authkeys that do not have an IP allowlist set.'),
+        ]);
         return $parser;
     }
 
@@ -431,6 +443,61 @@ class UserShell extends AppShell
         }
     }
 
+    public function require_password_change_for_old_passwords()
+    {
+        list($days) = $this->args;
+        if(!is_numeric($days)){
+            $this->error("The amount of days after which a password change is required (the argument) should be numeric.");
+        }
+        $interval  = 'P' . $days . 'D';
+
+        $current_time = new DateTime();
+        $time_before_change_required = $current_time->sub(new DateInterval($interval))->getTimestamp();
+        $users = $this->User->find('all', [
+            'conditions' => [
+                'OR' => [
+                    'last_pw_change <' => $time_before_change_required
+                ]
+            ],
+            'fields' => ['id'],
+            'recursive' => 0
+        ]);
+        foreach ($users as $user) {
+            $user['User']['change_pw'] = true;
+            $userId = $user['User']['id'];
+            if (!$this->User->save($user['User'], true, ["change_pw"])) {
+                $this->out("Could not update user $userId.");
+                $this->out($this->json($this->User->validationErrors));
+                $this->_stop(self::CODE_ERROR);
+            }
+        }
+    }
+
+    public function expire_authkeys_without_ip_allowlist()
+    {
+        $time = time();
+        $authkeys = $this->User->AuthKey->find('all', [
+            'conditions' => [
+                'OR' => [
+                    'AuthKey.expiration >' => $time,
+                    'AuthKey.expiration' => 0
+                ],
+                'allowed_ips' => NULL
+            ],
+            'fields' => ['id', 'user_id'],
+            'recursive' => 0
+        ]);
+        foreach ($authkeys as $authkey) {
+            $authkey['AuthKey']['expiration'] = $time;
+            $authkeyId = $authkey['AuthKey']['id'];
+            if (!$this->User->AuthKey->save($authkey['AuthKey'])) {
+                $this->out("Could not update authkey $authkeyId.");
+                $this->out($this->json($this->User->AuthKey->validationErrors));
+                $this->_stop(self::CODE_ERROR);
+            }
+        }
+    }
+
     /**
      * @param string|int $userId
      * @return array

app/Controller/AppController.php

@@ -33,8 +33,8 @@ class AppController extends Controller
 
     public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
 
-    private $__queryVersion = '152';
-    public $pyMispVersion = '2.4.172';
+    private $__queryVersion = '155';
+    public $pyMispVersion = '2.4.176';
     public $phpmin = '7.2';
     public $phprec = '7.4';
     public $phptoonew = '8.0';
@@ -222,8 +222,10 @@ class AppController extends Controller
             !$userLoggedIn &&
             (
                 $controller !== 'users' ||
-                $action !== 'register' ||
-                empty(Configure::read('Security.allow_self_registration'))
+                (
+                    ($action !== 'register' || empty(Configure::read('Security.allow_self_registration'))) &&
+                    (!in_array($action, ['forgot', 'password_reset']) || empty(Configure::read('Security.allow_password_forgotten')))
+                )
             )
         ) {
             // REST authentication
@@ -314,6 +316,10 @@ class AppController extends Controller
             if (!empty(Configure::read('Security.email_otp_enabled'))) {
                 $preAuthActions[] = 'email_otp';
             }
+            if (!empty(Configure::read('Security.allow_password_forgotten'))) {
+                $preAuthActions[] = 'forgot';
+                $preAuthActions[] = 'password_reset';
+            }
             if (!$this->_isControllerAction(['users' => $preAuthActions, 'servers' => ['cspReport']])) {
                 if ($isAjax) {
                     $response = $this->RestResponse->throwException(401, "Unauthorized");
@@ -956,6 +962,14 @@ class AppController extends Controller
         return $user;
     }
 
+    private function __captureParam($data, $param, $value)
+    {
+        if ($this->modelClass->checkParam($param)) {
+            $data[$param] = $value;
+        }
+        return $data;
+    }
+
     /**
      * generic function to standardise on the collection of parameters. Accepts posted request objects, url params, named url params
      * @param array $options
@@ -976,9 +990,21 @@ class AppController extends Controller
                 return false;
             } else {
                 if (isset($request->data['request'])) {
-                    $data = array_merge($data, $request->data['request']);
+                    $temp = $request->data['request'];
                 } else {
-                    $data = array_merge($data, $request->data);
+                    $temp = $request->data;
+                }
+                if (empty($options['paramArray'])) {
+                    foreach ($options['paramArray'] as $param => $value) {
+                        $data = $this->__captureParam($data, $param, $value);
+                    }
+                    $data = array_merge($data, $temp);
+                } else {
+                    foreach ($options['paramArray'] as $param) {
+                        if (isset($temp[$param])) {
+                            $data[$param] = $temp[$param];
+                        }
+                    }
                 }
             }
         }
@@ -1078,7 +1104,7 @@ class AppController extends Controller
     {
         $result = false;
         if (Configure::read('Plugin.CustomAuth_enable')) {
-            $header = Configure::read('Plugin.CustomAuth_header') ? Configure::read('Plugin.CustomAuth_header') : 'Authorization';
+            $header = Configure::read('Plugin.CustomAuth_header') ? Configure::read('Plugin.CustomAuth_header') : 'AUTHORIZATION';
             $authName = Configure::read('Plugin.CustomAuth_name') ? Configure::read('Plugin.CustomAuth_name') : 'External authentication';
             if (
                 !Configure::check('Plugin.CustomAuth_use_header_namespace') ||

app/Controller/AttributesController.php

@@ -1539,6 +1539,7 @@ class AttributesController extends AppController
         $user = $this->Auth->user();
         $exception = null;
         $filters = $this->__getSearchFilters($exception);
+        $this->set('passedArgsArray', ['results' => $continue]);
         if ($this->request->is('post') || !empty($this->request->params['named']['tags'])) {
             if ($filters === false) {
                 return $exception;

app/Controller/AuditLogsController.php

@@ -158,6 +158,7 @@ class AuditLogsController extends AppController
         }
 
         $this->paginate['conditions'] = $this->__createEventIndexConditions($event);
+        $this->set('passedArgsArray', ['eventId' => $eventId, 'org' => $org]);
 
         $params = $this->IndexFilter->harvestParameters(['created', 'org']);
         if ($org) {

app/Controller/Component/ACLComponent.php

@@ -484,9 +484,9 @@ class ACLComponent extends Component
                 'display' => array('*'),
             ),
             'posts' => array(
-                'add' => ['AND' => ['not_read_only_authkey', 'discussion_enabled']],
-                'delete' => ['AND' => ['not_read_only_authkey', 'discussion_enabled']],
-                'edit' => ['AND' => ['not_read_only_authkey', 'discussion_enabled']],
+                'add' => ['AND' => ['not_read_only_authkey', 'discussion_enabled', 'perm_add']],
+                'delete' => ['AND' => ['not_read_only_authkey', 'discussion_enabled', 'perm_add']],
+                'edit' => ['AND' => ['not_read_only_authkey', 'discussion_enabled', 'perm_add']],
                 'pushMessageToZMQ' => array()
             ),
             'regexp' => array(
@@ -613,7 +613,7 @@ class ACLComponent extends Component
             ),
             'sightings' => array(
                 'add' => array('perm_sighting'),
-                'restSearch' => array('perm_sighting'),
+                'restSearch' => array('*'),
                 'advanced' => array('perm_sighting'),
                 'delete' => ['AND' => ['perm_sighting', 'perm_modify_org']],
                 'index' => array('*'),
@@ -748,6 +748,7 @@ class ACLComponent extends Component
                 'downloadTerms' => array('*'),
                 'edit' => array('self_management_enabled'),
                 'email_otp' => array('*'),
+                'forgot' => array('*'),
                 'otp' => array('*'),
                 'hotp' => array('*'),
                 'totp_new' => array('*'),
@@ -760,6 +761,7 @@ class ACLComponent extends Component
                 'logout' => array('*'),
                 'logout401' => array('*'),
                 'notificationSettings' => ['*'],
+                'password_reset' => array('*'),
                 'register' => array('*'),
                 'registrations' => array(),
                 'resetAllSyncAuthKeys' => array(),

app/Controller/Component/IndexFilterComponent.php

@@ -54,7 +54,7 @@ class IndexFilterComponent extends Component
     private function __massageData($data, $request, $paramArray)
     {
         $data = array_filter($data, function($paramName) use ($paramArray) {
-            return !empty($paramArray[$paramName]);
+            return in_array($paramName, $paramArray, true);
         }, ARRAY_FILTER_USE_KEY);
 
         if (!empty($paramArray)) {

app/Controller/Component/RestSearchComponent.php

@@ -6,33 +6,188 @@ class RestSearchComponent extends Component
 {
     //This array determines the order for ordered_url_params, as a result it is not advised to remove or change existing values
     public $paramArray = array(
-        'Attribute' => array(
-            //following parameters are not used for attributes anymore: attackGalaxy
-            'returnFormat', 'value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp',
-            'published', 'timestamp','enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags',
-            'includeProposals', 'returnFormat', 'limit', 'page', 'requested_attributes', 'includeContext', 'headerless',
-            'includeWarninglistHits', 'attackGalaxy', 'object_relation', 'includeSightings', 'includeCorrelations', 'includeDecayScore',
-            'decayingModel', 'excludeDecayed', 'modelOverrides', 'includeFullModel', 'score', 'attribute_timestamp', 'first_seen', 'last_seen',
-            'threat_level_id'
-        ),
-        'Event' => array(
-            'returnFormat', 'value', 'type', 'category', 'org', 'tags', 'searchall', 'from', 'to', 'last', 'eventid', 'withAttachments',
-            'metadata', 'uuid', 'publish_timestamp', 'timestamp', 'published', 'enforceWarninglist', 'sgReferenceOnly',
-            'limit', 'page', 'requested_attributes', 'includeContext', 'headerless', 'includeWarninglistHits', 'attackGalaxy', 'to_ids', 'deleted',
-            'excludeLocalTags', 'date', 'includeSightingdb', 'tag', 'object_relation', 'threat_level_id'
-        ),
-        'Object' => array(
-            'returnFormat', 'value' , 'type', 'category', 'org', 'tags', 'from', 'to', 'last', 'eventid', 'withAttachments', 'uuid', 'publish_timestamp',
-            'published', 'timestamp','enforceWarninglist', 'to_ids', 'deleted', 'includeEventUuid', 'event_timestamp', 'threat_level_id', 'includeEventTags',
-            'includeProposals', 'returnFormat', 'limit', 'page', 'requested_attributes', 'includeContext', 'headerless',
-            'includeWarninglistHits', 'attackGalaxy', 'object_relation'
-        ),
-        'Sighting' => array(
-            'context', 'returnFormat', 'id', 'type', 'from', 'to', 'last', 'org_id', 'source', 'includeAttribute', 'includeEvent'
-        ),
-        'GalaxyCluster' => array(
-            'page', 'limit', 'id', 'uuid', 'galaxy_id', 'galaxy_uuid', 'version', 'distribution', 'org', 'orgc', 'tag', 'custom', 'sgReferenceOnly', 'minimal',
-        )
+        'Attribute' => [
+            'returnFormat',
+            'value',
+            'value1',
+            'value2',
+            'type',
+            'category',
+            'org',
+            'tags',
+            'from',
+            'to',
+            'last',
+            'eventid',
+            'withAttachments',
+            'uuid',
+            'publish_timestamp',
+            'published',
+            'timestamp',
+            'enforceWarninglist',
+            'to_ids',
+            'deleted',
+            'includeEventUuid',
+            'event_timestamp',
+            'threat_level_id',
+            'includeEventTags',
+            'includeProposals',
+            'limit',
+            'page',
+            'requested_attributes',
+            'includeContext',
+            'headerless',
+            'includeWarninglistHits',
+            'attackGalaxy',
+            'object_relation',
+            'includeSightings',
+            'includeCorrelations',
+            'includeDecayScore',
+            'decayingModel',
+            'excludeDecayed',
+            'modelOverrides',
+            'includeFullModel',
+            'score',
+            'attribute_timestamp',
+            'first_seen',
+            'last_seen',
+            'eventinfo',
+            'allow_proposal_blocking',
+            'flatten',
+            'list',
+            'event_ids',
+            'includeAllTags',
+            'includeAttributeUuid',
+            'includeGalaxy'
+        ],
+        'Event' => [
+            'returnFormat',
+            'value',
+            'type',
+            'category',
+            'org',
+            'tags',
+            'searchall',
+            'from',
+            'to',
+            'last',
+            'eventid',
+            'withAttachments',
+            'metadata',
+            'uuid',
+            'publish_timestamp',
+            'timestamp',
+            'event_timestamp', // redundant, but kept for backwards compatibility
+            'published',
+            'enforceWarninglist',
+            'sgReferenceOnly',
+            'limit',
+            'page',
+            'requested_attributes',
+            'includeContext',
+            'headerless',
+            'includeWarninglistHits',
+            'attackGalaxy',
+            'to_ids',
+            'deleted',
+            'excludeLocalTags',
+            'date',
+            'includeSightingdb',
+            'tag',
+            'object_relation',
+            'threat_level_id',
+            'eventinfo',
+            'sharinggroup',
+            'idList',
+            'includeAllTags',
+            'includeAttachments',
+            'event_uuid',
+            'distribution',
+            'sharing_group_id',
+            'disableSiteAdmin',
+            'flatten',
+            'blockedAttributeTags',
+            'eventsExtendingUuid',
+            'extended',
+            'extensionList',
+            'excludeGalaxy',
+            'includeRelatedTags',
+            'includeDecayScore',
+            'includeScoresOnEvent',
+            'includeFeedCorrelations',
+            'includeServerCorrelations',
+            'noEventReports',
+            'noShadowAttributes',
+            'order',
+            'protected'
+        ],
+        'Object' => [
+            'returnFormat',
+            'value',
+            'type',
+            'category',
+            'org',
+            'tags',
+            'from',
+            'to',
+            'last',
+            'eventid',
+            'withAttachments',
+            'uuid',
+            'publish_timestamp',
+            'published',
+            'timestamp',
+            'enforceWarninglist',
+            'to_ids',
+            'deleted',
+            'includeEventUuid',
+            'event_timestamp',
+            'threat_level_id',
+            'includeEventTags',
+            'includeProposals',
+            'limit',
+            'page',
+            'requested_attributes',
+            'includeContext',
+            'headerless',
+            'includeWarninglistHits',
+            'attackGalaxy',
+            'object_relation',
+            'metadata',
+            'includeAllTags'
+        ],
+        'Sighting' => [
+            'context',
+            'returnFormat',
+            'id',
+            'type',
+            'from',
+            'to',
+            'last',
+            'org_id',
+            'source',
+            'includeAttribute',
+            'includeEvent'
+        ],
+        'GalaxyCluster' => [
+            'page',
+            'limit',
+            'id',
+            'uuid',
+            'galaxy_id',
+            'galaxy_uuid',
+            'version',
+            'distribution',
+            'org',
+            'orgc',
+            'tag',
+            'custom',
+            'sgReferenceOnly',
+            'minimal',
+            'list',
+            'first',
+            'count'
+        ],
     );
 
     public function getFilename($filters, $scope, $responseType)

app/Controller/DashboardsController.php

@@ -93,6 +93,9 @@ class DashboardsController extends AppController
             if (empty($data['config'])) {
                 $data['config'] = '';
             }
+            if (!empty($data['id']) && !preg_match('/^[\w\d_]+$/i', $data['id'])) {
+                throw new BadRequestException(__('Invalid widget id provided.'));
+            }
             if ($action === 'add') {
                 $data['widget_options'] = $this->Dashboard->loadAllWidgets($this->Auth->user());
             } else if ($action === 'edit') {
@@ -194,6 +197,29 @@ class DashboardsController extends AppController
 
         if (!empty($this->request->params['named']['exportjson'])) {
             return $this->RestResponse->viewData($data);
+        } else if (!empty($this->request->params['named']['exportcsv'])) {
+            $csv = '';
+            $toConvert = !empty($data) ? (!empty($data['data']) ? $data['data'] : $data) : [];
+            if (!empty($toConvert)) {
+                $firstElement = key($toConvert);
+                if (is_string($firstElement)) {
+                    foreach ($toConvert as $key => $value) {
+                        $csv .= sprintf('%s,%s', $key, json_encode($value)) . PHP_EOL;
+                    }
+                } else { // second element is an array
+                    $csv = array_map(function($row) {
+                        $flattened = array_values(Hash::flatten($row));
+                        $stringified = array_map('strval', $flattened);
+                        $quotified = array_map(function($item) { return sprintf('"%s"', $item); }, $stringified);
+                        return implode(',', $quotified);
+                    }, $toConvert);
+                    $rowKey = implode(',', array_map(function ($item) {
+                        return sprintf('"%s"', $item);
+                    }, array_map('strval', array_keys(Hash::flatten($toConvert[0])))));
+                    $csv = $rowKey . PHP_EOL .  implode(PHP_EOL, array_values($csv));
+                }
+            }
+            return $this->RestResponse->viewData($csv, 'text/csv', false, true);
         }
 
         $this->layout = false;

app/Controller/EventsController.php

@@ -31,7 +31,7 @@ class EventsController extends AppController
         'sort', 'direction', 'focus', 'extended', 'overrideLimit', 'filterColumnsOverwrite', 'attributeFilter', 'page',
         'searchFor', 'proposal', 'correlation', 'warning', 'deleted', 'includeRelatedTags', 'includeDecayScore', 'distribution',
         'taggedAttributes', 'galaxyAttachedAttributes', 'objectType', 'attributeType', 'feed', 'server', 'toIDS',
-        'sighting', 'includeSightingdb', 'warninglistId', 'correlationId',
+        'sighting', 'includeSightingdb', 'warninglistId', 'correlationId', 'email', 'eventid', 'datefrom', 'dateuntil'
     );
 
     // private
@@ -1117,7 +1117,7 @@ class EventsController extends AppController
             'attribute' => array('OR' => array(), 'NOT' => array()),
             'hasproposal' => 2,
             'timestamp' => array('from' => "", 'until' => ""),
-            'publishtimestamp' => array('from' => "", 'until' => ""),
+            'publishtimestamp' => array('from' => "", 'until' => "")
         );
 
         if ($this->_isSiteAdmin()) {
@@ -5210,42 +5210,82 @@ class EventsController extends AppController
         $this->render('index');
     }
 
-    // expects an attribute ID and the module to be used
-    public function queryEnrichment($attribute_id, $module = false, $type = 'Enrichment')
+    // expects a model ID, model type, the module to be used (optional) and the type of enrichment (optional)
+    public function queryEnrichment($id, $module = false, $type = 'Enrichment', $model = 'Attribute')
     {
         if (!Configure::read('Plugin.' . $type . '_services_enable')) {
             throw new MethodNotAllowedException(__('%s services are not enabled.', $type));
         }
-        $attribute = $this->Event->Attribute->fetchAttributes($this->Auth->user(), [
-            'conditions' => [
-                'Attribute.id' => $attribute_id
-            ],
-            'flatten' => 1,
-            'includeEventTags' => 1,
-            'contain' => ['Event' => ['fields' => ['distribution', 'sharing_group_id']]],
-        ]);
-        if (empty($attribute)) {
-            throw new MethodNotAllowedException(__('Attribute not found or you are not authorised to see it.'));
+
+        if (!in_array($model, array('Attribute', 'ShadowAttribute', 'Object', 'Event'))) {
+            throw new MethodNotAllowedException(__('Invalid model.'));
         }
+
         $this->loadModel('Module');
         $enabledModules = $this->Module->getEnabledModules($this->Auth->user(), false, $type);
+        
         if (!is_array($enabledModules) || empty($enabledModules)) {
-            throw new MethodNotAllowedException(__('No valid %s options found for this attribute.', $type));
+            throw new MethodNotAllowedException(__('No valid %s options found for this %s.', $type, strtolower($model)));
         }
+
+        if ($model === 'Attribute' || $model === 'ShadowAttribute') {
+            $attribute = $this->Event->Attribute->fetchAttributes($this->Auth->user(), [
+                'conditions' => [
+                    'Attribute.id' => $id
+                ],
+                'flatten' => 1,
+                'includeEventTags' => 1,
+                'contain' => ['Event' => ['fields' => ['distribution', 'sharing_group_id']]],
+            ]);
+            if (empty($attribute)) {
+                throw new MethodNotAllowedException(__('Attribute not found or you are not authorised to see it.'));
+            }
+        }
+
+        if ($model === 'Object') {
+            $object = $this->Event->Object->fetchObjects($this->Auth->user(), [
+                'conditions' => [
+                    'Object.id' => $id
+                ],
+                'flatten' => 1,
+                'includeEventTags' => 1,
+                'contain' => ['Event' => ['fields' => ['distribution', 'sharing_group_id']]],
+            ]);
+            if (empty($object)) {
+                throw new MethodNotAllowedException(__('Object not found or you are not authorised to see it.'));
+            }
+        }
+        
         if ($this->request->is('ajax')) {
-            $modules = array();
-            foreach ($enabledModules['modules'] as $module) {
-                if (in_array($attribute[0]['Attribute']['type'], $module['mispattributes']['input'])) {
-                    $modules[] = array('name' => $module['name'], 'description' => $module['meta']['description']);
+            $modules = [];
+
+            if ($model === 'Attribute' || $model === 'ShadowAttribute') {
+                foreach ($enabledModules['modules'] as $module) {
+                    if (in_array($attribute[0]['Attribute']['type'], $module['mispattributes']['input'])) {
+                        $modules[] = array('name' => $module['name'], 'description' => $module['meta']['description']);
+                    }
                 }
             }
-            foreach (array('attribute_id', 'modules') as $viewVar) {
-                $this->set($viewVar, $$viewVar);
+
+            if ($model === 'Object') {
+                foreach ($enabledModules['modules'] as $module) {
+                    if (
+                        in_array($object[0]['Object']['name'], $module['mispattributes']['input']) ||
+                        in_array($object[0]['Object']['uuid'], $module['mispattributes']['input'])
+                    ) {
+                        $modules[] = array('name' => $module['name'], 'description' => $module['meta']['description']);
+                    }
+                }
             }
+            
+            $this->set('id', $id);
+            $this->set('modules', $modules);
             $this->set('type', $type);
+            $this->set('model', $model);
             $this->render('ajax/enrichmentChoice');
         } else {
-            $options = array();
+            $options = [];
+            $format = 'simplified';
             foreach ($enabledModules['modules'] as $temp) {
                 if ($temp['name'] == $module) {
                     $format = !empty($temp['mispattributes']['format']) ? $temp['mispattributes']['format'] : 'simplified';
@@ -5267,7 +5307,13 @@ class EventsController extends AppController
             $this->set('title_for_layout', __('Enrichment Results'));
             $this->set('title', __('Enrichment Results'));
             if ($format == 'misp_standard') {
-                $this->__queryEnrichment($attribute, $module, $options, $type);
+                if ($model === 'Attribute' || $model === 'ShadowAttribute') {
+                    $this->__queryEnrichment($attribute, $module, $options, $type);
+                }
+                
+                if ($model === 'Object') {
+                    $this->__queryObjectEnrichment($object, $module, $options, $type);
+                }
             } else {
                 $this->__queryOldEnrichment($attribute, $module, $options, $type);
             }
@@ -5323,6 +5369,57 @@ class EventsController extends AppController
         }
     }
 
+    private function __queryObjectEnrichment($object, $module, $options, $type)
+    {
+        $object[0]['Object']['Attribute'] = $object[0]['Attribute'];
+        foreach($object[0]['Object']['Attribute'] as &$attribute) {
+            if ($this->Event->Attribute->typeIsAttachment($attribute['type'])) {
+                $attribute['data'] = $this->Event->Attribute->base64EncodeAttachment($attribute);
+            }
+        }
+
+        $event_id = $object[0]['Event']['id'];
+        $data = array('module' => $module, 'object' => $object[0]['Object'], 'event_id' => $event_id);
+        if (!empty($options)) {
+            $data['config'] = $options;
+        }
+        $result = $this->Module->queryModuleServer($data, false, $type, false, $object[0]);
+        if (!$result) {
+            throw new InternalErrorException(__('%s service not reachable.', $type));
+        }
+        if (isset($result['error'])) {
+            $this->Flash->error($result['error']);
+        }
+        if (!is_array($result)) {
+            throw new Exception($result);
+        }
+        $event = $this->Event->handleMispFormatFromModuleResult($result);
+        if (empty($event['Attribute']) && empty($event['Object'])) {
+            throw new NotImplementedException(__('No Attribute or Object returned by the module.'));
+        } else {
+            $importComment = !empty($result['comment']) ? $result['comment'] : $object[0]['Object']['value'] . __(': Enriched via the ') . $module . ($type != 'Enrichment' ? ' ' . $type : '')  . ' module';
+            $this->set('importComment', $importComment);
+            $event['Event'] = $object[0]['Event'];
+            $org_name = $this->Event->Orgc->find('first', array(
+                'conditions' => array('Orgc.id' => $event['Event']['orgc_id']),
+                'fields' => array('Orgc.name')
+            ));
+            $event['Event']['orgc_name'] = $org_name['Orgc']['name'];
+            if ($attribute[0]['Object']['id']) {
+                $object_id = $attribute[0]['Object']['id'];
+                $initial_object = $this->Event->fetchInitialObject($event_id, $object_id);
+                if (!empty($initial_object)) {
+                    $event['initialObject'] = $initial_object;
+                }
+            }
+            $this->set('event', $event);
+            $this->set('menuItem', 'enrichmentResults');
+            $this->set('title_for_layout', __('Enrichment Results'));
+            $this->set('title', __('Enrichment Results'));
+            $this->render('resolved_misp_format');
+        }
+    }
+
     private function __queryOldEnrichment($attribute, $module, $options, $type)
     {
         $data = array('module' => $module, $attribute[0]['Attribute']['type'] => $attribute[0]['Attribute']['value'], 'event_id' => $attribute[0]['Attribute']['event_id'], 'attribute_uuid' => $attribute[0]['Attribute']['uuid']);

app/Controller/GalaxiesController.php

@@ -350,14 +350,14 @@ class GalaxiesController extends AppController
         $items = array(
             array(
                 'name' => __('All clusters'),
-                'value' => $this->baseurl . "/galaxies/selectCluster/" . h($target_id) . '/' . h($target_type) . '/0'. '/local:' . $local . '/eventid:' . $eventid
+                'value' => $this->baseurl . "/galaxies/selectCluster/" . h($target_id) . '/' . h($target_type) . '/0'. '/local:' . h($local) . '/eventid:' . h($eventid)
             )
         );
         foreach ($galaxies as $galaxy) {
             if (!isset($galaxy['Galaxy']['kill_chain_order']) || $noGalaxyMatrix) {
                 $items[] = array(
                     'name' => h($galaxy['Galaxy']['name']),
-                    'value' => $this->baseurl . "/galaxies/selectCluster/" . $target_id . '/' . $target_type . '/' . $galaxy['Galaxy']['id'] . '/local:' . $local . '/eventid:' . $eventid,
+                    'value' => $this->baseurl . "/galaxies/selectCluster/" . h($target_id) . '/' . h($target_type) . '/' . $galaxy['Galaxy']['id'] . '/local:' . h($local) . '/eventid:' . h($eventid),
                     'template' => array(
                         'preIcon' => 'fa-' . $galaxy['Galaxy']['icon'],
                         'name' => $galaxy['Galaxy']['name'],
@@ -367,14 +367,14 @@ class GalaxiesController extends AppController
             } else { // should use matrix instead
                 $param = array(
                     'name' => $galaxy['Galaxy']['name'],
-                    'value' => $this->baseurl . "/galaxies/selectCluster/" . $target_id . '/' . $target_type . '/' . $galaxy['Galaxy']['id'] . '/local:' . $local . '/eventid:' . $eventid,
+                    'value' => $this->baseurl . "/galaxies/selectCluster/" . h($target_id) . '/' . h($target_type) . '/' . $galaxy['Galaxy']['id'] . '/local:' . h($local) . '/eventid:' . h($eventid),
                     'functionName' => sprintf(
                         "getMatrixPopup('%s', '%s', '%s/local:%s/eventid:%s')",
-                        $target_type,
-                        $target_id,
-                        $galaxy['Galaxy']['id'],
-                        $local,
-                        $eventid
+                        h($target_type),
+                        h($target_id),
+                        h($galaxy['Galaxy']['id']),
+                        h($local),
+                        h($eventid)
                     ),
                     'isPill' => true,
                     'isMatrix' => true
@@ -405,12 +405,12 @@ class GalaxiesController extends AppController
         $noGalaxyMatrix = $noGalaxyMatrix ? '1' : '0';
         $items = [[
             'name' => __('All namespaces'),
-            'value' => $this->baseurl . "/galaxies/selectGalaxy/" . $target_id . '/' . $target_type . '/0' . '/' . $noGalaxyMatrix . '/local:' . $local . '/eventid:' . $eventid
+            'value' => $this->baseurl . "/galaxies/selectGalaxy/" . h($target_id) . '/' . h($target_type) . '/0' . '/' . h($noGalaxyMatrix) . '/local:' . h($local) . '/eventid:' . h($eventid)
         ]];
         foreach ($namespaces as $namespace) {
             $items[] = array(
                 'name' => $namespace,
-                'value' => $this->baseurl . "/galaxies/selectGalaxy/" . $target_id . '/' . $target_type . '/' . $namespace . '/' . $noGalaxyMatrix . '/local:' . $local . '/eventid:' . $eventid
+                'value' => $this->baseurl . "/galaxies/selectGalaxy/" . h($target_id) . '/' . h($target_type) . '/' . h($namespace) . '/' . h($noGalaxyMatrix) . '/local:' . h($local) . '/eventid:' . h($eventid)
             );
         }
 

app/Controller/GalaxyClustersController.php

@@ -38,7 +38,15 @@ class GalaxyClustersController extends AppController
     public function index($galaxyId)
     {
         $galaxyId = $this->Toolbox->findIdByUuid($this->GalaxyCluster->Galaxy, $galaxyId);
-        $filters = $this->_harvestParameters(array('context', 'searchall'));
+        $filterData = array(
+            'request' => $this->request,
+            'named_params' => $this->params['named'],
+            'paramArray' => ['context', 'searchall'],
+            'ordered_url_params' => [],
+            'additional_delimiters' => PHP_EOL
+        );
+        $exception = false;
+        $filters = $this->_harvestParameters($filterData, $exception);
         $aclConditions = $this->GalaxyCluster->buildConditions($this->Auth->user());
         $contextConditions = array();
         if (empty($filters['context'])) {

app/Controller/LogsController.php

@@ -273,7 +273,7 @@ class LogsController extends AppController
             $this->set('actionDefinitions', $this->{$this->defaultModel}->actionDefinitions);
 
             // reset the paginate_conditions
-            $this->Session->write('paginate_conditions_log', array());
+            //$this->Session->write('paginate_conditions_log', array());
             if ($this->request->is('post')) {
                 $filters['email'] = $this->request->data['Log']['email'];
                 if (!$orgRestriction) {
@@ -285,6 +285,12 @@ class LogsController extends AppController
                 $filters['model'] = $this->request->data['Log']['model'];
                 $filters['model_id'] = $this->request->data['Log']['model_id'];
                 $filters['title'] = $this->request->data['Log']['title'];
+                if (!empty ($this->request->data['Log']['from'])) {
+                    $filters['from'] = $this->request->data['Log']['from'];
+                }
+                if (!empty ($this->request->data['Log']['to'])) {
+                    $filters['to'] = $this->request->data['Log']['to'];
+                }
                 $filters['change'] = $this->request->data['Log']['change'];
                 if (Configure::read('MISP.log_client_ip')) {
                     $filters['ip'] = $this->request->data['Log']['ip'];
@@ -297,6 +303,8 @@ class LogsController extends AppController
                 $this->set('modelSearch', $filters['model']);
                 $this->set('model_idSearch', $filters['model_id']);
                 $this->set('titleSearch', $filters['title']);
+                $this->set('fromSearch', $filters['from'] ?? null);
+                $this->set('toSearch', $filters['to'] ?? null);
                 $this->set('changeSearch', $filters['change']);
                 if (Configure::read('MISP.log_client_ip')) {
                     $this->set('ipSearch', $filters['ip']);
@@ -329,10 +337,11 @@ class LogsController extends AppController
                     $this->Session->write('paginate_conditions_log_model_id', $filters['model_id']);
                     $this->Session->write('paginate_conditions_log_title', $filters['title']);
                     $this->Session->write('paginate_conditions_log_change', $filters['change']);
+                    $this->Session->write('paginate_conditions_log_from', $filters['from'] ?? null);
+                    $this->Session->write('paginate_conditions_log_to', $filters['to'] ?? null);
                     if (Configure::read('MISP.log_client_ip')) {
                         $this->Session->write('paginate_conditions_log_ip', $filters['ip']);
                     }
-
                     // set the same view as the index page
                     $this->render('index');
                 }
@@ -345,10 +354,11 @@ class LogsController extends AppController
                 $filters['model_id'] = $this->Session->read('paginate_conditions_log_model_id');
                 $filters['title'] = $this->Session->read('paginate_conditions_log_title');
                 $filters['change'] = $this->Session->read('paginate_conditions_log_change');
+                $filters['from'] = $this->Session->read('paginate_conditions_log_from') ?? null;
+                $filters['to'] = $this->Session->read('paginate_conditions_log_to') ?? null;
                 if (Configure::read('MISP.log_client_ip')) {
                     $filters['ip'] = $this->Session->read('paginate_conditions_log_ip');
                 }
-
                 // for info on what was searched for
                 $this->set('emailSearch', $filters['email']);
                 $this->set('orgSearch', $filters['org']);
@@ -357,6 +367,8 @@ class LogsController extends AppController
                 $this->set('model_idSearch', $filters['model_id']);
                 $this->set('titleSearch', $filters['title']);
                 $this->set('changeSearch', $filters['change']);
+                $this->set('changeSearch', $filters['from'] ?? null);
+                $this->set('changeSearch', $filters['to'] ?? null);
                 if (Configure::read('MISP.log_client_ip')) {
                     $this->set('ipSearch', $filters['ip']);
                 }
@@ -364,7 +376,7 @@ class LogsController extends AppController
 
                 // re-get pagination
                 $this->{$this->defaultModel}->recursive = 0;
-                $this->paginate = array_merge_recursive($this->Session->read('paginate_conditions_log'), $this->paginate);
+                $this->paginate = array_replace_recursive($this->paginate, $this->Session->read('paginate_conditions_log'));
                 if (!isset($this->paginate['order'])) {
                     $this->paginate['order'] = array('Log.id' => 'DESC');
                 }
@@ -449,6 +461,12 @@ class LogsController extends AppController
         if (isset($filters['change']) && !empty($filters['change'])) {
             $conditions['LOWER(Log.change) LIKE'] = '%' . strtolower($filters['change']) . '%';
         }
+        if (isset($filters['from']) && !empty($filters['from'])) {
+            $conditions['Log.created >='] = $filters['from'];
+        }
+        if (isset($filters['to']) && !empty($filters['to'])) {
+            $conditions['Log.created <='] = $filters['to'];
+        }
         if (Configure::read('MISP.log_client_ip') && isset($filters['ip']) && !empty($filters['ip'])) {
             $conditions['Log.ip LIKE'] = '%' . $filters['ip'] . '%';
         }

app/Controller/ServersController.php

@@ -503,7 +503,6 @@ class ServersController extends AppController
                     $this->Flash->error($error_msg);
                 }
             }
-
             if (!$fail && !empty($this->request->data['Server']['push_rules']) && !JsonTool::isValid($this->request->data['Server']['push_rules'])) {
                 $fail = true;
                 $error_msg = __('The push filter rules must be in valid JSON format.');
@@ -512,17 +511,22 @@ class ServersController extends AppController
                 } else {
                     $this->Flash->error($error_msg);
                 }
-            }
-            $pushRules = $this->_jsonDecode($this->request->data['Server']['push_rules']);
-            $this->loadModel('Tag');
-            foreach ($pushRules['tags'] as $operator => $list) {
-                foreach ($list as $i => $tagName) {
-                    if (!is_numeric($tagName)) { // tag added from freetext
-                        $tag_id = $this->Tag->captureTag(['name' => $tagName], $this->Auth->user());
-                        $list[$i] = $tag_id;
+            }            
+            if (!$fail && !empty($this->request->data['Server']['push_rules'])) {
+                $pushRules = $this->_jsonDecode($this->request->data['Server']['push_rules']);
+                if (!empty($pushRules['tags'])) {
+                    $this->loadModel('Tag');
+                    foreach ($pushRules['tags'] as $operator => $list) {
+                        foreach ($list as $i => $tagName) {
+                            if (!is_numeric($tagName)) { // tag added from freetext
+                                $tag_id = $this->Tag->captureTag(['name' => $tagName], $this->Auth->user());
+                                $list[$i] = $tag_id;
+                            }
+                        }
                     }
                 }
             }
+            
             if (!$fail) {
                 // say what fields are to be updated
                 $fieldList = array('id', 'url', 'push', 'pull', 'push_sightings', 'push_galaxy_clusters', 'pull_galaxy_clusters', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'remove_missing_tags', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
@@ -915,30 +919,52 @@ class ServersController extends AppController
             App::uses('File', 'Utility');
             App::uses('Folder', 'Utility');
             App::uses('FileAccessTool', 'Tools');
+            App::uses('SyncTool', 'Tools');
             if (isset($server['Server'][$subm]['name'])) {
                 if ($this->request->data['Server'][$subm]['size'] != 0) {
                     if (!$this->Server->checkFilename($server['Server'][$subm]['name'])) {
                         throw new Exception(__('Filename not allowed'));
                     }
-                    $file = new File($server['Server'][$subm]['name']);
-                    $ext = $file->ext();
+
+                    if (!is_uploaded_file($server['Server'][$subm]['tmp_name'])) {
+                        throw new Exception(__('File not uploaded correctly'));
+                    }
+
+                    $ext = pathinfo($server['Server'][$subm]['name'], PATHINFO_EXTENSION);
+                    if (!in_array($ext, SyncTool::ALLOWED_CERT_FILE_EXTENSIONS)) {
+                        $this->Flash->error(__('Invalid extension.'));
+                        $this->redirect(array('action' => 'index'));
+                    }
+
                     if (!$server['Server'][$subm]['size'] > 0) {
                         $this->Flash->error(__('Incorrect extension or empty file.'));
                         $this->redirect(array('action' => 'index'));
                     }
 
-                    // read pem file data
-                    $pemData = FileAccessTool::readFromFile($server['Server'][$subm]['tmp_name'], $server['Server'][$subm]['size']);
+                    // read certificate file data
+                    $certData = FileAccessTool::readFromFile($server['Server'][$subm]['tmp_name'], $server['Server'][$subm]['size']);
                 } else {
                     return true;
                 }
             } else {
-                $pemData = base64_decode($server['Server'][$subm]);
+                $ext = 'pem';
+                $certData = base64_decode($server['Server'][$subm]);
             }
+
+            // check if the file is a valid x509 certificate
+            try {
+                $cert = openssl_x509_parse($certData);
+                if (!$cert) {
+                    throw new Exception(__('Invalid certificate.'));
+                }
+            } catch (Exception $e) {
+                $this->Flash->error(__('Invalid certificate.'));
+                $this->redirect(array('action' => 'index'));
+            }
+
             $destpath = APP . "files" . DS . "certs" . DS;
-            $dir = new Folder(APP . "files" . DS . "certs", true);
             $pemfile = new File($destpath . $id . $ins . '.' . $ext);
-            $result = $pemfile->write($pemData);
+            $result = $pemfile->write($certData);
             $s = $this->Server->read(null, $id);
             $s['Server'][$attr] = $s['Server']['id'] . $ins . '.' . $ext;
             if ($result) {

app/Controller/ShadowAttributesController.php

@@ -78,7 +78,7 @@ class ShadowAttributesController extends AppController
             }
 
             if (isset($shadow['proposal_to_delete']) && $shadow['proposal_to_delete']) {
-                $this->Attribute->delete($activeAttribute['Attribute']['id']);
+                $this->Attribute->deleteAttribute($activeAttribute['Attribute']['id'], $this->Auth->user(), false);
             } else {
                 // Update the live attribute with the shadow data
                 $fieldsToUpdate = array('value1', 'value2', 'value', 'type', 'category', 'comment', 'to_ids', 'first_seen', 'last_seen');
@@ -906,7 +906,24 @@ class ShadowAttributesController extends AppController
         }
         $params = array(
             'conditions' => $conditions,
-            'fields' => array('ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.timestamp', 'ShadowAttribute.first_seen', 'ShadowAttribute.last_seen'),
+            'fields' => array(
+                'ShadowAttribute.id',
+                'ShadowAttribute.old_id',
+                'ShadowAttribute.event_id',
+                'ShadowAttribute.type',
+                'ShadowAttribute.category',
+                'ShadowAttribute.uuid',
+                'ShadowAttribute.to_ids',
+                'ShadowAttribute.value',
+                'ShadowAttribute.comment',
+                'ShadowAttribute.org_id',
+                'ShadowAttribute.timestamp',
+                'ShadowAttribute.first_seen',
+                'ShadowAttribute.last_seen',
+                'ShadowAttribute.deleted',
+                'ShadowAttribute.proposal_to_delete',
+                'ShadowAttribute.disable_correlation'
+            ),
             'contain' => array(
                     'Event' => array(
                             'fields' => array('id', 'org_id', 'info', 'orgc_id', 'uuid'),

app/Controller/SightingsController.php

@@ -66,7 +66,8 @@ class SightingsController extends AppController
                 $filters = !empty($this->request->data['filters']) ? $this->request->data['filters'] : false;
             }
             if (!$error) {
-                $result = $this->Sighting->saveSightings($id, $values, $timestamp, $this->Auth->user(), $type, $source, false, true, false, $filters);
+                $publish_sighting = !empty(Configure::read('Sightings_enable_realtime_publish'));
+                $result = $this->Sighting->saveSightings($id, $values, $timestamp, $this->Auth->user(), $type, $source, false, $publish_sighting, false, $filters);
             }
             if (!is_numeric($result)) {
                 $error = $result;

app/Controller/TaxonomiesController.php

@@ -77,11 +77,7 @@ class TaxonomiesController extends AppController
         $urlparams = '';
         App::uses('CustomPaginationTool', 'Tools');
         $filter = isset($this->passedArgs['filter']) ? $this->passedArgs['filter'] : false;
-        $options = ['full' => true, 'filter' => $filter];
-        if (isset($this->passedArgs['enabled'])) {
-            $options['enabled'] = $this->passedArgs['enabled'];
-        }
-        $taxonomy = $this->Taxonomy->getTaxonomy($id, $options);
+        $taxonomy = $this->Taxonomy->getTaxonomy($id, true, $filter);
         if (empty($taxonomy)) {
             throw new NotFoundException(__('Taxonomy not found.'));
         }
@@ -189,16 +185,26 @@ class TaxonomiesController extends AppController
             'recursive' => -1,
             'conditions' => array('Taxonomy.id' => $id),
         ));
-        $taxonomy['Taxonomy']['enabled'] = true;
-        $this->Taxonomy->save($taxonomy);
-
-        $this->__log('enable', $id, 'Taxonomy enabled', $taxonomy['Taxonomy']['namespace'] . ' - enabled');
-
-        if ($this->_isRest()) {
-            return $this->RestResponse->saveSuccessResponse('Taxonomy', 'enable', $id, $this->response->type());
+        if (empty($taxonomy)) {
+            $message = __('Invalid taxonomy.');
+            if ($this->_isRest()) {
+                return $this->RestResponse->saveFailResponse('Taxonomy', 'enable', $id, $message);
+            } else {
+                $this->Flash->error($message);
+                $this->redirect($this->referer());
+            }
         } else {
-            $this->Flash->success(__('Taxonomy enabled.'));
-            $this->redirect($this->referer());
+            $taxonomy['Taxonomy']['enabled'] = true;
+            $this->Taxonomy->save($taxonomy);
+
+            $this->__log('enable', $id, 'Taxonomy enabled', $taxonomy['Taxonomy']['namespace'] . ' - enabled');
+
+            if ($this->_isRest()) {
+                return $this->RestResponse->saveSuccessResponse('Taxonomy', 'enable', $id, $this->response->type());
+            } else {
+                $this->Flash->success(__('Taxonomy enabled.'));
+                $this->redirect($this->referer());
+            }
         }
     }
 
@@ -210,17 +216,27 @@ class TaxonomiesController extends AppController
             'recursive' => -1,
             'conditions' => array('Taxonomy.id' => $id),
         ));
-        $this->Taxonomy->disableTags($id);
-        $taxonomy['Taxonomy']['enabled'] = 0;
-        $this->Taxonomy->save($taxonomy);
-
-        $this->__log('disable', $id, 'Taxonomy disabled', $taxonomy['Taxonomy']['namespace'] . ' - disabled');
-
-        if ($this->_isRest()) {
-            return $this->RestResponse->saveSuccessResponse('Taxonomy', 'disable', $id, $this->response->type());
+        if (empty($taxonomy)) {
+            $message = __('Invalid taxonomy.');
+            if ($this->_isRest()) {
+                return $this->RestResponse->saveFailResponse('Taxonomy', 'disable', $id, $message);
+            } else {
+                $this->Flash->error($message);
+                $this->redirect($this->referer());
+            }
         } else {
-            $this->Flash->success(__('Taxonomy disabled.'));
-            $this->redirect($this->referer());
+            $this->Taxonomy->disableTags($id);
+            $taxonomy['Taxonomy']['enabled'] = 0;
+            $this->Taxonomy->save($taxonomy);
+
+            $this->__log('disable', $id, 'Taxonomy disabled', $taxonomy['Taxonomy']['namespace'] . ' - disabled');
+
+            if ($this->_isRest()) {
+                return $this->RestResponse->saveSuccessResponse('Taxonomy', 'disable', $id, $this->response->type());
+            } else {
+                $this->Flash->success(__('Taxonomy disabled.'));
+                $this->redirect($this->referer());
+            }
         }
     }
 

app/Controller/UsersController.php

@@ -30,6 +30,10 @@ class UsersController extends AppController
 
         // what pages are allowed for non-logged-in users
         $allowedActions = array('login', 'logout', 'getGpgPublicKey', 'logout401', 'otp');
+        if (!empty(Configure::read('Security.allow_password_forgotten'))) {
+            $allowedActions[] = 'forgot';
+            $allowedActions[] = 'password_reset';
+        }
         if(!empty(Configure::read('Security.email_otp_enabled'))) {
           $allowedActions[] = 'email_otp';
         }
@@ -262,6 +266,78 @@ class UsersController extends AppController
         $this->set('canFetchPgpKey', $this->__canFetchPgpKey());
     }
 
+    private function __pw_change($user, $source, &$abortPost, $token = false)
+    {
+        if (!isset($this->request->data['User'])) {
+            $this->request->data = array('User' => $this->request->data);
+        }
+        if (Configure::read('Security.require_password_confirmation')) {
+            if (!empty($this->request->data['User']['current_password'])) {
+                $hashed = $this->User->verifyPassword($this->Auth->user('id'), $this->request->data['User']['current_password']);
+                if (!$hashed) {
+                    $message = __('Invalid password. Please enter your current password to continue.');
+                    if ($this->_isRest()) {
+                        return $this->RestResponse->saveFailResponse('Users', $source, false, $message, $this->response->type());
+                    }
+                    $abortPost = true;
+                    $this->Flash->error($message);
+                }
+                unset($this->request->data['User']['current_password']);
+            } else if (!$this->_isRest()) {
+                $message = __('Please enter your current password to continue.');
+                if ($this->_isRest()) {
+                    return $this->RestResponse->saveFailResponse('Users', $source, false, $message, $this->response->type());
+                }
+                $abortPost = true;
+                $this->Flash->info($message);
+            }
+        }
+        $hashed = $this->User->verifyPassword($this->Auth->user('id'), $this->request->data['User']['password']);
+        if ($hashed) {
+            $message = __('Submitted new password cannot be the same as the current one');
+            $abortPost = true;
+        }
+        if (!$abortPost) {
+            // What fields should be saved (allowed to be saved)
+            $user['User']['change_pw'] = 0;
+            $user['User']['password'] = $this->request->data['User']['password'];
+            $user['User']['last_pw_change'] = time();
+            if ($this->_isRest()) {
+                $user['User']['confirm_password'] = $this->request->data['User']['password'];
+            } else {
+                $user['User']['confirm_password'] = $this->request->data['User']['confirm_password'];
+            }
+            $temp = $user['User']['password'];
+            // Save the data
+            if ($this->User->save($user)) {
+                if ($token) {
+                    $this->User->purgeForgetToken($token);
+                }
+                $message = __('Password Changed.');
+                // log as System if the reset comes from an unauthed user using password_reset tokens
+                $logUser = empty($this->Auth->user()) ? 'SYSTEM' : $this->Auth->user();
+                $this->User->extralog($logUser, $source, null, null, $user);
+                if ($this->_isRest()) {
+                    return $this->RestResponse->saveSuccessResponse('User', $source, false, $this->response->type(), $message);
+                }
+                $this->Flash->success($message);
+                $this->redirect(array('action' => 'view', $user['User']['id']));
+            } else {
+                $message = __('The password could not be updated. Make sure you meet the minimum password length / complexity requirements.');
+                if ($this->_isRest()) {
+                    return $this->RestResponse->saveFailResponse('Users', $source, false, $message, $this->response->type());
+                }
+                $this->Flash->error($message);
+            }
+        } else {
+            if ($this->_isRest()) {
+                return $this->RestResponse->saveFailResponse('Users', $source, false, $message, $this->response->type());
+            } else {
+                $this->Flash->error($message);
+            }
+        }
+    }
+
     public function change_pw()
     {
         $id = $this->Auth->user('id');
@@ -270,69 +346,8 @@ class UsersController extends AppController
             'recursive' => -1
         ));
         if ($this->request->is('post') || $this->request->is('put')) {
-            if (!isset($this->request->data['User'])) {
-                $this->request->data = array('User' => $this->request->data);
-            }
             $abortPost = false;
-            if (Configure::read('Security.require_password_confirmation')) {
-                if (!empty($this->request->data['User']['current_password'])) {
-                    $hashed = $this->User->verifyPassword($this->Auth->user('id'), $this->request->data['User']['current_password']);
-                    if (!$hashed) {
-                        $message = __('Invalid password. Please enter your current password to continue.');
-                        if ($this->_isRest()) {
-                            return $this->RestResponse->saveFailResponse('Users', 'change_pw', false, $message, $this->response->type());
-                        }
-                        $abortPost = true;
-                        $this->Flash->error($message);
-                    }
-                    unset($this->request->data['User']['current_password']);
-                } else if (!$this->_isRest()) {
-                    $message = __('Please enter your current password to continue.');
-                    if ($this->_isRest()) {
-                        return $this->RestResponse->saveFailResponse('Users', 'change_pw', false, $message, $this->response->type());
-                    }
-                    $abortPost = true;
-                    $this->Flash->info($message);
-                }
-            }
-            $hashed = $this->User->verifyPassword($this->Auth->user('id'), $this->request->data['User']['password']);
-            if ($hashed) {
-                $message = __('Submitted new password cannot be the same as the current one');
-                $abortPost = true;
-            }
-            if (!$abortPost) {
-                // What fields should be saved (allowed to be saved)
-                $user['User']['change_pw'] = 0;
-                $user['User']['password'] = $this->request->data['User']['password'];
-                if ($this->_isRest()) {
-                    $user['User']['confirm_password'] = $this->request->data['User']['password'];
-                } else {
-                    $user['User']['confirm_password'] = $this->request->data['User']['confirm_password'];
-                }
-                $temp = $user['User']['password'];
-                // Save the data
-                if ($this->User->save($user)) {
-                    $message = __('Password Changed.');
-                    $this->User->extralog($this->Auth->user(), "change_pw", null, null, $user);
-                    if ($this->_isRest()) {
-                        return $this->RestResponse->saveSuccessResponse('User', 'change_pw', false, $this->response->type(), $message);
-                    }
-                    $this->Flash->success($message);
-                    $this->redirect(array('action' => 'view', $id));
-                } else {
-                    $message = __('The password could not be updated. Make sure you meet the minimum password length / complexity requirements.');
-                    if ($this->_isRest()) {
-                        return $this->RestResponse->saveFailResponse('Users', 'change_pw', false, $message, $this->response->type());
-                    }
-                    $this->Flash->error($message);
-                }
-            } else {
-                if ($this->_isRest()) {
-                    return $this->RestResponse->saveFailResponse('Users', 'change_pw', false, $message, $this->response->type());
-                } else {
-                    $this->Flash->error($message);
-                }
-            }
+            return $this->__pw_change($user, 'change_pw', $abortPost);
         }
         if ($this->_isRest()) {
             return $this->RestResponse->describe('Users', 'change_pw', false, $this->response->type());
@@ -461,7 +476,8 @@ class UsersController extends AppController
                     'last_api_access',
                     'force_logout',
                     'date_created',
-                    'date_modified'
+                    'date_modified',
+                    'last_pw_change'
                 ),
                 'contain' => array(
                     'Organisation' => array('id', 'name'),
@@ -673,6 +689,7 @@ class UsersController extends AppController
                 }
             }
             $this->request->data['User']['date_created'] = time();
+            $this->request->data['User']['last_pw_change'] = $this->request->data['User']['date_created'];
             if (!array_key_exists($this->request->data['User']['role_id'], $syncRoles)) {
                 $this->request->data['User']['server_id'] = 0;
             }
@@ -744,7 +761,7 @@ class UsersController extends AppController
                         $this->Flash->error(__('The user could not be saved. Invalid organisation.'));
                     }
                 } else {
-                    $fieldList = array('password', 'email', 'external_auth_required', 'external_auth_key', 'enable_password', 'confirm_password', 'org_id', 'role_id', 'authkey', 'nids_sid', 'server_id', 'gpgkey', 'certif_public', 'autoalert', 'contactalert', 'disabled', 'invited_by', 'change_pw', 'termsaccepted', 'newsread', 'date_created', 'date_modified');
+                    $fieldList = array('password', 'email', 'external_auth_required', 'external_auth_key', 'enable_password', 'confirm_password', 'org_id', 'role_id', 'authkey', 'nids_sid', 'server_id', 'gpgkey', 'certif_public', 'autoalert', 'contactalert', 'disabled', 'invited_by', 'change_pw', 'termsaccepted', 'newsread', 'date_created', 'date_modified', 'last_pw_change');
                     if ($this->User->save($this->request->data, true, $fieldList)) {
                         $notification_message = '';
                         if (!empty($this->request->data['User']['notify'])) {
@@ -939,6 +956,8 @@ class UsersController extends AppController
                     $this->__canChangePassword()
                 ) {
                     $fields[] = 'password';
+                    $fields[] = 'last_pw_change';
+                    $this->request->data['User']['last_pw_change'] = time();
                     if ($this->_isRest() && !isset($this->request->data['User']['confirm_password'])) {
                         $this->request->data['User']['confirm_password'] = $this->request->data['User']['password'];
                         $fields[] = 'confirm_password';
@@ -1369,6 +1388,7 @@ class UsersController extends AppController
         unset($user['User']['password']);
         $user['User']['action'] = 'logout';
         $this->User->save($user['User'], true, array('id'));
+        $this->Session->write('otp_secret', null);
         $this->redirect($this->Auth->logout());
     }
 
@@ -1820,7 +1840,7 @@ class UsersController extends AppController
     public function totp_new()
     {
         if (Configure::read('LinOTPAuth.enabled')) {
-            $this->Flash->error(__("LinOTP is enabled for this instance. Build-in TOTP should not be used."));
+            $this->Flash->error(__("LinOTP is enabled for this instance. Built-in TOTP should not be used."));
             $this->redirect($this->referer());
         }
         if (!class_exists('\OTPHP\TOTP') || !class_exists('\BaconQrCode\Writer')) {
@@ -1841,17 +1861,23 @@ class UsersController extends AppController
         }
         // do not allow this page to be accessed if the current already has a TOTP. Just redirect to the users details page with a Flash->error()
         if ($user['User']['totp']) { 
-            $this->Flash->error(__("Your account already has an TOTP. Please contact your organisational administrator to change or delete it."));
+            $this->Flash->error(__("Your account already has a TOTP. Please contact your organisational administrator to change or delete it."));
             $this->redirect($this->referer());
         }
 
-        $secret = $this->Session->read('otp_secret');  // Reload secret from session.
-        if ($secret) {
-            $totp = \OTPHP\TOTP::create($secret);
-        } else {
+        if ($this->request->is('get')) {
             $totp = \OTPHP\TOTP::create();
             $secret = $totp->getSecret();
-            $this->Session->write('otp_secret', $secret);  // Store in session, this is to keep the same QR code even if the page refreshes.
+            $this->Session->write('otp_secret', $secret);  // Store in session, we want to create a new secret each time the totp_new() function is queried via a GET (this will not impede incorrect confirmation attempty)
+        } else {
+            $secret = $this->Session->read('otp_secret');  // Reload secret from session.
+            if ($secret) {
+                $totp = \OTPHP\TOTP::create($secret);
+            } else {
+                $totp = \OTPHP\TOTP::create();
+                $secret = $totp->getSecret();
+                $this->Session->write('otp_secret', $secret); // Store in session, we want to keep reusing the same QR code until the user correctly enters the generated key on their authenticator
+            }
         }
         if ($this->request->is('post') && isset($this->request->data['User']['otp'])) {
             if ($totp->verify(trim($this->request->data['User']['otp']))) {
@@ -2963,7 +2989,8 @@ class UsersController extends AppController
     public function viewPeriodicSummary(string $period)
     {
         $userId = $this->Auth->user('id');
-        $summary = $this->User->generatePeriodicSummary($userId, $period);
+        $lastdays = $this->request->params['named']['lastdays'] ?? false;
+        $summary = $this->User->generatePeriodicSummary($userId, $period, true, $lastdays);
         $periodicSettings = $this->User->fetchPeriodicSettingForUser($userId);
         $this->set('periodic_settings', $periodicSettings);
         $this->set('summary', $summary);
@@ -3084,4 +3111,69 @@ class UsersController extends AppController
         # To use this, set Plugin.CustomAuth_custom_logout to /users/logout401
         $this->response->statusCode(401);
     }
+
+    public function forgot()
+    {
+        if (empty(Configure::read('Security.allow_password_forgotten'))) {
+            $this->Flash->error(__('This feature is disabled.'));
+            $this->redirect('/');
+        }
+        if (!empty($this->Auth->user()) && !$this->_isRest()) {
+            $this->Flash->info(__('You are already logged in, no need to ask for a password reset. Log out first.'));
+            $this->redirect('/');
+        }
+        if ($this->request->is('post')) {
+            if (empty($this->request->data['User'])) {
+                $this->request->data = ['User' => $this->request->data];
+            }
+            if (empty($this->request->data['User']['email'])) {
+                throw new MethodNotAllowedException(__('No email provided, cannot generate password reset message.'));
+            }
+            $user = [
+                'id' => 0,
+                'email' => 'SYSTEM',
+                'Organisation' => [
+                    'name' => 'SYSTEM'
+                ]
+            ];
+            $this->loadModel('Log');
+            $this->Log->createLogEntry($user, 'forgot', 'User', 0, 'Password reset requested for: ' . $this->request->data['User']['email']);
+            $this->User->forgotRouter($this->request->data['User']['email'], $this->_remoteIp());
+            $message = __('Password reset request submitted. If a valid user is found, you should receive an e-mail with a temporary reset link momentarily. Please be advised that this link is only valid for 10 minutes.');
+            if ($this->_isRest()) {
+                return $this->RestResponse->saveSuccessResponse('User', 'forgot', false, $this->response->type(), $message);
+            }
+            $this->Flash->info($message);
+            $this->redirect('/');
+        }
+    }
+
+    public function password_reset($token)
+    {
+        if (empty(Configure::read('Security.allow_password_forgotten'))) {
+            $this->Flash->error(__('This feature is disabled.'));
+            $this->redirect('/');
+        }
+        $this->loadModel('Server');
+        $this->set('complexity', !empty(Configure::read('Security.password_policy_complexity')) ? Configure::read('Security.password_policy_complexity') : $this->Server->serverSettings['Security']['password_policy_complexity']['value']);
+        $this->set('length', !empty(Configure::read('Security.password_policy_length')) ? Configure::read('Security.password_policy_length') : $this->Server->serverSettings['Security']['password_policy_length']['value']);
+        if (!empty($this->Auth->user()) && !$this->_isRest()) {
+            $this->redirect('/');
+        }
+        $user = $this->User->fetchForgottenPasswordUser($token);
+        if (empty($user)) {
+            $message = __('Invalid token, or password request token already expired.');
+            if ($this->_isRest()) {
+                throw new MethodNotAllowedException($message);
+            } else {
+                $this->Flash->error($message);
+                $this->redirect('/');
+            }
+        }
+        if ($this->request->is('post') || $this->request->is('put')) {
+            $abortPost = false;
+            return $this->__pw_change(['User' => $user], 'password_reset', $abortPost, $token);
+        }
+    }
+
 }

app/Lib/Dashboard/APIActivityWidget.php

@@ -13,6 +13,8 @@ class APIActivityWidget
         'month' => 'Who contributed most this month? (boolean)',
         'previous_month' => 'Who contributed most the previous, finished month? (boolean)',
         'year' => 'Which contributed most this year? (boolean)',
+        'start_date' => 'The ISO 8601 date format at which to start',
+        'end_date' => 'The ISO 8601 date format at which to end. (Leave empty for today)',
     ];
     public $description = 'Basic widget showing some server statistics in regards to MISP.';
     public $cacheLifetime = 10;
@@ -32,6 +34,11 @@ class APIActivityWidget
             $end = new DateTime(date('Y-m-d', strtotime('last day of last month 23:59:59', time())));
         } else if (!empty($options['year'])) {
             $begin = new DateTime(date('Y-m-d', strtotime('first day of this year 00:00:00', time())));
+        } else if (!empty($options['start_date'])) {
+            $begin = new DateTime($options['start_date']);
+            if (!empty($options['end_date'])) {
+                $end = new DateTime($options['end_date']);
+            }
         } else {
             $begin = new DateTime(date('Y-m-d', strtotime('-7 days', time())));;
         }

app/Lib/Dashboard/EventEvolutionLineWidget.php

@@ -92,7 +92,7 @@ class EventEvolutionLineWidget
             'recursive' => -1,
             'conditions' => $eparams['conditions'],
             'fields' => ['DATE_FORMAT(FROM_UNIXTIME(Event.publish_timestamp), "%Y-%m") AS date', 'count(id) AS count'],
-            'group' => 'MONTH(FROM_UNIXTIME(Event.publish_timestamp)), YEAR(FROM_UNIXTIME(Event.publish_timestamp))'
+            'group' => ['MONTH(FROM_UNIXTIME(Event.publish_timestamp)), YEAR(FROM_UNIXTIME(Event.publish_timestamp))',]
             
         ]);
 

app/Lib/Dashboard/LoginsWidget.php

@@ -13,6 +13,8 @@ class LoginsWidget
         'month' => 'Who contributed most this month? (boolean)',
         'previous_month' => 'Who contributed most the previous, finished month? (boolean)',
         'year' => 'Which contributed most this year? (boolean)',
+        'start_date' => 'The ISO 8601 date format at which to start',
+        'end_date' => 'The ISO 8601 date format at which to end. (Leave empty for today)',
     ];
     public $description = 'Basic widget showing some server statistics in regards to MISP.';
     public $cacheLifetime = 10;
@@ -32,6 +34,12 @@ class LoginsWidget
             $end = date('Y-m-d H:i:s', strtotime('last day of last month 23:59:59', time()));
         } else if (!empty($options['year'])) {
             $begin = date('Y-m-d', strtotime('first day of this year 00:00:00', time()));
+        } else if (!empty($options['start_date'])) {
+            $begin = date($options['start_date']);
+            $end = [];
+            if (!empty($options['end_date'])) {
+                $end = date($options['end_date']);
+            }
         } else {
             $begin = date('Y-m-d H:i:s', strtotime('-7 days', time()));
         }

app/Lib/Dashboard/NewOrgsWidget.php

@@ -16,6 +16,10 @@ class NewOrgsWidget
         'days' => 'How many days back should the list go - for example, setting 7 will only show the organisations that were added in the past 7 days. (integer)',
         'month' => 'Which organisations have been added this month? (boolean)',
         'previous_month' => 'Who contributed most the previous, finished month? (boolean)',
+        'first_half_year' => 'Who contributed most the first half-year (between Jan and June)? (boolean)',
+        'second_half_year' => 'Who contributed most the second half-year (between July and Dec)? (boolean)',
+        'start_date' => 'The ISO 8601 date format at which to start',
+        'end_date' => 'The ISO 8601 date format at which to end. (Leave empty for today)',
         'year' => 'Which organisations have been added this year? (boolean)',
         'local' => 'Should the list only show local organisations? (boolean or list of booleans, defaults to 1. To get both sets, use [0,1])',
         'fields' => 'Which fields should be displayed, by default all are selected. Pass a list with the following options: [id, uuid, name, sector, type, nationality, creation_date]'
@@ -59,6 +63,23 @@ class NewOrgsWidget
         } else if (!empty($options['year'])) {
             $condition = strtotime('first day of this year 00:00:00', time());
             $this->tableDescription = __('The %d newest organisations created during the current year', $limit);
+        } else if (!empty($options['first_half_year'])) {
+            $condition =  strtotime('first day of january this year 00:00:00', time());
+            $end_condition =  strtotime('last day of june this year 23:59:59', time());
+            $this->tableDescription = __('The %d newest organisations created during the last half year', $limit);
+        } else if (!empty($options['second_half_year'])) {
+            $condition =  strtotime('first day of july this year 00:00:00', time());
+            $end_condition = strtotime('last day of december this year 23:59:59', time());
+            $this->tableDescription = __('The %d newest organisations created during the current half year', $limit);
+        } else if (!empty($options['start_date'])) {
+            $condition = strtotime($options['start_date'], time());
+            $end_condition = [];
+            if (empty($options['end_date'])) {
+                $end_condition = time();
+            } else {
+                $end_condition = strtotime($options['end_date'], time());
+            }
+            $this->tableDescription = __('The %d newest organisations created since %s', $limit, $options['start_date']);
         } else {
             $this->tableDescription = __('The %d newest organisations created', $limit);
             return null;
@@ -142,7 +163,7 @@ class NewOrgsWidget
         }
         $timeConditions = $this->timeConditions($options);
         if ($timeConditions) {
-            $params['conditions']['AND'][] = ['Organisation.date_created >=' => $timeConditions];
+            $params['conditions']['AND'][]['AND'] = $timeConditions;
         }
         if (isset($options['fields'])) {
             $fields = [];

app/Lib/Dashboard/NewUsersWidget.php

@@ -17,6 +17,8 @@ class NewUsersWidget
         'month' => 'Which organisations have been added this month? (boolean)',
         'previous_month' => 'Who contributed most the previous, finished month? (boolean)',
         'year' => 'Which organisations have been added this year? (boolean)',
+        'start_date' => 'The ISO 8601 date format at which to start',
+        'end_date' => 'The ISO 8601 date format at which to end. (Leave empty for today)',
         'fields' => 'Which fields should be displayed, by default all are selected. Pass a list with the following options: [id, email, Organisation.name, Role.name, date_created]'
     ];
     private $validFilterKeys = [
@@ -65,6 +67,15 @@ class NewUsersWidget
         } else if (!empty($options['year'])) {
             $condition = strtotime('first day of this year 00:00:00', time());
             $this->tableDescription = __('The %d newest users created during the current year', $limit);
+        } else if (!empty($options['start_date'])) {
+            $condition = strtotime($options['start_date'], time());
+            $end_condition = [];
+            if (empty($options['end_date'])) {
+                $end_condition = time();
+            } else {
+                $end_condition = strtotime($options['end_date'], time());
+            }
+            $this->tableDescription = __('The %d newest organisations created since %s', $limit, $options['start_date']);
         } else {
             $this->tableDescription = __('The %d newest users created', $limit);
             return null;

app/Lib/Dashboard/OrgContributionToplistWidget.php

@@ -11,6 +11,10 @@ class OrgContributionToplistWidget
         'month' => 'Who contributed most this month? (boolean)',
         'previous_month' => 'Who contributed most the previous, finished month? (boolean)',
         'year' => 'Which contributed most this year? (boolean)',
+        'first_half_year' => 'Which contributed most the first half-year (between Jan and June)? (boolean)',
+        'second_half_year' => 'Which contributed most the second half-year (between July and Dec)? (boolean)',
+        'start_date' => 'The ISO 8601 date format at which to start',
+        'end_date' => 'The ISO 8601 date format at which to end. (Leave empty for today)',
         'filter' => 'A list of filters by organisation meta information (nationality, sector, type, name, uuid, local (- expects a boolean or a list of boolean values)) to include. (dictionary, prepending values with ! uses them as a negation)',
         'limit' => 'Limits the number of displayed tags. Default: 10'
     ];
@@ -37,7 +41,6 @@ class OrgContributionToplistWidget
 
     private function timeConditions($options)
     {
-        $limit = empty($options['limit']) ? 10 : $options['limit'];
         if (!empty($options['days'])) {
             $condition = strtotime(sprintf("-%s days", $options['days']));
         } else if (!empty($options['month'])) {
@@ -47,6 +50,20 @@ class OrgContributionToplistWidget
             $end_condition = strtotime('last day of last month 23:59:59', time());
         } else if (!empty($options['year'])) {
             $condition = strtotime('first day of this year 00:00:00', time());
+        } else if (!empty($options['last_half_year'])) {
+            $condition =  strtotime('first day of january this year 00:00:00', time());
+            $end_condition =  strtotime('last day of june this year 23:59:59', time());
+        } else if (!empty($options['current_half'])) {
+            $condition =  strtotime('first day of july this year 00:00:00', time());
+            $end_condition = strtotime('last day of december this year 23:59:59', time());
+        } else if (!empty($options['start_date'])) {
+            $condition = strtotime($options['start_date'], time());
+            $end_condition = [];
+            if (empty($options['end_date'])) {
+                $end_condition = time();
+            } else {
+                $end_condition = strtotime($options['end_date'], time());
+            }
         } else {
             return null;
         }
@@ -54,12 +71,12 @@ class OrgContributionToplistWidget
         if (!empty($condition)) {
             $datetime = new DateTime();
             $datetime->setTimestamp($condition);
-            $conditions['Event.timestamp >='] = $datetime->format('Y-m-d H:i:s');
+            $conditions['Event.timestamp >='] = $datetime->getTimestamp();
         }
         if (!empty($end_condition)) {
             $datetime = new DateTime();
             $datetime->setTimestamp($end_condition);
-            $conditions['Event.timestamp <='] = $datetime->format('Y-m-d H:i:s');
+            $conditions['Event.timestamp <='] = $datetime->getTimestamp();
         }
         return $conditions;
     }
@@ -68,10 +85,6 @@ class OrgContributionToplistWidget
     public function handler($user, $options = array())
     {
         $params = ['conditions' => []];
-        $timeConditions = $this->timeConditions($options);
-        if ($timeConditions) {
-            $params['conditions']['AND'][] = $timeConditions;
-        }
         if (!empty($options['filter']) && is_array($options['filter'])) {
             foreach ($this->validFilterKeys as $filterKey) {
                 if (!empty($options['filter'][$filterKey])) {
@@ -101,7 +114,12 @@ class OrgContributionToplistWidget
             'fields' => ['Organisation.id', 'Organisation.name'],
             'conditions' => $params['conditions']
         ]);
-        $conditions = ['Event.orgc_id IN' => array_keys($org_ids)];
+        $conditions = [];
+        $conditions['AND'][] = ['Event.orgc_id IN' => array_keys($org_ids)];
+        $timeConditions = $this->timeConditions($options);
+        if ($timeConditions) {
+            $conditions['AND'][]['AND'] = $timeConditions;
+        }
         $this->Event = ClassRegistry::init('Event');
         $this->Event->virtualFields['frequency'] = 0;
         $orgs = $this->Event->find('all', [

app/Lib/Dashboard/OrgEventsWidget.php

@@ -0,0 +1,121 @@
+<?php
+/**
+* Org Events widget which reportes the number of events created monthly by each local organizations
+*
+*/
+class OrgEventsWidget
+{
+    public $title = 'Org Events';
+    public $render = 'MultiLineChart';
+    public $width = 8;
+    public $height = 6;
+    public $description = 'A graph to show the monthly number of events per organisation';
+    public $cacheLifetime = 10;
+    public $autoRefreshDelay = false;
+    public $params = array (
+        'blocklist_orgs' => 'A list of organisation names to filter out',
+        'months' => 'Number of past months to consider for the graph',
+        'logarithmic' => 'Visualize data on logarithmic scale'
+    );
+
+    public $placeholder =
+'{
+    "blocklist_orgs": ["Orgs to filter"],
+    "months": "6",
+    "logarithmic": "true"
+}';
+
+
+
+
+
+    /*
+    * Target_month must be from 1 to 12
+    * Target year must be 4 digits
+    */
+    private function org_events_count($user, $org, $target_month, $target_year) {
+        $events_count = 0;
+
+        $start_date = $target_year.'-'.$target_month.'-01';
+        if($target_month == 12) {
+            $end_date = ($target_year+1).'-01-01';
+        } else {
+            $end_date = $target_year.'-'.($target_month+1).'-01';
+        }
+        $conditions = array('Event.orgc_id' => $org['Organisation']['id'], 'Event.date >=' => $start_date, 'Event.date <' => $end_date);
+
+        //This is required to enforce the ACL (not pull directly from the DB)
+        $eventIds = $this->Event->fetchSimpleEventIds($user, array('conditions' => $conditions));
+
+        if(!empty($eventIds)) {
+            $params = array('Event.id' => $eventIds);
+            $events = $this->Event->find('all', array('conditions' => array('AND' => $params)));
+            foreach($events as $event) {
+                $events_count+= 1;
+            }
+        }
+        return $events_count;
+    }
+
+    private function filter_ghost_orgs(&$data, $orgs){
+        foreach ($data['data'] as &$item) {
+            foreach(array_keys($orgs) as $org_name) {
+                unset($item[$org_name]);
+            }
+        }
+    }
+
+    public function handler($user, $options = array())
+    {
+        $this->Log = ClassRegistry::init('Log');
+        $this->Org = ClassRegistry::init('Organisation');
+        $this->Event = ClassRegistry::init('Event');
+        $orgs = $this->Org->find('all', array( 'conditions' => array('Organisation.local' => 1)));
+        $current_month = date('n');
+        $current_year = date('Y');
+        $limit = 6; // months
+        if(!empty($options['months'])) {
+            $limit = (int) ($options['months']);
+        }
+        $offset = 0;
+        $ghost_orgs = array(); // track orgs without any contribution
+        // We start by putting all orgs_id in there:
+        foreach($orgs as $org) {
+            // We check for blocklisted orgs
+            if(!empty($options['blocklist_orgs']) && in_array($org['Organisation']['name'], $options['blocklist_orgs'])) {
+                unset($orgs[$offset]);
+            } else {
+                $ghost_orgs[$org['Organisation']['name']] = true;
+            }
+            $offset++;
+        }
+        $data = array();
+        $data['data'] = array();
+        for ($i=0; $i < $limit; $i++) {
+            $target_month = $current_month - $i;
+            $target_year = $current_year;
+            if ($target_month < 1) {
+                $target_month += 12;
+                $target_year -= 1;
+            }
+            $item = array();
+            $item ['date'] = $target_year.'-'.$target_month.'-01';
+            foreach($orgs as $org) {
+                    $count = $this->org_events_count($user, $org, $target_month, $target_year);
+                    if($options['logarithmic'] === "true" || $options['logarithmic'] === "1") {
+                        $item[$org['Organisation']['name']] = (int) round(log($count, 1.1));  // taking the logarithmic view
+                    } else if(empty($options['logarithmic']) || $options['logarithmic'] === "true" || $options['logarithmic'] === "1"){
+                        $item[$org['Organisation']['name']] = $count;
+                    }
+                    // if a positive score is detected at least once it's enough to be
+                    // considered for the graph
+                    if($count > 0) {
+                        unset($ghost_orgs[$org['Organisation']['name']]);
+                    }
+                }
+            $data['data'][] = $item;
+        }
+        $this->filter_ghost_orgs($data, $ghost_orgs);
+        return $data;
+    }
+}

app/Lib/Dashboard/OrgEvolutionLineWidget.php

@@ -83,7 +83,7 @@ class OrgEvolutionLineWidget
             'recursive' => -1,
             'conditions' => $params['conditions'],
             'fields' => ['DATE_FORMAT(date_created, "%Y-%m") AS date', 'count(id) AS count'],
-            'group' => 'MONTH(date_created), YEAR(date_created)'
+            'group' => ['MONTH(date_created), YEAR(date_created)', ]
             
         ]);
         usort($raw, [$this, 'sortByCreationDate']);

app/Lib/Dashboard/OrganisationMapWidget.php

@@ -8,7 +8,9 @@ class OrganisationMapWidget
     public $height = 4;
     public $params = [
         'filter' => 'A list of filters by organisation meta information (sector, type, local (- expects a boolean or a list of boolean values)) to include. (dictionary, prepending values with ! uses them as a negation)',
-        'limit' => 'Limits the number of displayed tags. Default: 10'
+        'start_date' => 'The ISO 8601 date format at which to start',
+        'end_date' => 'The ISO 8601 date format at which to end. (Leave empty for today)',
+        'limit' => 'Limits the number of displayed tags. Default: 10',
     ];
     public $cacheLifetime = null;
     public $autoRefreshDelay = false;
@@ -58,6 +60,12 @@ class OrganisationMapWidget
                 }
             }
         }
+        if (!empty($options['start_date'])) {
+            $params['conditions']['AND']['Organisation.date_created >='] = (new DateTime($options['start_date']))->format('Y-m-d H:i:s');
+            if (!empty($options['end_date'])) {
+                $params['conditions']['AND']['Organisation.date_created <='] = (new DateTime($options['end_date']))->format('Y-m-d H:i:s');
+            }
+        }
         $this->Organisation = ClassRegistry::init('Organisation');
         $orgs = $this->Organisation->find('all', [
             'recursive' => -1,

app/Lib/Dashboard/TrendingAttributesWidget.php

@@ -99,7 +99,7 @@ class TrendingAttributesWidget
             $values = $attributeModel->find('all', [
                 'recursive' => -1,
                 'fields' => ['Attribute.value1', 'count(Attribute.value1) as Attribute__frequency'],
-                'group' => ['Attribute.value1'],
+                'group' => ['Attribute.value1', ],
                 'conditions' => $conditions,
                 'contain' => ['Event.orgc_id'],
                 'order' => 'count(Attribute.value1) desc',

app/Lib/Dashboard/TrendingTagsWidget.php

@@ -30,7 +30,7 @@ class TrendingTagsWidget
 	    /** @var Event $eventModel */
         $eventModel = ClassRegistry::init('Event');
         $threshold = empty($options['threshold']) ? 10 : $options['threshold'];
-        if (is_string($options['time_window']) && substr($options['time_window'], -1) === 'd') {
+        if (!empty($options['time_window']) && is_string($options['time_window']) && substr($options['time_window'], -1) === 'd') {
             $time_window = ((int)substr($options['time_window'], 0, -1)) * 24 * 60 * 60;
         } else {
             $time_window = empty($options['time_window']) ? (7 * 24 * 60 * 60) : (int)$options['time_window'];
@@ -44,6 +44,7 @@ class TrendingTagsWidget
 
         $tagColours = [];
         $allTags = [];
+        $data = [];
         $this->render = $this->getRenderer($options);
         if (!empty($options['over_time'])) {
 
@@ -71,7 +72,6 @@ class TrendingTagsWidget
                 }
             }
 
-            $data = [];
             $data['data'] = [];
             foreach($tagOvertime as $date => $tagCount) {
                 $item = [];

app/Lib/Dashboard/UsageDataWidget.php

@@ -10,6 +10,8 @@ class UsageDataWidget
     public $autoRefreshDelay = false;
     public $params = [
         'filter' => 'A list of filters by organisation meta information (nationality, sector, type, name, uuid) to include. (dictionary, prepending values with ! uses them as a negation)',
+        'start_date' => 'The ISO 8601 date format for which to show changes',
+        'end_date' => 'The ISO 8601 date format for which to show changes. (Leave empty for today)',
     ];
     private $User = null;
     private $Event = null;
@@ -52,6 +54,7 @@ class UsageDataWidget
         $this->Thread = ClassRegistry::init('Thread');
         $this->Correlation = ClassRegistry::init('Correlation');
         $thisMonth = strtotime('first day of this month');
+        $hasDateRange = !empty($options['start_date']);
         $orgConditions = [];
         $orgIdList = null;
         if (!empty($options['filter']) && is_array($options['filter'])) {
@@ -98,12 +101,12 @@ class UsageDataWidget
             'Events' => [
                 'title' => 'Events',
                 'value' => $eventsCount,
-                'change' => $this->getEventsCountMonth($orgConditions, $orgIdList, $thisMonth)
+                'change' => $hasDateRange ? $this->getEventsCountDateRange($orgConditions, $orgIdList, $options) : $this->getEventsCountMonth($orgConditions, $orgIdList, $thisMonth)
             ],
             'Attributes' => [
                 'title' => 'Attributes',
                 'value' => $attributesCount,
-                'change' => $this->getAttributesCountMonth($orgConditions, $orgIdList, $thisMonth)
+                'change' => $hasDateRange ? $this->getAttributesCountDateRange($orgConditions, $orgIdList, $options) : $this->getAttributesCountMonth($orgConditions, $orgIdList, $thisMonth)
             ],
             'Attributes / event' => [
                 'title' => 'Attributes / event',
@@ -120,7 +123,7 @@ class UsageDataWidget
             'Users' => [
                 'title' => 'Users',
                 'value' => $usersCount,
-                'change' => $this->getUsersCountMonth($orgConditions, $orgIdList, $thisMonth)
+                'change' => $hasDateRange ? $this->getUsersCountDateRange($orgConditions, $orgIdList, $options) : $this->getUsersCountMonth($orgConditions, $orgIdList, $thisMonth)
             ],
             'Users with PGP keys' => [
                 'title' => 'Users with PGP keys',
@@ -133,28 +136,28 @@ class UsageDataWidget
             'Organisations' => [
                 'title' => 'Organisations',
                 'value' => $this->getOrgsCount($orgConditions, $orgIdList, $thisMonth),
-                'change' => $this->getOrgsCountMonth($orgConditions, $orgIdList, $thisMonth)
+                'change' => $hasDateRange ? $this->getOrgsCountDateRange($orgConditions, $orgIdList, $options) : $this->getOrgsCountMonth($orgConditions, $orgIdList, $thisMonth)
             ],
             'Local organisations' => [
                 'title' => 'Local organisations',
                 'value' => $localOrgsCount,
-                'change' => $this->getLocalOrgsCountMonth($orgConditions, $orgIdList, $thisMonth)
+                'change' => $hasDateRange ? $this->getLocalOrgsCountDateRange($orgConditions, $orgIdList, $options) : $this->getLocalOrgsCountMonth($orgConditions, $orgIdList, $thisMonth)
             ],
             'Event creator orgs' => [
                 'title' => 'Event creator orgs', 'value' => $this->getContributingOrgsCount($orgConditions, $orgIdList, $thisMonth)
             ],
             'Average users / org' => [
-                'title' => 'Average users / org', 'value' => round($usersCount / $localOrgsCount, 1)
+                'title' => 'Average users / org', 'value' => (!empty($localOrgsCount) ? round($usersCount / $localOrgsCount, 1) : 'N/A')
             ],
             'Discussion threads' => [
                 'title' => 'Discussions threads',
                 'value' => $this->getThreadsCount($orgConditions, $orgIdList, $thisMonth),
-                'change' => $this->getThreadsCountMonth($orgConditions, $orgIdList, $thisMonth)
+                'change' => $hasDateRange ? $this->getThreadsCountDateRange($orgConditions, $orgIdList, $options) : $this->getThreadsCountMonth($orgConditions, $orgIdList, $thisMonth)
             ],
             'Discussion posts' => [
                 'title' => 'Discussion posts',
                 'value' => $this->getPostsCount($orgConditions, $orgIdList, $thisMonth),
-                'change' => $this->getPostsCountMonth($orgConditions, $orgIdList, $thisMonth)
+                'change' => $hasDateRange ? $this->getPostsCountDateRange($orgConditions, $orgIdList, $options) : $this->getPostsCountMonth($orgConditions, $orgIdList, $thisMonth)
             ]
         ];
         if(!empty(Configure::read('Security.advanced_authkeys'))){
@@ -165,6 +168,20 @@ class UsageDataWidget
         return $statistics;
     }
 
+    private function prepareDateRangeConditions(array $options, $datefield, $convertToTimestamp = false): array
+    {
+        $timeConditions = [];
+        if (!empty($options['start_date'])) {
+            $sd = new DateTime($options['start_date']);
+            $timeConditions[sprintf('%s >=', $datefield)] = $convertToTimestamp? $sd->getTimestamp() : $sd->format('Y-m-d H:i:s');
+            if (!empty($options['end_date'])) {
+                $ed = new DateTime($options['end_date']);
+                $timeConditions[sprintf('%s <=', $datefield)] = $convertToTimestamp? $ed->getTimestamp() : $ed->format('Y-m-d H:i:s');
+            }
+        }
+        return $timeConditions;
+    }
+
     private function getEventsCount($orgConditions, $orgIdList, $thisMonth)
     {
         $conditions = [];
@@ -202,6 +219,18 @@ class UsageDataWidget
         ]);
     }
 
+    private function getEventsCountDateRange($orgConditions, $orgIdList, $options)
+    {
+        $conditions = $this->prepareDateRangeConditions($options, 'Event.timestamp', true);
+        if (!empty($orgIdList)) {
+            $conditions['AND'][] = ['Event.orgc_id IN' => $orgIdList];
+        }
+        return $this->Event->find('count', [
+            'conditions' => $conditions,
+            'recursive' => -1
+        ]);
+    }
+
     private function getAttributesCount($orgConditions, $orgIdList, $thisMonth)
     {
         $conditions = ['Attribute.deleted' => 0];
@@ -234,6 +263,20 @@ class UsageDataWidget
         ]);
     }
 
+    private function getAttributesCountDateRange($orgConditions, $orgIdList, $options)
+    {
+        $conditions = $this->prepareDateRangeConditions($options, 'Attribute.timestamp', true);
+        $conditions['Attribute.deleted'] = 0;
+        if (!empty($orgIdList)) {
+            $conditions['AND'][] = ['Event.orgc_id IN' => $orgIdList];
+        }
+        return $this->Event->Attribute->find('count', [
+            'conditions' => $conditions,
+            'contain' => 'Event.orgc_id',
+            'recursive' => -1
+        ]);
+    }
+
     private function getOrgsCount($orgConditions, $orgIdList, $thisMonth)
     {
         return $this->User->Organisation->find('count', [
@@ -256,6 +299,16 @@ class UsageDataWidget
         ]);
     }
 
+    private function getOrgsCountDateRange($orgConditions, $orgIdList, $options)
+    {
+        return $this->User->Organisation->find('count', [
+            'conditions' => [
+                'AND' => $orgConditions,
+                $this->prepareDateRangeConditions($options, 'Organisation.date_created'),
+            ]
+        ]);
+    }
+
     private function getLocalOrgsCount($orgConditions, $orgIdList, $thisMonth)
     {
         return $this->User->Organisation->find('count', [
@@ -280,6 +333,17 @@ class UsageDataWidget
         ]);
     }
 
+    private function getLocalOrgsCountDateRange($orgConditions, $orgIdList, $options)
+    {
+        return $this->User->Organisation->find('count', [
+            'conditions' => [
+                'Organisation.local' => 1,
+                'AND' => $orgConditions,
+                $this->prepareDateRangeConditions($options, 'Organisation.date_created'),
+            ]
+        ]);
+    }
+
     private function getProposalsCount($orgConditions, $orgIdList, $thisMonth)
     {
         $conditions = ['deleted' => 0];
@@ -316,6 +380,18 @@ class UsageDataWidget
         ]);
     }
 
+    private function getUsersCountDateRange($orgConditions, $orgIdList, $options)
+    {
+        $conditions = $this->prepareDateRangeConditions($options, 'User.date_created', true);
+        if (!empty($orgIdList)) {
+            $conditions['User.org_id IN'] = $orgIdList;
+        }
+        return $this->User->find('count', [
+            'recursive' => -1,
+            'conditions' => $conditions
+        ]);
+    }
+
     private function getUsersCountPgp($orgConditions, $orgIdList, $thisMonth)
     {
         $conditions = ['User.gpgkey !=' => ''];
@@ -332,7 +408,7 @@ class UsageDataWidget
     {
         $conditions = [];
         if ($orgConditions) {
-            $conditions['AND'][] = ['Event.orgc_id IN' => $orgIdList];
+            $conditions['AND'][] = ['Event.orgc_id IN' => (!empty($orgIdList) ? $orgIdList : [-1])];
         }
         return $this->Event->find('count', [
             'recursive' => -1,
@@ -345,7 +421,7 @@ class UsageDataWidget
     {
         $conditions = ['Thread.post_count >' => 0];
         if ($orgConditions) {
-            $conditions['AND'][] = ['Thread.org_id IN' => $orgIdList];
+            $conditions['AND'][] = ['Thread.org_id IN' => (!empty($orgIdList) ? $orgIdList : [-1])];
         }
         return $this->Thread->find('count', [
             'conditions' => $conditions,
@@ -360,7 +436,20 @@ class UsageDataWidget
             'Thread.date_created >=' => $thisMonth
         ];
         if ($orgConditions) {
-            $conditions['AND'][] = ['Thread.org_id IN' => $orgIdList];
+            $conditions['AND'][] = ['Thread.org_id IN' => (!empty($orgIdList) ? $orgIdList : [-1])];
+        }
+        return $this->Thread->find('count', [
+            'conditions' => $conditions,
+            'recursive' => -1
+        ]);
+    }
+
+    private function getThreadsCountDateRange($orgConditions, $orgIdList, $options)
+    {
+        $conditions = $this->prepareDateRangeConditions($options, 'Thread.date_created');
+        $conditions['Thread.post_count >'] = 0;
+        if ($orgConditions) {
+            $conditions['AND'][] = ['Thread.org_id IN' => (!empty($orgIdList) ? $orgIdList : [-1])];
         }
         return $this->Thread->find('count', [
             'conditions' => $conditions,
@@ -372,7 +461,7 @@ class UsageDataWidget
     {
         $conditions = [];
         if ($orgConditions) {
-            $conditions['AND'][] = ['User.org_id IN' => $orgIdList];
+            $conditions['AND'][] = ['User.org_id IN' => (!empty($orgIdList) ? $orgIdList : [-1])];
         }
         return $this->Thread->Post->find('count', [
             'conditions' => $conditions,
@@ -387,7 +476,20 @@ class UsageDataWidget
             'Post.date_created >=' => $thisMonth
         ];
         if ($orgConditions) {
-            $conditions['AND'][] = ['User.org_id IN' => $orgIdList];
+            $conditions['AND'][] = ['User.org_id IN' => (!empty($orgIdList) ? $orgIdList : [-1])];
+        }
+        return $this->Thread->Post->find('count', [
+            'conditions' => $conditions,
+            'contain' => ['User.org_id'],
+            'recursive' => -1
+        ]);
+    }
+
+    private function getPostsCountDateRange($orgConditions, $orgIdList, $options)
+    {
+        $conditions = $this->prepareDateRangeConditions($options, 'Post.date_created');
+        if ($orgConditions) {
+            $conditions['AND'][] = ['User.org_id IN' => (!empty($orgIdList) ? $orgIdList : [-1])];
         }
         return $this->Thread->Post->find('count', [
             'conditions' => $conditions,

app/Lib/Dashboard/UserContributionToplistWidget.php

@@ -11,6 +11,8 @@ class UserContributionToplistWidget
         'month' => 'Who contributed most this month? (boolean)',
         'previous_month' => 'Who contributed most the previous, finished month? (boolean)',
         'year' => 'Which contributed most this year? (boolean)',
+        'start_date' => 'The ISO 8601 date format at which to start',
+        'end_date' => 'The ISO 8601 date format at which to end. (Leave empty for today)',
         'filter' => 'A list of filters by organisation meta information (nationality, sector, type, name, uuid, local (- expects a boolean or a list of boolean values)) to include. (dictionary, prepending values with ! uses them as a negation)',
         'limit' => 'Limits the number of displayed tags. Default: 10'
     ];
@@ -47,6 +49,14 @@ class UserContributionToplistWidget
             $end_condition = strtotime('last day of last month 23:59:59', time());
         } else if (!empty($options['year'])) {
             $condition = strtotime('first day of this year 00:00:00', time());
+        } else if (!empty($options['start_date'])) {
+            $condition = strtotime($options['start_date'], time());
+            $end_condition = [];
+            if (empty($options['end_date'])) {
+                $end_condition = time();
+            } else {
+                $end_condition = strtotime($options['end_date'], time());
+            }
         } else {
             return null;
         }
@@ -54,12 +64,12 @@ class UserContributionToplistWidget
         if (!empty($condition)) {
             $datetime = new DateTime();
             $datetime->setTimestamp($condition);
-            $conditions['Event.timestamp >='] = $datetime->format('Y-m-d H:i:s');
+            $conditions['Event.timestamp >='] = $datetime->getTimestamp();
         }
         if (!empty($end_condition)) {
             $datetime = new DateTime();
             $datetime->setTimestamp($end_condition);
-            $conditions['Event.timestamp <='] = $datetime->format('Y-m-d H:i:s');
+            $conditions['Event.timestamp <='] = $datetime->getTimestamp();
         }
         return $conditions;
     }

app/Lib/Export/StixExport.php

@@ -83,6 +83,12 @@ abstract class StixExport
         if ($this->__empty_file) {
             $this->__tmp_file->close();
             $this->__tmp_file->delete();
+            if (empty($this->__filenames)) {
+                $framing = $this->getFraming();
+                $tmpFile = new TmpFileTool();
+                $tmpFile->write($framing['header'] . $framing['footer']);
+                return $tmpFile;
+            }
         } else {
             if (!empty($this->__event_galaxies)) {
                 $this->__write_event_galaxies();

app/Lib/Tools/AttributeValidationTool.php

@@ -224,7 +224,6 @@ class AttributeValidationTool
         switch ($type) {
             case 'md5':
             case 'imphash':
-            case 'telfhash':
             case 'sha1':
             case 'sha224':
             case 'sha256':
@@ -255,6 +254,11 @@ class AttributeValidationTool
                     return true;
                 }
                 return __('Checksum has an invalid length or format (expected: at least 35 hexadecimal characters, optionally starting with t1 instead of hexadecimal characters). Please double check the value or select type "other".');
+            case 'telfhash':
+                if (self::isTelfhashValid($value)) {
+                    return true;
+                }
+                return __('Checksum has an invalid length or format (expected: %s or %s hexadecimal characters). Please double check the value or select type "other".', 70, 72);
             case 'pehash':
                 if (self::isHashValid('pehash', $value)) {
                     return true;
@@ -635,6 +639,15 @@ class AttributeValidationTool
         return strlen($value) > 35 && ctype_xdigit($value);
     }
 
+    /**
+     * @param string $value
+     * @return bool
+     */
+    private static function isTelfhashValid($value)
+    {
+        return strlen($value) == 70 || strlen($value) == 72;
+    }
+
 
     /**
      * @param string $type

app/Lib/Tools/EventGraphTool.php

@@ -372,6 +372,8 @@
                         'id' => 'tag_edge_id_' . $i,
                         'from' => $attr['id'],
                         'to' => $tag['name'],
+                        'type' => isset($tag['relationship_type']) ? $tag['relationship_type'] : '',
+                        'comment' => '',
                     );
                     $tagSet[$tag['name']] = $tag;
                     array_push($this->__json['relations'], $toPush);
@@ -379,6 +381,7 @@
                 }
             }
 
+            $j = 0;
             foreach ($object as $obj) {
                 $toPush = array(
                     'id' => sprintf('o-%s', $obj['id']),
@@ -393,23 +396,44 @@
                     'comment' => $obj['comment'],
                 );
                 array_push($this->__json['items'], $toPush);
-
-                // Record existing object_relation
-                foreach ($obj['Attribute'] as $attr) {
-                    $this->__json['existing_object_relation'][$attr['object_relation']] = 0; // set-alike
-                }
-
-                // get all tags in the Object's Attributes
+                
+                
+                // get all  attributes and tags in the Object's Attributes
                 $added_value = array();
                 foreach ($obj['Attribute'] as $ObjAttr) {
+                    // Record existing object_relation
+                    $this->__json['existing_object_relation'][$attr['object_relation']] = 0; // set-alike
                     $Tags = $ObjAttr['AttributeTag'];
                     foreach ($Tags as $tag) {
                         $tag = $tag['Tag'];
                         if (!in_array($tag['name'], $added_value)) {
                             $toPush = array(
-                                'id' => "tag_edge_id_" . $i,
+                                'id' => $ObjAttr['id'],
+                                'uuid' => $ObjAttr['uuid'],
+                                'type' => $ObjAttr['type'],
+                                'label' => $ObjAttr['value'],
+                                'event_id' => $ObjAttr['event_id'],
+                                'node_type' => 'attribute',
+                                'comment' => $ObjAttr['comment'],
+                            );
+                            array_push($this->__json['items'], $toPush);
+                            
+                            $toPush = array(
+                                'id' => 'obj_edge_id_' . $j,
                                 'from' => sprintf('o-%s', $obj['id']),
+                                'to' => $ObjAttr['id'],
+                                'type' => '',
+                                'comment' => '',
+                            );
+                            $j = $j + 1;
+                            array_push($this->__json['relations'], $toPush);
+
+                            $toPush = array(
+                                'id' => "tag_edge_id_" . $i,
+                                'from' => $ObjAttr['id'],
                                 'to' => $tag['name'],
+                                'type' => isset($tag['relationship_type']) ? $tag['relationship_type'] : '',
+                                'comment' => '',
                             );
                             $tagSet[$tag['name']] = $tag;
                             array_push($added_value, $tag['name']);

app/Lib/Tools/SyncTool.php

@@ -2,6 +2,9 @@
 
 class SyncTool
 {
+
+    const ALLOWED_CERT_FILE_EXTENSIONS = ['pem', 'crt'];
+
     /**
      * Take a server as parameter and return a HttpSocket object using the ssl options defined in the server settings
      * @param array|null $server
@@ -15,10 +18,10 @@ class SyncTool
         $params = ['compress' => true];
         if (!empty($server)) {
             if (!empty($server[$model]['cert_file'])) {
-                $params['ssl_cafile'] = APP . "files" . DS . "certs" . DS . $server[$model]['id'] . '.pem';
+                $params['ssl_cafile'] = APP . "files" . DS . "certs" . DS . $server[$model]['cert_file'];
             }
             if (!empty($server[$model]['client_cert_file'])) {
-                $params['ssl_local_cert'] = APP . "files" . DS . "certs" . DS . $server[$model]['id'] . '_client.pem';
+                $params['ssl_local_cert'] = APP . "files" . DS . "certs" . DS . $server[$model]['client_cert_file'];
             }
             if (!empty($server[$model]['self_signed'])) {
                 $params['ssl_allow_self_signed'] = true;

app/Lib/Tools/WorkflowGraphTool.php

@@ -6,16 +6,18 @@ class GraphUtil
 {
     public function __construct($graphData)
     {
-        $this->graph = $graphData;
+        $this->graph = array_filter($graphData, function($i) {
+            return $i != '_frames';
+        }, ARRAY_FILTER_USE_KEY);
         $this->numberNodes = count($this->graph);
-        $this->edgeList = $this->_buildEdgeList($graphData);
+        $this->edgeList = $this->_buildEdgeList($this->graph);
         $this->properties = [];
     }
 
     private function _buildEdgeList($graphData): array
     {
         $list = [];
-        foreach ($graphData as $node) {
+        foreach ($graphData as $i => $node) {
             $list[(int)$node['id']] = [];
             foreach (($node['outputs'] ?? []) as $output_id => $outputs) {
                 foreach ($outputs as $connections) {
@@ -356,6 +358,20 @@ class WorkflowRoamingData
 
 class WorkflowGraphTool
 {
+
+    /**
+     * cleanGraphData Remove frame nodes from the graph data
+     *
+     * @param  array $graphData
+     * @return array
+     */
+    public static function cleanGraphData(array $graphData): array
+    {
+        return array_filter($graphData, function($i) {
+            return $i != '_frames';
+        }, ARRAY_FILTER_USE_KEY);
+    }
+
     /**
      * extractTriggerFromWorkflow Return the trigger id (or full module) that are specified in the workflow
      *
@@ -382,8 +398,9 @@ class WorkflowGraphTool
      */
     public static function extractTriggersFromWorkflow(array $graphData, bool $fullNode = false): array
     {
+        $graphData = self::cleanGraphData($graphData);
         $triggers = [];
-        foreach ($graphData as $node) {
+        foreach ($graphData as $i => $node) {
             if ($node['data']['module_type'] == 'trigger') {
                 if (!empty($fullNode)) {
                     $triggers[] = $node;
@@ -404,8 +421,9 @@ class WorkflowGraphTool
      */
     public static function extractConcurrentTasksFromWorkflow(array $graphData, bool $fullNode = false): array
     {
+        $graphData = self::cleanGraphData($graphData);
         $nodes = [];
-        foreach ($graphData as $node) {
+        foreach ($graphData as $i => $node) {
             if ($node['data']['module_type'] == 'logic' && $node['data']['id'] == 'concurrent-task') {
                 if (!empty($fullNode)) {
                     $nodes[] = $node;
@@ -426,8 +444,9 @@ class WorkflowGraphTool
      */
     public static function extractFilterNodesFromWorkflow(array $graphData, bool $fullNode = false): array
     {
+        $graphData = self::cleanGraphData($graphData);
         $nodes = [];
-        foreach ($graphData as $node) {
+        foreach ($graphData as $i => $node) {
             if ($node['data']['module_type'] == 'logic' && $node['data']['id'] == 'generic-filter-data') {
                 if (!empty($fullNode)) {
                     $nodes[] = $node;
@@ -448,8 +467,9 @@ class WorkflowGraphTool
      */
     public static function extractResetFilterFromWorkflow(array $graphData, bool $fullNode = false): array
     {
+        $graphData = self::cleanGraphData($graphData);
         $nodes = [];
-        foreach ($graphData as $node) {
+        foreach ($graphData as $i => $node) {
             if ($node['data']['module_type'] == 'logic' && $node['data']['id'] == 'generic-filter-reset') {
                 if (!empty($fullNode)) {
                     $nodes[] = $node;

app/Lib/cakephp

@@ -1 +1 @@
-Subproject commit c31bb4b4be00d2a0db22c9a038f9fad8a5950efe
+Subproject commit cbd482740f9b472296e01622a3cebb34edf39623

app/Model/AppModel.php

@@ -85,7 +85,7 @@ class AppModel extends Model
         93 => false, 94 => false, 95 => true, 96 => false, 97 => true, 98 => false,
         99 => false, 100 => false, 101 => false, 102 => false, 103 => false, 104 => false,
         105 => false, 106 => false, 107 => false, 108 => false, 109 => false, 110 => false,
-        111 => false, 112 => false, 113 => true, 114 => false
+        111 => false, 112 => false, 113 => true, 114 => false, 115 => false
     );
 
     const ADVANCED_UPDATES_DESCRIPTION = array(
@@ -1973,6 +1973,10 @@ class AppModel extends Model
             case 114:
                 $indexArray[] = ['object_references', 'uuid'];
                 break;
+            case 115:
+                $sqlArray[] = "ALTER TABLE `users` ADD COLUMN `last_pw_change` BIGINT(20) NULL DEFAULT NULL;";
+                $sqlArray[] = "UPDATE `users` SET last_pw_change=date_modified WHERE last_pw_change IS NULL";
+                break;
             case 'fixNonEmptySharingGroupID':
                 $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
                 $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
@@ -3288,25 +3292,32 @@ class AppModel extends Model
             foreach ($filters as $f) {
                 if ($f === -1) {
                     foreach ($keys as $key) {
-                        $temp['OR'][$key][] = -1;
+                        if ($this->checkParam($key)) {
+                            $temp['OR'][$key][] = -1;
+                        }
                     }
                     continue;
                 }
                 // split the filter params into two lists, one for substring searches one for exact ones
                 if (is_string($f) && ($f[strlen($f) - 1] === '%' || $f[0] === '%')) {
                     foreach ($keys as $key) {
-                        if ($operator === 'NOT') {
-                            $temp[] = array($key . ' NOT LIKE' => $f);
-                        } else {
-                            $temp[] = array($key . ' LIKE' => $f);
+                        if ($this->checkParam($key)) {
+                            if ($operator === 'NOT') {
+                                $temp[] = array($key . ' NOT LIKE' => $f);
+                            } else {
+                                $temp[] = array($key . ' LIKE' => $f);
+                                $temp[] = array($key => $f);
+                            }
                         }
                     }
                 } else {
                     foreach ($keys as $key) {
-                        if ($operator === 'NOT') {
-                            $temp[$key . ' !='][] = $f;
-                        } else {
-                            $temp['OR'][$key][] = $f;
+                        if ($this->checkParam($key)) {
+                            if ($operator === 'NOT') {
+                                $temp[$key . ' !='][] = $f;
+                            } else {
+                                $temp['OR'][$key][] = $f;
+                            }
                         }
                     }
                 }
@@ -3334,10 +3345,12 @@ class AppModel extends Model
             $filter = array();
             foreach ($temp as $f) {
                 $f = strval($f);
-                if ($f[0] === '!') {
-                    $filter['NOT'][] = substr($f, 1);
-                } else {
-                    $filter['OR'][] = $f;
+                if ($f !== '') {
+                    if ($f[0] === '!') {
+                        $filter['NOT'][] = substr($f, 1);
+                    } else {
+                        $filter['OR'][] = $f;
+                    }
                 }
             }
             return $filter;
@@ -3986,4 +3999,37 @@ class AppModel extends Model
         }
         return $_SERVER['REMOTE_ADDR'] ?? null;
     }
+
+    public function find($type = 'first', $query = array()) {
+        if (!empty($query['order']) && $this->validOrderClause($query['order']) === false) {
+            throw new InvalidArgumentException('Invalid order clause');
+        }
+
+        return parent::find($type, $query);
+    }
+
+    private function validOrderClause($order){
+        $pattern = '/^[\w\_\-\.\(\) ]+$/';
+        if(is_string($order) && preg_match($pattern, $order)){
+            return true;
+        }
+
+        if (is_array($order)) {
+            foreach ($order as $key => $value) {
+                if (is_string($key) && is_string($value) && preg_match($pattern, $key) && in_array(strtolower($value), ['asc', 'desc'])) {
+                    return true;
+                }
+                if(is_numeric($key) && is_string($value) && preg_match($pattern, $value)){
+                    return true;
+                }
+            }
+        }
+
+        return false;
+    }
+
+    public function checkParam($param)
+    {
+        return preg_match('/^[\w\_\-\. ]+$/', $param);
+    }
 }

app/Model/AuthKey.php

@@ -55,6 +55,7 @@ class AuthKey extends AppModel
             $this->data['AuthKey']['authkey_raw'] = $authkey;
         }
 
+        $validAllowedIpFound = false;
         if (!empty($this->data['AuthKey']['allowed_ips'])) {
             $allowedIps = &$this->data['AuthKey']['allowed_ips'];
             if (is_string($allowedIps)) {
@@ -70,12 +71,18 @@ class AuthKey extends AppModel
             if (!is_array($allowedIps)) {
                 $this->invalidate('allowed_ips', 'Allowed IPs must be array');
             }
+
             foreach ($allowedIps as $cidr) {
                 if (!CidrTool::validate($cidr)) {
                     $this->invalidate('allowed_ips', "$cidr is not valid IP range");
+                } else {
+                    $validAllowedIpFound = true;
                 }
             }
         }
+        if (!empty(Configure::read('Security.mandate_ip_allowlist_advanced_authkeys')) && $validAllowedIpFound === false){
+            $this->invalidate('allowed_ips', "Setting an ip allowlist is mandatory on this instance.");
+        }
 
         $creationTime = isset($this->data['AuthKey']['created']) ? $this->data['AuthKey']['created'] : time();
         $validity = Configure::read('Security.advanced_authkeys_validity');
@@ -219,9 +226,9 @@ class AuthKey extends AppModel
         $user['authkey_read_only'] = (bool)$authkey['AuthKey']['read_only'];
 
         if ($authkey['AuthKey']['read_only']) {
-            // Disable all permissions, keep just `perm_auth` unchanged
+            // Disable all permissions, keep just `perm_auth` and `perm_audit` unchanged
             foreach ($user['Role'] as $key => &$value) {
-                if (substr($key, 0, 5) === 'perm_' && $key !== 'perm_auth') {
+                if (substr($key, 0, 5) === 'perm_' && $key !== 'perm_auth' && $key !== 'perm_audit') {
                     $value = 0;
                 }
             }

app/Model/Correlation.php

@@ -140,9 +140,14 @@ class Correlation extends AppModel
         $attributeCount = 0;
         if (Configure::read('MISP.background_jobs') && $jobId) {
             $this->Job = ClassRegistry::init('Job');
+            $this->Job->id = $jobId;
+            if (!$this->Job->exists()) {
+                $jobId = false;
+            }
         } else {
             $jobId = false;
         }
+
         if (!empty($eventIds)) {
             $eventCount = count($eventIds);
             foreach ($eventIds as $j => $currentEventId) {

app/Model/Event.php

@@ -126,6 +126,47 @@ class Event extends AppModel
         'yara-json' => array('json', 'YaraExport', 'json')
     );
 
+    public $possibleOptions = array(
+        'eventid',
+        'idList',
+        'tags',
+        'from',
+        'to',
+        'last',
+        'to_ids',
+        'includeAllTags', // include also non exportable tags, default `false`
+        'includeAttachments',
+        'event_uuid',
+        'distribution',
+        'sharing_group_id',
+        'disableSiteAdmin',
+        'metadata',
+        'enforceWarninglist', // return just attributes that contains no warnings
+        'sgReferenceOnly', // do not fetch additional information about sharing groups
+        'flatten',
+        'blockedAttributeTags',
+        'eventsExtendingUuid',
+        'extended',
+        'extensionList',
+        'excludeGalaxy',
+        // 'includeCustomGalaxyCluster', // not used
+        'includeRelatedTags',
+        'excludeLocalTags',
+        'includeDecayScore',
+        'includeScoresOnEvent',
+        'includeSightingdb',
+        'includeFeedCorrelations',
+        'includeServerCorrelations',
+        'includeWarninglistHits',
+        'noEventReports', // do not include event report in event data
+        'noShadowAttributes', // do not fetch proposals,
+        'limit',
+        'page',
+        'order',
+        'protected',
+        'published',
+    );
+
     public $validate = array(
         'org_id' => array(
             'rule' => 'numeric',
@@ -540,7 +581,8 @@ class Event extends AppModel
                 'local' => $local,
                 'relationship_type' => $relationship,
             ];
-            $success = $success || $this->EventTag->attachTagToEvent($event_id, $tag, $nothingToChange);
+            $attachSuccess = $this->EventTag->attachTagToEvent($event_id, $tag, $nothingToChange);
+            $success = $success || $attachSuccess;
             $touchEvent = $touchEvent || !$nothingToChange;
         }
         if ($touchEvent) {
@@ -562,7 +604,8 @@ class Event extends AppModel
                 $success = $success || true;
                 continue;
             }
-            $success = $success || $this->EventTag->detachTagFromEvent($event_id, $tag_id, $local, $nothingToChange);
+            $detachSuccess = $this->EventTag->detachTagFromEvent($event_id, $tag_id, $local, $nothingToChange);
+            $success = $success || $detachSuccess;
             $touchEvent = $touchEvent || !$nothingToChange;
         }
         if ($touchEvent) {
@@ -1682,46 +1725,6 @@ class Event extends AppModel
         if (isset($options['Event.id'])) {
             $options['eventid'] = $options['Event.id'];
         }
-        $possibleOptions = array(
-            'eventid',
-            'idList',
-            'tags',
-            'from',
-            'to',
-            'last',
-            'to_ids',
-            'includeAllTags', // include also non exportable tags, default `false`
-            'includeAttachments',
-            'event_uuid',
-            'distribution',
-            'sharing_group_id',
-            'disableSiteAdmin',
-            'metadata',
-            'enforceWarninglist', // return just attributes that contains no warnings
-            'sgReferenceOnly', // do not fetch additional information about sharing groups
-            'flatten',
-            'blockedAttributeTags',
-            'eventsExtendingUuid',
-            'extended',
-            'extensionList',
-            'excludeGalaxy',
-            // 'includeCustomGalaxyCluster', // not used
-            'includeRelatedTags',
-            'excludeLocalTags',
-            'includeDecayScore',
-            'includeScoresOnEvent',
-            'includeSightingdb',
-            'includeFeedCorrelations',
-            'includeServerCorrelations',
-            'includeWarninglistHits',
-            'noEventReports', // do not include event report in event data
-            'noShadowAttributes', // do not fetch proposals,
-            'limit',
-            'page',
-            'order',
-            'protected',
-            'published',
-        );
         if (!isset($options['excludeLocalTags']) && !empty($user['Role']['perm_sync']) && empty($user['Role']['perm_site_admin'])) {
             $options['excludeLocalTags'] = 1;
         }
@@ -1734,7 +1737,7 @@ class Event extends AppModel
         if (!isset($options['fetchFullClusterRelationship'])) {
             $options['fetchFullClusterRelationship'] = false;
         }
-        foreach ($possibleOptions as $opt) {
+        foreach ($this->possibleOptions as $opt) {
             if (!isset($options[$opt])) {
                 $options[$opt] = false;
             }
@@ -2860,10 +2863,35 @@ class Event extends AppModel
         return $conditions;
     }
 
+    /**
+     * @param string $value
+     * @return string
+     */
+    private static function compressIpv6($value)
+    {
+        if (strpos($value, ':') && $converted = inet_pton($value)) {
+            return inet_ntop($converted);
+        }
+        return $value;
+    }
+
     public function set_filter_value(&$params, $conditions, $options)
     {
         if (!empty($params['value'])) {
             $params[$options['filter']] = $this->convert_filters($params['value']);
+            foreach (['OR', 'AND', 'NOT'] as $operand) {
+                if (!empty($params[$options['filter']][$operand])) {
+                    foreach ($params[$options['filter']][$operand] as $k => $v) {
+                        if ($operand === 'NOT') {
+                            $v = mb_substr($v, 1);
+                        }
+                        if (filter_var($v, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) {
+                            $v = $this->compressIpv6($v);
+                        }
+                        $params[$options['filter']][$operand][$k] = $operand === 'NOT' ? '!' . $v : $v;
+                    }
+                }
+            }
             $conditions = $this->generic_add_filter($conditions, $params['value'], ['Attribute.value1', 'Attribute.value2']);
         }
 
@@ -3546,6 +3574,9 @@ class Event extends AppModel
         if (isset($dataArray['Event'])) {
             $dataArray['response']['Event'] = $dataArray['Event'];
             unset($dataArray['Event']);
+        } elseif (!isset($dataArray['response'])){
+            // Accept an event not containing the `Event` key
+            $dataArray['response']['Event'] = $dataArray;
         }
         if (!isset($dataArray['response']) || !isset($dataArray['response']['Event'])) {
             $exception = $isXml ? __('This is not a valid MISP XML file.') : __('This is not a valid MISP JSON file.');
@@ -4476,7 +4507,7 @@ class Event extends AppModel
             /** @var Job $job */
             $job = ClassRegistry::init('Job');
             $message = empty($sightingUuids) ? __('Publishing sightings.') : __('Publishing %s sightings.', count($sightingUuids));
-            $jobId = $job->createJob($user, Job::WORKER_PRIO, 'publish_event', "Event ID: $id", $message);
+            $jobId = $job->createJob($user, Job::WORKER_DEFAULT, 'publish_event', "Event ID: $id", $message);
 
             $args = ['publish_sightings', $id, $passAlong, $jobId, $user['id']];
             if (!empty($sightingUuids)) {
@@ -4484,7 +4515,7 @@ class Event extends AppModel
             }
 
             return $this->getBackgroundJobsTool()->enqueue(
-                BackgroundJobsTool::PRIO_QUEUE,
+                BackgroundJobsTool::DEFAULT_QUEUE,
                 BackgroundJobsTool::CMD_EVENT,
                 $args,
                 true,

app/Model/EventReport.php

@@ -643,6 +643,9 @@ class EventReport extends AppModel
         foreach ($proxyElements['attribute'] as $uuid => $attribute) {
             $count = 0;
             $textToInject = sprintf('@[attribute](%s)', $uuid);
+            if (strlen($attribute['value']) < 3) {
+                continue;
+            }
             $content = str_replace($attribute['value'], $textToInject, $content, $count);
             if ($count > 0 || strpos($originalContent, $attribute['value'])) { // Check if the value has been replaced by the first match
                 if (!isset($replacedValues[$attribute['value']])) {
@@ -784,6 +787,9 @@ class EventReport extends AppModel
             $tags = $this->Tag->fetchUsableTags($user);
             foreach ($tags as $tag) {
                 $tagName = $tag['Tag']['name'];
+                if (strlen($tagName) < 3) {
+                    continue;
+                }
                 $found = $this->isValidReplacementTag($content, $tagName);
                 if ($found) {
                     $replacedContext[$tagName][$tagName] = $tag['Tag'];

app/Model/Log.php

@@ -44,6 +44,7 @@ class Log extends AppModel
                     'export',
                     'fetchEvent',
                     'file_upload',
+                    'forgot',
                     'galaxy',
                     'include_formula',
                     'load_module',
@@ -51,6 +52,7 @@ class Log extends AppModel
                     'login_fail',
                     'logout',
                     'merge',
+                    'password_reset',
                     'pruneUpdateLogs',
                     'publish',
                     'publish_sightings',

app/Model/MispObject.php

@@ -108,7 +108,8 @@ class MispObject extends AppModel
         ),
         'description' => array(
             'stringNotEmpty' => array(
-                'rule' => array('stringNotEmpty')
+                'rule' => array('stringNotEmpty'),
+                'on' => 'create'
             ),
         ),
         'template_uuid' => array(

app/Model/Server.php

@@ -4135,7 +4135,11 @@ class Server extends AppModel
         $current = implode('.', $version_array);
 
         $upToDate = version_compare($current, substr($newest, 1));
-        if ($upToDate === 0) {
+        if ($newest === null && (Configure::read('MISP.online_version_check') || !Configure::check('MISP.online_version_check'))) {
+            $upToDate = 'error';
+        } elseif ($newest === null && (!Configure::read('MISP.online_version_check') && Configure::check('MISP.online_version_check'))) {
+            $upToDate = 'disabled';
+        } elseif ($upToDate === 0) {
             $upToDate = 'same';
         } else {
             $upToDate = $upToDate === -1 ? 'older' : 'newer';
@@ -4171,11 +4175,15 @@ class Server extends AppModel
      */
     public function getCurrentGitStatus($checkVersion = false)
     {
-        $HttpSocket = $this->setupHttpSocket(null, null, 3);
-        try {
-            $latestCommit = GitTool::getLatestCommit($HttpSocket);
-        } catch (Exception $e) {
-            $latestCommit = false;
+        $latestCommit = false;
+
+        if (Configure::read('MISP.online_version_check') || !Configure::check('MISP.online_version_check')) {
+            $HttpSocket = $this->setupHttpSocket(null, null, 3);
+            try {
+                $latestCommit = GitTool::getLatestCommit($HttpSocket);
+            } catch (Exception $e) {
+                $latestCommit = false;
+            }
         }
 
         $output = [
@@ -4184,7 +4192,7 @@ class Server extends AppModel
             'latestCommit' => $latestCommit,
         ];
         if ($checkVersion) {
-            $output['version'] = $latestCommit ? $this->checkRemoteVersion($HttpSocket) : false;
+            $output['version'] = $latestCommit ? $this->checkRemoteVersion($HttpSocket) : $this->checkVersion(null);
         }
         return $output;
     }
@@ -6114,6 +6122,24 @@ class Server extends AppModel
                     'type' => 'boolean',
                     'null' => true,
                 ],
+                'self_update' => [
+                    'level' => self::SETTING_CRITICAL,
+                    'description' => __('Enable the GUI button for MISP self-update on the Diagnostics page.'),
+                    'value' => true,
+                    'test' => 'testBool',
+                    'type' => 'boolean',
+                    'null' => true,
+                    'cli_only' => true,
+                ],
+                'online_version_check' => [
+                    'level' => self::SETTING_CRITICAL,
+                    'description' => __('Enable the online MISP version check when loading the Diagnostics page.'),
+                    'value' => true,
+                    'test' => 'testBool',
+                    'type' => 'boolean',
+                    'null' => true,
+                    'cli_only' => true,
+                ],
             ),
             'GnuPG' => array(
                 'branch' => 1,
@@ -6373,6 +6399,14 @@ class Server extends AppModel
                     'type' => 'boolean',
                     'null' => true
                 ),
+                'mandate_ip_allowlist_advanced_authkeys' => array(
+                    'level' => 2,
+                    'description' => __('If enabled, setting an ip allowlist will be mandatory when adding or editing an advanced authkey.'),
+                    'value' => false,
+                    'test' => 'testBool',
+                    'type' => 'boolean',
+                    'null' => true
+                ),
                 'disable_browser_cache' => array(
                     'level' => 0,
                     'description' => __('If enabled, HTTP headers that block browser cache will be send. Static files (like images or JavaScripts) will still be cached, but not generated pages.'),
@@ -6465,6 +6499,14 @@ class Server extends AppModel
                     'type' => 'boolean',
                     'null' => true
                 ),
+                'allow_password_forgotten' => array(
+                    'level' => 1,
+                    'description' => __('Enabling this setting will allow users to request automated password reset tokens via mail and initiate a reset themselves. Users with no encryption keys will not be able to use this feature.'),
+                    'value' => false,
+                    'test' => 'testBool',
+                    'type' => 'boolean',
+                    'null' => true
+                ),
                 'self_registration_message' => array(
                     'level' => 1,
                     'bigField' => true,
@@ -7251,6 +7293,13 @@ class Server extends AppModel
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
+                'Sightings_enable_realtime_publish' => array(
+                    'level' => 1,
+                    'description' => __('By default, sightings will not be immediately pushed to connected instances, as this can have a heavy impact on the performance of sighting attributes. Enable realtime publishing to trigger the publishing of sightings immediately as they are added.'),
+                    'value' => false,
+                    'test' => 'testBool',
+                    'type' => 'boolean'
+                ),
                 'CustomAuth_enable' => array(
                     'level' => 2,
                     'description' => __('Enable this functionality if you would like to handle the authentication via an external tool and authenticate with MISP using a custom header.'),
@@ -7263,7 +7312,7 @@ class Server extends AppModel
                 'CustomAuth_header' => array(
                     'level' => 2,
                     'description' => __('Set the header that MISP should look for here. If left empty it will default to the Authorization header.'),
-                    'value' => 'Authorization',
+                    'value' => 'AUTHORIZATION',
                     'test' => 'testForEmpty',
                     'type' => 'string',
                     'null' => true

app/Model/SystemSetting.php

@@ -21,6 +21,8 @@ class SystemSetting extends AppModel
         'MISP.tmpdir',
         'MISP.system_setting_db',
         'MISP.attachments_dir',
+        'MISP.self_update',
+        'MISP.online_version_check',
     ];
 
     // Allow to set config values just for these categories

app/Model/TaxiiServer.php

@@ -4,6 +4,8 @@ App::uses('EncryptedValue', 'Tools');
 App::uses('Folder', 'Utility');
 App::uses('File', 'Utility');
 App::uses('RandomTool', 'Tools');
+App::uses('JSONConverterTool', 'Tools');
+App::uses('JsonTool', 'Tools');
 
 class TaxiiServer extends AppModel
 {
@@ -102,7 +104,11 @@ class TaxiiServer extends AppModel
         $this->Job->id = $jobId;
         foreach ($result as $event) {
             $temporaryFile = $this->temporaryFile($temporaryFolderPath);
-            $temporaryFile->write(json_encode($event));
+            $temporaryFile->write(
+                JsonTool::encode(
+                    JSONConverterTool::convert($event, false, true)
+                )
+            );
             $temporaryFile->close();
             if ($jobId && $i % 10 == 0) {
                 $this->Job->saveField('progress', intval((100 * $i) / $eventCount));

app/Model/Taxonomy.php

@@ -179,9 +179,10 @@ class Taxonomy extends AppModel
 
     /**
      * @param int|string $id Taxonomy ID or namespace
+     * @param string|boolean $filter String to filter to apply to the tags
      * @return array|false
      */
-    private function __getTaxonomy($id)
+    private function __getTaxonomy($id, $filter = false)
     {
         if (!is_numeric($id)) {
             $conditions = ['Taxonomy.namespace' => trim(mb_strtolower($id))];
@@ -215,7 +216,9 @@ class Taxonomy extends AppModel
                     if (isset($entry['numerical_value'])) {
                         $temp['numerical_value'] = $entry['numerical_value'];
                     }
-                    $entries[] = $temp;
+                    if (empty($filter) || mb_strpos(mb_strtolower($temp['tag']), mb_strtolower($filter)) !== false) {
+                        $entries[] = $temp;
+                    }
                 }
             } else {
                 $temp = [
@@ -231,7 +234,9 @@ class Taxonomy extends AppModel
                 if (isset($predicate['numerical_value'])) {
                     $temp['numerical_value'] = $predicate['numerical_value'];
                 }
-                $entries[] = $temp;
+                if (empty($filter) || mb_strpos(mb_strtolower($temp['tag']), mb_strtolower($filter)) !== false) {
+                    $entries[] = $temp;
+                }
             }
         }
         $taxonomy = [
@@ -349,11 +354,12 @@ class Taxonomy extends AppModel
     /**
      * @param int|string $id Taxonomy ID or namespace
      * @param bool $full Add tag information to entries
+     * @param string|boolean $filter String filter to apply to the tag names
      * @return array|false
      */
-    public function getTaxonomy($id, $full = true)
+    public function getTaxonomy($id, $full = true, $filter = false)
     {
-        $taxonomy = $this->__getTaxonomy($id);
+        $taxonomy = $this->__getTaxonomy($id, $filter);
         if (empty($taxonomy)) {
             return false;
         }

app/Model/User.php

@@ -845,6 +845,10 @@ class User extends AppModel
      */
     public function sendEmail(array $user, $body, $bodyNoEnc = false, $subject, $replyToUser = false)
     {
+        if (Configure::read('MISP.disable_emailing')) {
+            return true;
+        }
+
         if ($user['User']['disabled'] || !$this->checkIfUserIsValid($user['User'])) {
             return true;
         }
@@ -980,6 +984,7 @@ class User extends AppModel
         if ($result) {
             $this->id = $user['User']['id'];
             $this->saveField('password', $password);
+            $this->saveField('last_pw_change', time());
             $this->updateField($user['User'], 'change_pw', 1);
             if ($simpleReturn) {
                 return true;
@@ -1229,6 +1234,15 @@ class User extends AppModel
 
     public function extralog($user, $action = null, $description = null, $fieldsResult = null, $modifiedUser = null)
     {
+        if (!is_array($user) && $user === 'SYSTEM') {
+            $user = [
+                'id' => 0,
+                'email' => 'SYSTEM',
+                'Organisation' => [
+                    'name' => 'SYSTEM'
+                ]
+            ];
+        }
         // new data
         $model = 'User';
         $modelId = $user['id'];
@@ -1707,10 +1721,10 @@ class User extends AppModel
      * @param string $period
      * @return array
      */
-    private function getUsablePeriodicSettingForUser(array $period_filters, $period='daily'): array
+    private function getUsablePeriodicSettingForUser(array $period_filters, $period='daily', $lastdays=7): array
     {
         $filters = [
-            'last' => $this->__genTimerangeFilter($period),
+            'last' => $this->__genTimerangeFilter($period, $lastdays),
             'published' => true,
         ];
         if (!empty($period_filters['orgc_id'])) {
@@ -1803,11 +1817,12 @@ class User extends AppModel
      * @throws InvalidArgumentException
      * @throws JsonException
      */
-    public function generatePeriodicSummary(int $userId, string $period, $rendered = true)
+    public function generatePeriodicSummary(int $userId, string $period, $rendered = true, $lastdays=7)
     {
         $allowedPeriods = array_map(function($period) {
             return substr($period, strlen('notification_'));
         }, self::PERIODIC_NOTIFICATIONS);
+        $allowedPeriods[] = 'custom';
         if (!in_array($period, $allowedPeriods, true)) {
             throw new InvalidArgumentException(__('Invalid period. Must be one of %s', JsonTool::encode(self::PERIODIC_NOTIFICATIONS)));
         }
@@ -1815,7 +1830,7 @@ class User extends AppModel
         $user = $this->getAuthUser($userId);
         App::import('Tools', 'SendEmail');
         $periodicSettings = $this->fetchPeriodicSettingForUser($userId, true);
-        $filters = $this->getUsablePeriodicSettingForUser($periodicSettings, $period);
+        $filters = $this->getUsablePeriodicSettingForUser($periodicSettings, $period, $lastdays);
         $filtersForRestSearch = $filters; // filters for restSearch are slightly different than fetchEvent
         $filters['last'] = $this->resolveTimeDelta($filters['last']);
         $filters['sgReferenceOnly'] = true;
@@ -1845,7 +1860,7 @@ class User extends AppModel
         $finalContext = JsonTool::decode($finalContext->intoString());
         $aggregated_context = $this->__renderAggregatedContext($finalContext);
         $rollingWindows = $periodicSettings['trending_period_amount'] ?: 2;
-        $trendAnalysis = $this->Event->getTrendsForTagsFromEvents($events, $this->periodToDays($period), $rollingWindows, $periodicSettings['trending_for_tags']);
+        $trendAnalysis = $this->Event->getTrendsForTagsFromEvents($events, $this->periodToDays($period, $lastdays), $rollingWindows, $periodicSettings['trending_for_tags']);
         $tagFilterPrefixes = $periodicSettings['trending_for_tags'] ?: array_keys($trendAnalysis['all_tags']);
         $trendData = [
             'trendAnalysis' => $trendAnalysis,
@@ -1857,12 +1872,13 @@ class User extends AppModel
         ];
         $security_recommendations = $this->__renderSecurityRecommendations($securityRecommendationsData);
 
-        $emailTemplate = $this->prepareEmailTemplate($period);
+        $templateName = $period == 'custom' ? 'daily' : $period;
+        $emailTemplate = $this->prepareEmailTemplate($templateName);
         $emailTemplate->set('baseurl', $this->Event->__getAnnounceBaseurl());
         $emailTemplate->set('events', $events);
         $emailTemplate->set('filters', $filters);
         $emailTemplate->set('periodicSettings', $periodicSettings);
-        $emailTemplate->set('period_days', $this->periodToDays($period));
+        $emailTemplate->set('period_days', $this->periodToDays($period, $lastdays));
         $emailTemplate->set('period', $period);
         $emailTemplate->set('aggregated_context', $aggregated_context);
         $emailTemplate->set('trending_summary', $trending_summary);
@@ -1928,13 +1944,19 @@ class User extends AppModel
         }
         return $filters;
     }
-    private function __genTimerangeFilter(string $period='daily'): string
+    private function __genTimerangeFilter(string $period='daily', $lastdays = 7): string
     {
+        if ($period == 'custom') {
+            return strval($lastdays) . 'd';
+        }
         return $this->periodToDays($period) . 'd';
     }
 
-    private function periodToDays(string $period='daily'): int
+    private function periodToDays(string $period='daily', $lastdays = false): int
     {
+        if ($lastdays !== false) {
+            return $lastdays;
+        }
         if ($period === 'daily') {
             return 1;
         } else if ($period === 'weekly') {
@@ -2007,4 +2029,89 @@ class User extends AppModel
         }
         return false;
     }
+
+    public function forgotRouter($email, $ip)
+    {
+        if (Configure::read('MISP.background_jobs')) {
+            /** @var Job $job */
+            $job = ClassRegistry::init('Job');
+            $dummyUser = [
+                'email' => 'SYSTEM',
+                'org_id' => 0,
+                'role_id' => 0
+            ];
+            $jobId = $job->createJob($dummyUser, Job::WORKER_EMAIL, 'forgot_password', $email, 'Sending...');
+
+            $args = [
+                'jobForgot',
+                $email,
+                $ip,
+                $jobId,
+            ];
+
+            $this->getBackgroundJobsTool()->enqueue(
+                BackgroundJobsTool::EMAIL_QUEUE,
+                BackgroundJobsTool::CMD_ADMIN,
+                $args,
+                true,
+                $jobId
+            );
+
+            return true;
+        } else {
+            return $this->forgot($email);
+        }
+    }
+
+    public function forgot($email, $ip, $jobId = null)
+    {
+        $user = $this->find('first', [
+            'recursive' => -1,
+            'conditions' => [
+                'User.email' => $email,
+                'User.disabled' => 0
+            ]
+        ]);
+        if (empty($user)) {
+            return false;
+        }
+        $redis = $this->setupRedis();
+        $token = RandomTool::random_str(true, 40, '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ');
+        $redis->set('misp:forgot:' . $token, $user['User']['id'], ['nx', 'ex' => 600]);
+        $baseurl = Configure::check('MISP.external_baseurl') ? Configure::read('MISP.external_baseurl') : Configure::read('MISP.baseurl');
+        $body = __(
+            "Dear MISP user,\n\nyou have requested a password reset on the MISP instance at %s. Click the link below to change your password.\n\n%s\n\nThe link above is only valid for 10 minutes, feel free to request a new one if it has expired.\n\nIf you haven't requested a password reset, reach out to your admin team and let them know that someone has attempted it in your stead.\n\nMake sure you keep the contents of this e-mail confidential, do NOT ever forward it as it contains a reset token that is equivalent of a password if acted upon. The IP used to trigger the request was: %s\n\nBest regards,\nYour MISP admin team",
+            $baseurl,
+            $baseurl . '/users/password_reset/' . $token,
+            $ip
+        );
+        $bodyNoEnc = __(
+            "Dear MISP user,\n\nyou have requested a password reset on the MISP instance at %s, however, no valid encryption key was found for your user and thus we cannot deliver your reset token. Please get in touch with your org admin / with an instance site admin to ask for a reset.\n\nThe IP used to trigger the request was: %s\n\nBest regards,\nYour MISP admin team",
+            $baseurl,
+            $ip
+        );
+        $this->sendEmail($user, $body, $bodyNoEnc, __('MISP password reset'));
+        return true;
+    }
+
+    public function fetchForgottenPasswordUser($token)
+    {
+        if (!ctype_alnum($token)) {
+            return false;
+        }
+        $redis = $this->setupRedis();
+        $userId = $redis->get('misp:forgot:' . $token);
+        if (empty($userId)) {
+            return false;
+        }
+        $user = $this->getAuthUser($userId, true);
+        return $user;
+    }
+
+    public function purgeForgetToken($token)
+    {
+        $redis = $this->setupRedis();
+        $userId = $redis->del('misp:forgot:' . $token);
+        return true;
+    }
 }

app/Model/Workflow.php

@@ -13,7 +13,7 @@ class Workflow extends AppModel
     public $recursive = -1;
 
     public $actsAs = [
-        'AuditLog',
+        // 'AuditLog',
         'Containable',
         'SysLogLogable.SysLogLogable' => [
             'roleModel' => 'Role',
@@ -518,15 +518,14 @@ class Workflow extends AppModel
     {
         $this->Log = ClassRegistry::init('Log');
         $message =  __('Started executing workflow for trigger `%s` (%s)', $triggerModule->id, $workflow['Workflow']['id']);
-        $this->Log->createLogEntry('SYSTEM', 'execute_workflow', 'Workflow', $workflow['Workflow']['id'], $message);
-        $this->__logToFile($workflow, $message);
+        $this->logExecutionIfDebug($workflow, $message);
         $workflow = $this->__incrementWorkflowExecutionCount($workflow);
         $walkResult = [];
         $debugData = ['original' => $data];
         $data = $this->__normalizeDataForTrigger($triggerModule, $data);
         $debugData['normalized'] = $data;
         $for_path = !empty($triggerModule->blocking) ? GraphWalker::PATH_TYPE_BLOCKING : GraphWalker::PATH_TYPE_NON_BLOCKING;
-        $this->sendRequestToDebugEndpoint($workflow, [], '/init?type=' . $for_path, $debugData);
+        $this->sendRequestToDebugEndpointIfDebug($workflow, [], '/init?type=' . $for_path, $debugData);
 
         $blockingPathExecutionSuccess = $this->walkGraph($workflow, $startNodeID, $for_path, $data, $blockingErrors, $walkResult);
         $executionStoppedByStopModule = in_array('stop-execution', Hash::extract($walkResult, 'blocking_nodes.{n}.data.id'));
@@ -542,9 +541,8 @@ class Workflow extends AppModel
         }
         $message =  __('Finished executing workflow for trigger `%s` (%s). Outcome: %s', $triggerModule->id, $workflow['Workflow']['id'], $outcomeText);
 
-        $this->Log->createLogEntry('SYSTEM', 'execute_workflow', 'Workflow', $workflow['Workflow']['id'], $message);
-        $this->__logToFile($workflow, $message);
-        $this->sendRequestToDebugEndpoint($workflow, [], '/end?outcome=' . $outcomeText, $walkResult);
+        $this->logExecutionIfDebug($workflow, $message);
+        $this->sendRequestToDebugEndpointIfDebug($workflow, [], '/end?outcome=' . $outcomeText, $walkResult);
         return [
             'outcomeText' => $outcomeText,
             'walkResult' => $walkResult,
@@ -661,7 +659,7 @@ class Workflow extends AppModel
             $message = __('Could not execute disabled module `%s`.', $node['data']['id']);
             $this->logExecutionError($roamingData->getWorkflow(), $message);
             $errors[] = $message;
-            $this->sendRequestToDebugEndpoint($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s', $moduleClass->id, 'disabled_module'), $roamingData->getData());
+            $this->sendRequestToDebugEndpointIfDebug($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s', $moduleClass->id, 'disabled_module'), $roamingData->getData());
             return false;
         }
         if (!is_null($moduleClass)) {
@@ -671,17 +669,25 @@ class Workflow extends AppModel
                 $message = __('Error while executing module %s. Error: %s', $node['data']['id'], $e->getMessage());
                 $this->logExecutionError($roamingData->getWorkflow(), $message);
                 $errors[] = $message;
-                $this->sendRequestToDebugEndpoint($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s&message=%s', $moduleClass->id, 'error', $e->getMessage()), $roamingData->getData());
+                $this->sendRequestToDebugEndpointIfDebug($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s&message=%s', $moduleClass->id, 'error', $e->getMessage()), $roamingData->getData());
                 return false;
             }
         } else {
             $message = sprintf(__('Could not load class for module: %s'), $node['data']['id']);
             $this->logExecutionError($roamingData->getWorkflow(), $message);
             $errors[] = $message;
-            $this->sendRequestToDebugEndpoint($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s', $node['data']['id'], 'loading_error'), $roamingData->getData());
+            $this->sendRequestToDebugEndpointIfDebug($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s', $node['data']['id'], 'loading_error'), $roamingData->getData());
             return false;
         }
-        $this->sendRequestToDebugEndpoint($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s', $moduleClass->id, 'success'), $roamingData->getData());
+        $message = __('Executed node `%s`' .  PHP_EOL . 'Node `%s` (%s) from Workflow `%s` (%s) executed successfully',
+            $node['data']['id'],
+            $node['data']['id'],
+            $node['id'],
+            $roamingData->getWorkflow()['Workflow']['name'],
+            $roamingData->getWorkflow()['Workflow']['id']
+        );
+        $this->logExecutionIfDebug($roamingData->getWorkflow(), $message);
+        $this->sendRequestToDebugEndpointIfDebug($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s', $moduleClass->id, 'success'), $roamingData->getData());
         return $success;
     }
 
@@ -993,6 +999,14 @@ class Workflow extends AppModel
         $this->__logToFile($workflow, $message);
     }
 
+    public function logExecutionIfDebug(array $workflow, $message): void
+    {
+        if ($workflow['Workflow']['debug_enabled']) {
+            $this->Log->createLogEntry('SYSTEM', 'execute_workflow', 'Workflow', $workflow['Workflow']['id'], $message);
+            $this->__logToFile($workflow, $message);
+        }
+    }
+
     /**
      * __logToFile Log to file
      *
@@ -1321,19 +1335,32 @@ class Workflow extends AppModel
             $node = $graphNode['node'];
             $nodeID = $node['id'];
             $parsedPathList = GraphWalker::parsePathList($graphNode['path_list']);
+            if (!empty($parsedPathList)) {
+                $lastNodeInPath = $parsedPathList[count($parsedPathList)-1];
+                $previousNodeId = $lastNodeInPath['source_id'];
+                $connections[$nodeID][$previousNodeId] = [];
+            }
             foreach ($parsedPathList as $pathEntry) {
                 if (!empty($filterNodeIDToLabel[$pathEntry['source_id']])) {
-                    $connections[$nodeID][] = $filterNodeIDToLabel[$pathEntry['source_id']];
+                    $connections[$nodeID][$previousNodeId][] = $filterNodeIDToLabel[$pathEntry['source_id']];
                 }
                 if (!empty($resetFilterNodeIDToLabel[$pathEntry['source_id']])) {
                     if ($resetFilterNodeIDToLabel[$pathEntry['source_id']] == 'all') {
-                        $connections[$nodeID] = [];
+                        $connections[$nodeID][$previousNodeId] = [];
                     } else {
-                        $connections[$nodeID] = array_values(array_diff($connections[$nodeID], [$resetFilterNodeIDToLabel[$pathEntry['source_id']]]));
+                        $connections[$nodeID][$previousNodeId] = array_values(array_diff($connections[$nodeID][$previousNodeId], [$resetFilterNodeIDToLabel[$pathEntry['source_id']]]));
                     }
                 }
             }
         }
+        $connections = array_filter($connections, function($connection) {
+            foreach ($connection as $labels) {
+                if (!empty($labels)) {
+                    return true;
+                }
+            }
+            return false;
+        });
         return $connections;
     }
 
@@ -1346,6 +1373,9 @@ class Workflow extends AppModel
         }
         $labelsByNodes = $this->getLabelsForConnections($workflow, $trigger_id);
         foreach ($graphData as $i => $node) {
+            if ($i == '_frames') {
+                continue;
+            }
             if (!empty($labelsByNodes[$node['id']])) {
                 foreach ($node['inputs'] as $inputName => $inputs) {
                     foreach ($inputs['connections'] as $j => $connection) {
@@ -1355,7 +1385,7 @@ class Workflow extends AppModel
                                 'name' => $label,
                                 'variant' => 'info',
                             ];
-                        }, $labelsByNodes[$node['id']]);
+                        }, $labelsByNodes[$node['id']][$connection['node']]);
                     }
                 }
             }
@@ -1408,6 +1438,7 @@ class Workflow extends AppModel
                 'indexed_params' => $indexed_params,
                 'saved_filters' => $module_config['saved_filters'],
                 'module_data' => $module_config,
+                'expect_misp_core_format' => $module_config['expect_misp_core_format'],
             ],
             'inputs' => [],
             'outputs' => [],
@@ -1464,12 +1495,16 @@ class Workflow extends AppModel
         return $saveSuccess;
     }
 
+    public function sendRequestToDebugEndpointIfDebug(array $workflow, array $node, $path='/', array $data=[])
+    {
+        if ($workflow['Workflow']['debug_enabled']) {
+            $this->sendRequestToDebugEndpoint($workflow, $node, $path, $data);
+        }
+    }
+
     public function sendRequestToDebugEndpoint(array $workflow, array $node, $path='/', array $data=[])
     {
         $debug_url = Configure::read('Plugin.Workflow_debug_url');
-        if (empty($workflow['Workflow']['debug_enabled'])) {
-            return;
-        }
         App::uses('HttpSocket', 'Network/Http');
         $socket = new HttpSocket([
             'timeout' => 5

app/Model/WorkflowModules/WorkflowBaseModule.php

@@ -23,7 +23,7 @@ class WorkflowBaseModule
     ];
     public $params = [];
 
-    private $Event;
+    private $Workflow;
 
     /** @var PubSubTool */
     private static $loadedPubSubTool;
@@ -32,6 +32,16 @@ class WorkflowBaseModule
     {
     }
 
+    public function debug(array $node, WorkflowRoamingData $roamingData, array $data=[]): void
+    {
+        if (!isset($this->Workflow)) {
+            $this->Workflow = ClassRegistry::init('Workflow');
+        }
+        $workflow = $roamingData->getWorkflow();
+        $path = sprintf('/debug/%s', $node['data']['id'] ?? '');
+        $this->Workflow->sendRequestToDebugEndpoint($workflow, $node, $path, $data);
+    }
+
     protected function mergeNodeConfigIntoParameters($node): array
     {
         $fullIndexedParams = [];
@@ -213,11 +223,13 @@ class WorkflowBaseModule
     public function getItemsMatchingCondition($items, $value, $operator, $path)
     {
         foreach ($items as $i => $item) {
-            $subItem = $this->extractData($item, $path, $operator);
+            $subItem = $this->extractData($item, $path);
             if (in_array($operator, ['equals', 'not_equals'])) {
                 $subItem = !empty($subItem) ? $subItem[0] : $subItem;
             }
-            if (!$this->evaluateCondition($subItem, $operator, $value)) {
+            if ($operator == 'any_value' && !empty($subItem)) {
+                continue;
+            } else if (!$this->evaluateCondition($subItem, $operator, $value)) {
                 unset($items[$i]);
             }
         }
@@ -296,6 +308,49 @@ class WorkflowBaseLogicModule extends WorkflowBaseModule
 
 class WorkflowBaseActionModule extends WorkflowBaseModule
 {
+    protected $fastLookupArrayMispFormat = [];
+    protected $fastLookupArrayFlattened = [];
+
+    public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
+    {
+        $rData = $roamingData->getData();
+        $this->_buildFastLookupForRoamingData($rData);
+        return true;
+    }
+
+    protected function _buildFastLookupForRoamingData($rData): void
+    {
+        if (!empty($rData['Event']['Attribute'])) {
+            foreach ($rData['Event']['Attribute'] as $i => $attribute) {
+                $this->fastLookupArrayMispFormat[$attribute['id']] = $i;
+            }
+        }
+        if (!empty($rData['Event']['Object'])) {
+            foreach ($rData['Event']['Object'] as $j => $object) {
+                foreach ($object['Attribute'] as $i => $attribute) {
+                    $this->fastLookupArrayMispFormat[$attribute['id']] = [$j, $i];
+                }
+            }
+        }
+        foreach ($rData['Event']['_AttributeFlattened'] as $i => $attribute) {
+            $this->fastLookupArrayFlattened[$attribute['id']] = $i;
+        }
+    }
+
+    protected function _overrideAttribute(array $oldAttribute, array $newAttribute, array $rData): array
+    {
+        $attributeID = $oldAttribute['id'];
+        $rData['Event']['_AttributeFlattened'][$this->fastLookupArrayFlattened[$attributeID]] = $newAttribute;
+        if (is_array($this->fastLookupArrayMispFormat[$attributeID])) {
+            $objectID = $this->fastLookupArrayMispFormat[$attributeID][0];
+            $attributeID = $this->fastLookupArrayMispFormat[$attributeID][1];
+            $rData['Event']['Object'][$objectID]['Attribute'][$attributeID] = $newAttribute;
+        } else {
+            $attributeID = $this->fastLookupArrayMispFormat[$attributeID];
+            $rData['Event']['Attribute'][$attributeID] = $newAttribute;
+        }
+        return $rData;
+    }
 }
 
 class WorkflowFilteringLogicModule extends WorkflowBaseLogicModule

app/Model/WorkflowModules/action/Module_assign_country_from_enrichment.php

@@ -0,0 +1,152 @@
+<?php
+include_once APP . 'Model/WorkflowModules/action/Module_tag_operation.php';
+
+class Module_assign_country_from_enrichment extends Module_tag_operation
+{
+    public $version = '0.2';
+    public $blocking = false;
+    public $id = 'assign_country';
+    public $name = 'Assign country';
+    public $description = 'Add or remove country Galaxy Cluster based on provided data';
+    public $icon = 'globe';
+    public $inputs = 1;
+    public $outputs = 1;
+    public $support_filters = true;
+    public $expect_misp_core_format = true;
+    public $params = [];
+
+    private $Galaxy;
+    private $countryClusters;
+
+
+    public function __construct()
+    {
+        parent::__construct();
+        $this->params = [
+            [
+                'id' => 'scope',
+                'label' => __('Scope'),
+                'type' => 'select',
+                'options' => [
+                    'event' => __('Event'),
+                    'attribute' => __('Attributes'),
+                ],
+                'default' => 'event',
+            ],
+            [
+                'id' => 'hash_path',
+                'label' => 'Country Hash path',
+                'type' => 'hashpath',
+                'placeholder' => 'enrichment.{n}.{n}.values.0',
+                'default' => 'enrichment.{n}.{n}.values.0'
+            ],
+            [
+                'id' => 'locality',
+                'label' => __('Tag Locality'),
+                'type' => 'select',
+                'options' => [
+                    'local' => __('Local'),
+                    'global' => __('Global'),
+                ],
+                'default' => 'local',
+            ],
+            [
+                'id' => 'galaxy_name',
+                'label' => __('Galaxy Name'),
+                'type' => 'select',
+                'options' => [
+                    'country' => 'country',
+                ],
+                'placeholder' => __('Pick a galaxy name'),
+            ],
+            [
+                'id' => 'relationship_type',
+                'label' => __('Relationship Type'),
+                'type' => 'input',
+                'display_on' => [
+                    'action' => 'add',
+                ],
+            ],
+        ];
+
+        $this->Galaxy = ClassRegistry::init('Galaxy');
+        $this->countryClusters = $this->_fetchCountryGalaxyClusters();
+    }
+
+    public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
+    {
+        $params = $this->getParamsWithValues($node);
+
+        $rData = $roamingData->getData();
+        $user = $roamingData->getUser();
+
+        $countryExtractionPath = $params['hash_path']['value'];
+        if ($this->filtersEnabled($node)) {
+            $filters = $this->getFilters($node);
+            $extracted = $this->extractData($rData, $filters['selector']);
+            if ($extracted === false) {
+                return false;
+            }
+            $matchingItems = $this->getItemsMatchingCondition($extracted, $filters['value'], $filters['operator'], $filters['path']);
+        } else {
+            $matchingItems = $rData;
+        }
+        $matchingAttributes = Hash::extract($matchingItems, 'Event._AttributeFlattened.{n}');
+        if ($params['scope']['value'] == 'event') {
+            if (substr($countryExtractionPath, 0, 4) !== '{n}.') {
+                $countryExtractionPath = '{n}.' . $countryExtractionPath;
+            }
+        }
+
+        $result = false;
+        if ($params['scope']['value'] == 'event') {
+            $extractedCountries = array_unique(Hash::extract($matchingAttributes, $countryExtractionPath));
+            $guessedCountryTags = $this->guessTagFromPath($extractedCountries);
+            $options = [
+                'tags' => $guessedCountryTags,
+                'local' => ($params['locality']['value'] == 'local' ? true : false),
+                'relationship_type' => $params['relationship_type']['value'],
+            ];
+            $result = $this->__addTagsToEvent($matchingItems, $options, $user);
+        } else {
+            foreach ($matchingAttributes as $attribute) {
+                $extractedCountries = Hash::extract($attribute, $countryExtractionPath);
+                $guessedCountryTags = $this->guessTagFromPath($extractedCountries);
+                $options = [
+                    'tags' => $guessedCountryTags,
+                    'local' => ($params['locality']['value'] == 'local' ? true : false),
+                    'relationship_type' => $params['relationship_type']['value'],
+                ];
+                $result = $this->__addTagsToAttributes([$attribute], $options, $user);
+            }
+        }
+        return $result;
+    }
+
+    protected function _fetchCountryGalaxyClusters(): array
+    {
+        $clusters = $this->Galaxy->find('first', [
+            'recursive' => -1,
+            'conditions' => [
+                'name' => 'Country',
+            ],
+            'contain' => [
+                'GalaxyCluster' => ['fields' => ['id', 'uuid', 'value', 'tag_name']],
+            ],
+        ]);
+        return $clusters['GalaxyCluster'];
+    }
+
+    protected function guessTagFromPath($countries)
+    {
+        $matchingTags = [];
+        foreach ($countries as $country) {
+            foreach ($this->countryClusters as $countryCluster) {
+                if (strtolower($countryCluster['value']) == strtolower($country)) {
+                    $matchingTags[] = $countryCluster['tag_name'];
+                }
+            }
+        }
+        return $matchingTags;
+    }
+}

app/Model/WorkflowModules/action/Module_attach_enrichment.php

@@ -5,6 +5,7 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
 {
     public $id = 'attach-enrichment';
     public $name = 'Attach enrichment';
+    public $version = '0.3';
     public $description = 'Attach selected enrichment result to Attributes.';
     public $icon = 'asterisk';
     public $inputs = 1;
@@ -14,8 +15,7 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
     public $params = [];
 
     private $Module;
-    private $fastLookupArrayMispFormat = [];
-    private $fastLookupArrayFlattened = [];
+    private $allModulesByName = [];
 
 
     public function __construct()
@@ -23,19 +23,30 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
         parent::__construct();
         $this->Module = ClassRegistry::init('Module');
         $modules = $this->Module->getModules('Enrichment');
-        $moduleOptions = [];
         if (is_array($modules)) {
+            $this->allModulesByName = Hash::combine($modules, '{n}.name', '{n}');
+        }
+        $moduleOptions = [];
+        $pickerOptions = [];
+        $enrichmentAvailable = false;
+        if (!empty($modules) && is_array($modules)) {
+            $enrichmentAvailable = true;
             $moduleOptions = array_merge([''], Hash::combine($modules, '{n}.name', '{n}.name'));
         } else {
-            $moduleOptions[] = $modules;
+            $pickerOptions = [
+                'placeholder_text_multiple' => __('No enrichment module available'),
+            ];
         }
         sort($moduleOptions);
         $this->params = [
             [
                 'id' => 'modules',
                 'label' => 'Modules',
-                'type' => 'select',
+                'type' => 'picker',
+                'multiple' => true,
+                'disabled' => !$enrichmentAvailable,
                 'options' => $moduleOptions,
+                'picker_options' => $pickerOptions,
             ],
         ];
     }
@@ -47,14 +58,17 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
         if (empty($params['modules']['value'])) {
             $errors[] = __('No enrichmnent module selected');
             return false;
+        } else if (is_string($params['modules']['value'])) {
+            $params['modules']['value'] = [$params['modules']['value']];
         }
+        $selectedModules = array_filter($params['modules']['value'], function($module) {
+            return $module !== '';
+        });
         $rData = $roamingData->getData();
         $event_id = $rData['Event']['id'];
         $options = [
             'user' => $roamingData->getUser(),
             'event_id' => $event_id,
-            'module' => $params['modules']['value'],
-            'config' => ['_' => '_'], // avoid casting empty associative array in to empty list
         ];
 
         $matchingItems = $this->getMatchingItemsForAttributes($node, $rData);
@@ -64,10 +78,22 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
         $this->_buildFastLookupForRoamingData($rData);
 
         foreach ($matchingItems as $attribute) {
-            $moduleData = $options;
-            $moduleData['attribute'] = $attribute;
-            $queryResult = $this->_queryModules($moduleData, $attribute, $rData);
-            $rData = $this->_attachEnrichmentData($attribute, $queryResult, $rData);
+            foreach ($selectedModules as $selectedModule) {
+                $moduleConfig = $this->allModulesByName[$selectedModule];
+                $moduleData = $options;
+                $moduleData['config'] = $this->getModuleOptions($moduleConfig);
+                $moduleData['module'] = $selectedModule;
+                $moduleData['attribute'] = $attribute;
+                if (!$this->_checkIfInputSupported($attribute, $moduleConfig)) { // Queried module doesn't support the Attribute's type
+                    continue;
+                }
+                if (empty($moduleConfig['mispattributes']['format'])) { // Adapt payload if modules doesn't support the misp-format
+                    $moduleData = $this->_convertPayloadToModuleFormat($moduleData, $moduleConfig);
+                }
+                $queryResult = $this->_queryModules($moduleData, $attribute, $rData);
+                $queryResult = $this->_handleModuleResult($queryResult, $moduleConfig);
+                $rData = $this->_attachEnrichmentData($attribute, $queryResult, $rData);
+            }
         }
         $roamingData->setData($rData);
         return true;
@@ -82,19 +108,31 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
         return $result;
     }
 
-    protected function _buildFastLookupForRoamingData($rData): void
+    protected function _convertPayloadToModuleFormat(array $options, array $moduleConfig): array
     {
-        foreach ($rData['Event']['Attribute'] as $i => $attribute) {
-            $this->fastLookupArrayMispFormat[$attribute['id']] = $i;
-        }
-        foreach ($rData['Event']['Object'] as $j => $object) {
-            foreach ($object['Attribute'] as $i => $attribute) {
-                $this->fastLookupArrayMispFormat[$attribute['id']] = [$j, $i];
+        $attribute = $options['attribute'];
+        unset($options['attribute']);
+        foreach ($moduleConfig['mispattributes']['input'] as $supportedAttributeType) {
+            if ($supportedAttributeType == $attribute['type']) {
+                $options[$supportedAttributeType] = $attribute['value'];
             }
         }
-        foreach ($rData['Event']['_AttributeFlattened'] as $i => $attribute) {
-            $this->fastLookupArrayFlattened[$attribute['id']] = $i;
+        return $options;
+    }
+
+    protected function _checkIfInputSupported(array $attribute, array $moduleConfig): bool
+    {
+        foreach ($moduleConfig['mispattributes']['input'] as $supportedAttributeType) {
+            if ($supportedAttributeType == $attribute['type']) {
+                return true;
+            }
         }
+        return false;
+    }
+
+    protected function _handleModuleResult(array $queryResult, array $moduleConfig): array
+    {
+        return $queryResult;
     }
 
     protected function _attachEnrichmentData(array $attribute, array $queryResult, array $rData): array
@@ -111,4 +149,16 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
         }
         return $rData;
     }
+
+    protected function getModuleOptions(array $moduleConfig): array
+    {
+        $type = 'Enrichment';
+        $options = [];
+        if (isset($moduleConfig['meta']['config'])) {
+            foreach ($moduleConfig['meta']['config'] as $conf) {
+                $options[$conf] = Configure::read('Plugin.' . $type . '_' . $moduleConfig['name'] . '_' . $conf);
+            }
+        }
+        return !empty($options) ? $options : ['_' => '_']; // avoid casting empty associative array in to empty list
+    }
 }

app/Model/WorkflowModules/action/Module_attach_warninglist.php

@@ -17,8 +17,6 @@ class Module_attach_warninglist extends WorkflowBaseActionModule
     /** @var Warninglist */
     private $Warninglist;
     private $warninglists;
-    private $fastLookupArrayMispFormat = [];
-    private $fastLookupArrayFlattened = [];
 
 
     public function __construct()
@@ -54,7 +52,6 @@ class Module_attach_warninglist extends WorkflowBaseActionModule
         if ($matchingItems === false) {
             return true;
         }
-        $this->_buildFastLookupForRoamingData($rData);
 
         $warninglists = [];
         if ($params['warninglists']['value'] == 'ALL') {
@@ -84,34 +81,4 @@ class Module_attach_warninglist extends WorkflowBaseActionModule
         $roamingData->setData($rData);
         return true;
     }
-
-    protected function _buildFastLookupForRoamingData($rData): void
-    {
-        foreach ($rData['Event']['Attribute'] as $i => $attribute) {
-            $this->fastLookupArrayMispFormat[$attribute['id']] = $i;
-        }
-        foreach ($rData['Event']['Object'] as $j => $object) {
-            foreach ($object['Attribute'] as $i => $attribute) {
-                $this->fastLookupArrayMispFormat[$attribute['id']] = [$j, $i];
-            }
-        }
-        foreach ($rData['Event']['_AttributeFlattened'] as $i => $attribute) {
-            $this->fastLookupArrayFlattened[$attribute['id']] = $i;
-        }
-    }
-
-    protected function _overrideAttribute(array $oldAttribute, array $newAttribute, array $rData): array
-    {
-        $attributeID = $oldAttribute['id'];
-        $rData['Event']['_AttributeFlattened'][$this->fastLookupArrayFlattened[$attributeID]] = $newAttribute;
-        if (is_array($this->fastLookupArrayMispFormat[$attributeID])) {
-            $objectID = $this->fastLookupArrayMispFormat[$attributeID][0];
-            $attributeID = $this->fastLookupArrayMispFormat[$attributeID][1];
-            $rData['Event']['Object'][$objectID]['Attribute'][$attributeID] = $newAttribute;
-        } else {
-            $attributeID = $this->fastLookupArrayMispFormat[$attributeID];
-            $rData['Event']['Attribute'][$attributeID] = $newAttribute;
-        }
-        return $rData;
-    }
 }

app/Model/WorkflowModules/action/Module_attribute_comment_operation.php

@@ -0,0 +1,61 @@
+<?php
+include_once APP . 'Model/WorkflowModules/action/Module_attribute_edition_operation.php';
+
+class Module_attribute_comment_operation extends Module_attribute_edition_operation
+{
+    public $version = '0.1';
+    public $blocking = false;
+    public $id = 'Module_attribute_comment_operation';
+    public $name = 'Attribute comment operation';
+    public $description = 'Set the Attribute\'s comment to the selected value';
+    public $icon = 'edit';
+    public $inputs = 1;
+    public $outputs = 1;
+    public $support_filters = true;
+    public $expect_misp_core_format = true;
+    public $params = [];
+
+
+    public function __construct()
+    {
+        parent::__construct();
+        $this->params = [
+            [
+                'id' => 'comment',
+                'label' => __('Comment'),
+                'type' => 'textarea',
+                'placeholder' => 'Comment to be set',
+            ],
+        ];
+    }
+
+    public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
+    {
+        parent::exec($node, $roamingData, $errors);
+        $params = $this->getParamsWithValues($node);
+
+        $rData = $roamingData->getData();
+        $user = $roamingData->getUser();
+
+        $matchingItems = $this->getMatchingItemsForAttributes($node, $rData);
+        if ($matchingItems === false) {
+            return true;
+        }
+        $result = $this->__saveAttributes($matchingItems, $rData, $params, $user);
+        $success = $result['success'];
+        $updatedRData = $result['updated_rData'];
+        $roamingData->setData($updatedRData);
+        return $success;
+    }
+
+    protected function _editAttribute(array $attribute, array $rData, array $params): array
+    {
+        $currentRData = $rData;
+        $currentRData['__currentAttribute'] = $attribute;
+        $renderedComment = $this->render_jinja_template($params['comment']['value'], $currentRData);
+        if ($attribute['comment'] !== $params['comment']['value']) {
+            $attribute['comment'] = $renderedComment;
+        }
+        return $attribute;
+    }
+}

app/Model/WorkflowModules/action/Module_attribute_edition_operation.php

@@ -36,15 +36,21 @@ class Module_attribute_edition_operation extends WorkflowBaseActionModule
         return $attribute;
     }
 
-    protected function __saveAttribute(array $attributes, array $rData, array $params, array $user): bool
+    protected function __saveAttributes(array $attributes, array $rData, array $params, array $user): array
     {
         $success = false;
         foreach ($attributes as $attribute) {
-            $attribute = $this->_editAttribute($attribute, $rData, $params);
-            unset($attribute['timestamp']);
-            $saveSuccess = $this->Attribute->editAttribute($attribute, $rData, $user, $attribute['object_id']);
+            $newAttribute = $this->_editAttribute($attribute, $rData, $params);
+            unset($newAttribute['timestamp']);
+            $saveSuccess = $this->Attribute->editAttribute($newAttribute, $rData, $user, $newAttribute['object_id']);
+            if ($saveSuccess) {
+                $rData = $this->_overrideAttribute($attribute, $newAttribute, $rData);
+            }
             $success = $success || !empty($saveSuccess);
         }
-        return $success;
+        return [
+            'success' => $success,
+            'updated_rData' => $rData,
+        ];
     }
 }

app/Model/WorkflowModules/action/Module_attribute_ids_flag_operation.php

@@ -45,8 +45,11 @@ class Module_attribute_ids_flag_operation extends Module_attribute_edition_opera
         if ($matchingItems === false) {
             return true;
         }
-        $result = $this->__saveAttribute($matchingItems, $rData, $params, $user);
-        return $result;
+        $result = $this->__saveAttributes($matchingItems, $rData, $params, $user);
+        $success = $result['success'];
+        $updatedRData = $result['updated_rData'];
+        $roamingData->setData($updatedRData);
+        return $success;
     }
 
     protected function _editAttribute(array $attribute, array $rData, array $params): array

app/Model/WorkflowModules/action/Module_event_distribution_operation.php

@@ -0,0 +1,116 @@
+<?php
+include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
+
+class Module_event_distribution_operation extends WorkflowBaseModule
+{
+    public $version = '0.1';
+    public $blocking = false;
+    public $id = 'Module_event_distribution_operation';
+    public $name = 'Event distribution operation';
+    public $description = 'Set the Event\'s distribution to the selected level';
+    public $icon = 'edit';
+    public $inputs = 1;
+    public $outputs = 1;
+    public $support_filters = false;
+    public $expect_misp_core_format = true;
+    public $params = [];
+
+    private $Attribute, $SharingGroup, $Event;
+
+    public function __construct()
+    {
+        parent::__construct();
+        $this->Attribute = ClassRegistry::init('Attribute');
+        $this->Event = ClassRegistry::init('Event');
+        $distributionLevels = $this->Attribute->shortDist;
+        unset($distributionLevels[5]);
+        $distribution_param = [];
+        foreach ($distributionLevels as $i => $text) {
+            $distribution_param[] = ['name' => $text, 'value' => $i];
+        }
+
+        $this->SharingGroup = ClassRegistry::init('SharingGroup');
+        $sharing_groups = Hash::combine($this->SharingGroup->fetchAllSharingGroup(), '{n}.SharingGroup.id', '{n}.SharingGroup.name');
+
+        $this->params = [
+            [
+                'id' => 'freeze',
+                'label' => __('Preserve timestamp and published state'),
+                'type' => 'select',
+                'options' => [
+                    'modify' => __('Modify timestamp and unpublish'),
+                    'freeze' => __('Freeze timestamp and keep published state'),
+                ],
+                'default' => 'modify',
+            ],
+            [
+                'id' => 'distribution',
+                'label' => 'Distribution',
+                'type' => 'select',
+                'default' => '0',
+                'options' => $distribution_param,
+                'placeholder' => __('Pick a distribution'),
+            ],
+            [
+                'id' => 'sharing_group_id',
+                'label' => 'Sharing Groups',
+                'type' => 'picker',
+                'multiple' => false,
+                'options' => $sharing_groups,
+                'default' => [],
+                'placeholder' => __('Pick a sharing group'),
+                'display_on' => [
+                    'distribution' => '4',
+                ],
+            ],
+        ];
+    }
+
+    public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
+    {
+        parent::exec($node, $roamingData, $errors);
+        $params = $this->getParamsWithValues($node);
+
+        $rData = $roamingData->getData();
+        $user = $roamingData->getUser();
+
+        $matchingItems = $this->getMatchingItemsForAttributes($node, $rData);
+        if ($matchingItems === false || count($matchingItems) == 0) {
+            return true;
+        }
+
+        $result = $this->__saveEvent($rData, $rData, $params, $user);
+        $success = $result['success'];
+        $updatedRData = $result['updated_rData'];
+        $roamingData->setData($updatedRData);
+        return $success;
+    }
+
+    protected function _editEvent(array $event, array $rData, array $params): array
+    {
+        $event['Event']['distribution'] = $params['distribution']['value'];
+        if ($event['Event']['distribution'] == 4) {
+            $event['Event']['sharing_group_id'] = $params['sharing_group_id']['value'];
+        }
+        return $event;
+        
+    }
+
+    protected function __saveEvent(array $event, array $rData, array $params, array $user): array
+    {
+        $freezeTimestamp = $params['freeze']['value'] == 'freeze';
+        $newEvent = $this->_editEvent($event, $rData, $params);
+        $saved = $this->Event->save($newEvent);
+        $saveSuccess = !empty($saved);
+        if ($saveSuccess) {
+            if (!$freezeTimestamp) {
+                $this->Event->touch($newEvent['Event']['id']);
+            }
+            $rData = $newEvent;
+        }
+        return [
+            'success' => $saveSuccess,
+            'updated_rData' => $rData,
+        ];
+    }
+}

app/Model/WorkflowModules/action/Module_ms_teams_webhook.php

@@ -5,7 +5,7 @@ class Module_ms_teams_webhook extends Module_webhook
 {
     public $id = 'ms-teams-webhook';
     public $name = 'MS Teams Webhook';
-    public $version = '0.3';
+    public $version = '0.4';
     public $description = 'Perform callbacks to the MS Teams webhook provided by the "Incoming Webhook" connector';
     public $icon_path = 'MS_Teams.png';
 
@@ -31,7 +31,7 @@ class Module_ms_teams_webhook extends Module_webhook
             [
                 'id' => 'data_extraction_path',
                 'label' => 'Data extraction path',
-                'type' => 'input',
+                'type' => 'hashpath',
                 'default' => '',
                 'placeholder' => 'Attribute.{n}.AttributeTag.{n}.Tag.name',
             ],

app/Model/WorkflowModules/action/Module_push_zmq.php

@@ -6,6 +6,7 @@ class Module_push_zmq extends WorkflowBaseActionModule
     public $blocking = false;
     public $id = 'push-zmq';
     public $name = 'Push to ZMQ';
+    public $version = '0.2';
     public $description = 'Push to the ZMQ channel';
     public $icon_path = 'zeromq.png';
     public $inputs = 1;
@@ -19,7 +20,7 @@ class Module_push_zmq extends WorkflowBaseActionModule
             [
                 'id' => 'data_extraction_path',
                 'label' => 'Data extraction path',
-                'type' => 'input',
+                'type' => 'hashpath',
                 'default' => '',
                 'placeholder' => 'Attribute.{n}.AttributeTag.{n}.Tag.name',
             ],

app/Model/WorkflowModules/action/Module_tag_operation.php

@@ -130,7 +130,7 @@ class Module_tag_operation extends WorkflowBaseActionModule
         return $result;
     }
 
-    private function __addTagsToAttributes(array $attributes, array $options, array $user): bool
+    protected function __addTagsToAttributes(array $attributes, array $options, array $user): bool
     {
         $success = false;
         foreach ($attributes as $attribute) {
@@ -140,7 +140,7 @@ class Module_tag_operation extends WorkflowBaseActionModule
         return $success;
     }
     
-    private function __removeTagsFromAttributes(array $attributes, array $options): bool
+    protected function __removeTagsFromAttributes(array $attributes, array $options): bool
     {
         $success = false;
         foreach ($attributes as $attribute) {
@@ -150,12 +150,12 @@ class Module_tag_operation extends WorkflowBaseActionModule
         return $success;
     }
 
-    private function __addTagsToEvent(array $event, array $options, array $user): bool
+    protected function __addTagsToEvent(array $event, array $options, array $user): bool
     {
         return !empty($this->Event->attachTagsToEventAndTouch($event['Event']['id'], $options, $user));
     }
 
-    private function __removeTagsFromEvent(array $event, array $options): bool
+    protected function __removeTagsFromEvent(array $event, array $options): bool
     {
         return !empty($this->Event->detachTagsFromEventAndTouch($event['Event']['id'], $options));
     }

app/Model/WorkflowModules/action/Module_tag_replacement_generic.php

@@ -0,0 +1,184 @@
+<?php
+include_once APP . 'Model/WorkflowModules/action/Module_tag_operation.php';
+
+class Module_tag_replacement_generic extends Module_tag_operation
+{
+    public $version = '0.1';
+    public $blocking = false;
+    public $id = 'tag_replacement_generic';
+    public $name = 'Tag Replacement Generic';
+    public $description = 'Toggle or remove the IDS flag on selected attributes.';
+    public $icon = 'tags';
+    public $inputs = 1;
+    public $outputs = 1;
+    public $support_filters = true;
+    public $expect_misp_core_format = true;
+    public $params = [];
+
+    public $searchRegex = '';
+    public $substitutionTemplate = '';  // Format the template using CakeText::insert where variables are surround by `{{ var }}`
+
+
+    public function __construct()
+    {
+        parent::__construct();
+        $this->params = [
+            [
+                'id' => 'scope',
+                'label' => __('Scope'),
+                'type' => 'select',
+                'options' => [
+                    'event' => __('Event'),
+                    'attribute' => __('Attributes'),
+                    'all' => __('All'),
+                ],
+                'default' => 'event',
+            ],
+            [
+                'id' => 'remove_substituted',
+                'label' => 'Removed substituted tag',
+                'type' => 'select',
+                'default' => '1',
+                'options' => [
+                    'no' => __('No'),
+                    'yes' => __('Yes'),
+                ],
+            ],
+            [
+                'id' => 'locality',
+                'label' => __('Tag Locality'),
+                'type' => 'select',
+                'options' => [
+                    'local' => __('Local'),
+                    'global' => __('Global'),
+                ],
+                'default' => 'local',
+            ],
+            [
+                'id' => 'relationship_type',
+                'label' => __('Relationship Type'),
+                'type' => 'input',
+                'display_on' => [
+                    'action' => 'add',
+                ],
+            ],
+        ];
+    }
+
+    public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
+    {
+        $params = $this->getParamsWithValues($node);
+
+        $rData = $roamingData->getData();
+        $user = $roamingData->getUser();
+
+        if ($this->filtersEnabled($node)) {
+            $filters = $this->getFilters($node);
+            $extracted = $this->extractData($rData, $filters['selector']);
+            if ($extracted === false) {
+                return false;
+            }
+            $matchingItems = $this->getItemsMatchingCondition($extracted, $filters['value'], $filters['operator'], $filters['path']);
+        } else {
+            $matchingItems = $rData;
+        }
+
+        $matchingEvent = $matchingItems;
+        $matchingAttributes = [];
+        if ($params['scope']['value'] == 'attribute' || $params['scope']['value'] == 'all') {
+            $matchingAttributes = Hash::extract($matchingItems, 'Event._AttributeFlattened.{n}');
+        }
+
+        if (empty($matchingItems)) {
+            return true;
+        }
+
+        $result = false;
+        $optionsRemove = [
+            'local' => [0, 1],
+        ];
+        $optionsAdd = [
+            'local' => $params['locality']['value'] == 'local' ? true : false,
+            'relationship_type' => $params['relationship_type']['value'],
+        ];
+        if ($params['scope']['value'] == 'event' || $params['scope']['value'] == 'all') {
+            $result = $this->replaceOnEvent($matchingEvent, $params, $user, $optionsRemove, $optionsAdd);
+        }
+        if ($params['scope']['value'] == 'attribute' || $params['scope']['value'] == 'all') {
+            $result = $this->replaceOnAttribute($matchingAttributes, $params, $user, $optionsRemove, $optionsAdd);
+        }
+         return $result;
+    }
+
+
+    protected function replaceOnEvent(array $matchingItems, array $params, array $user, array $optionsRemove, array $optionsAdd): bool
+    {
+        $result = true;
+        $extractedTags = Hash::extract($matchingItems['Event']['Tag'], '{n}.name');
+        $options = $this->getReplacementOptions($extractedTags);
+        $optionsRemove['tags'] = $options['remove'];
+        $optionsAdd['tags'] = $options['add'];
+        if ($params['remove_substituted']['value'] == 'yes' && !empty($optionsRemove['tags'])) {
+            $result = $this->__removeTagsFromEvent($matchingItems, $optionsRemove);
+        }
+        if (!empty($optionsAdd['tags'])) {
+            $result = $this->__addTagsToEvent($matchingItems, $optionsAdd, $user);
+        }
+        return $result;
+    }
+
+    protected function replaceOnAttribute(array $matchingItems, array $params, array $user, array $optionsRemove, array $optionsAdd): bool
+    {
+        $result = true;
+        foreach ($matchingItems as $attribute) {
+            $extractedTags = Hash::extract($attribute['Tag'], '{n}.name');
+            $options = $this->getReplacementOptions($extractedTags);
+            $optionsRemove['tags'] = $options['remove'];
+            $optionsAdd['tags'] = $options['add'];
+            if ($params['remove_substituted']['value'] == 'yes' && !empty($optionsRemove['tags'])) {
+                $result = $this->__removeTagsFromAttributes([$attribute], $optionsRemove);
+            }
+            if (!empty($optionsAdd['tags'])) {
+                $result = $this->__addTagsToAttributes([$attribute], $optionsAdd, $user);
+            }
+        }
+        return $result;
+    }
+
+    protected function isAMatch($matches): bool
+    {
+        return !empty($matches);
+    }
+
+    protected function searchAndReplaceTag(array $tags): array
+    {
+        $toReturn = [];
+        foreach ($tags as $tag) {
+            $matches = [];
+            preg_match($this->searchRegex, $tag, $matches);
+            if ($this->isAMatch($matches)) {
+                $toReturn[] = [
+                    'matched' => $tag,
+                    'substitution' => $this->formatSubstitution($matches),
+                ];
+            }
+        }
+        return $toReturn;
+    }
+
+    protected function getReplacementOptions(array $extractedTags)
+    {
+        $substitutionResult = $this->searchAndReplaceTag($extractedTags);
+        $tagsToRemove = Hash::extract($substitutionResult, '{n}.matched');
+        $tagsToAdd = Hash::extract($substitutionResult, '{n}.substitution');
+        return [
+            'remove' => $tagsToRemove,
+            'add' => $tagsToAdd,
+        ];
+    }
+
+    protected function formatSubstitution($matches)
+    {
+        return CakeText::insert($this->substitutionTemplate, $matches, ['before' => '{{', 'after' => '}}']);
+    }
+}

app/Model/WorkflowModules/action/Module_tag_replacement_pap.php

@@ -0,0 +1,36 @@
+<?php
+include_once APP . 'Model/WorkflowModules/action/Module_tag_replacement_generic.php';
+
+class Module_tag_replacement_pap extends Module_tag_replacement_generic
+{
+    public $version = '0.1';
+    public $blocking = false;
+    public $id = 'tag_replacement_pap';
+    public $name = 'Tag Replacement - PAP';
+    public $description = 'Attach a tag (or substitue) a tag by another for the PAP taxonomy';
+    public $icon = 'tags';
+    public $inputs = 1;
+    public $outputs = 1;
+    public $support_filters = true;
+    public $expect_misp_core_format = true;
+    public $params = [];
+
+    public $searchRegex = '/(\w*pap\w*|permissible\W+actions\W+protocol)([:\s\-=]{1,2})"?(?P<predicate>white|clear|green|amber|red)"?/i';
+
+
+    protected function isAMatch($matches): bool
+    {
+        $namespace = 'pap';
+        $predicates = ['white', 'clear', 'green', 'amber', 'red'];
+        if (empty($matches)) {
+            return false;
+        }
+        return strtolower($matches[1]) == $namespace && in_array(strtolower($matches['predicate']), $predicates);
+    }
+
+    protected function formatSubstitution($matches)
+    {
+        return sprintf('PAP:%s', strtoupper($matches['predicate']));
+    }
+
+}

app/Model/WorkflowModules/action/Module_tag_replacement_tlp.php

@@ -0,0 +1,36 @@
+<?php
+include_once APP . 'Model/WorkflowModules/action/Module_tag_replacement_generic.php';
+
+class Module_tag_replacement_tlp extends Module_tag_replacement_generic
+{
+    public $version = '0.1';
+    public $blocking = false;
+    public $id = 'tag_replacement_tlp';
+    public $name = 'Tag Replacement - TLP';
+    public $description = 'Attach a tag (or substitue) a tag by another for the TLP taxonomy';
+    public $icon = 'tags';
+    public $inputs = 1;
+    public $outputs = 1;
+    public $support_filters = true;
+    public $expect_misp_core_format = true;
+    public $params = [];
+
+    public $searchRegex = '/(\w*tlp\w*|traffic\W+light\W+protocol)([:\s\-=]{1,2})"?(?P<predicate>white|clear|green|amber|amber\+strict|red)"?/i';
+
+
+    protected function isAMatch($matches): bool
+    {
+        $namespace = 'tlp';
+        $predicates = ['white', 'clear', 'green', 'amber', 'amber+strict', 'red'];
+        if (empty($matches)) {
+            return false;
+        }
+        return strtolower($matches[1]) == $namespace && in_array(strtolower($matches['predicate']), $predicates);
+    }
+
+    protected function formatSubstitution($matches)
+    {
+        return sprintf('tlp:%s', strtolower($matches['predicate']));
+    }
+
+}

app/Model/WorkflowModules/action/Module_telegram_send_alert.php

@@ -51,11 +51,11 @@ class Module_telegram_send_alert extends Module_webhook
 		'parse_mode' => "HTML",
 	];
 
-	$url = $this->telegram_url . $bot_token . "/sendMessage";
+	$url = $this->telegram_url . "bot" . $bot_token . "/sendMessage";
 
 	$response = $this->doRequest(
 		$url,
-		'json',
+		'application/json',
 		$data
 	);
 

app/Model/WorkflowModules/action/Module_webhook.php

@@ -8,7 +8,7 @@ class Module_webhook extends WorkflowBaseActionModule
 {
     public $id = 'webhook';
     public $name = 'Webhook';
-    public $version = '0.3';
+    public $version = '0.4';
     public $description = 'Allow to perform custom callbacks to the provided URL';
     public $icon_path = 'webhook.png';
     public $inputs = 1;
@@ -42,7 +42,7 @@ class Module_webhook extends WorkflowBaseActionModule
             [
                 'id' => 'data_extraction_path',
                 'label' => __('Data extraction path'),
-                'type' => 'input',
+                'type' => 'hashpath',
                 'default' => '',
                 'placeholder' => 'Attribute.{n}.AttributeTag.{n}.Tag.name',
             ],

app/Model/WorkflowModules/logic/Module_generic_filter_data.php

@@ -5,6 +5,7 @@ class Module_generic_filter_data extends WorkflowFilteringLogicModule
 {
     public $id = 'generic-filter-data';
     public $name = 'Filter :: Generic';
+    public $version = '0.2';
     public $description = 'Generic data filtering block. The module filters incoming data and forward the matching data to its output.';
     public $icon = 'filter';
     public $inputs = 1;
@@ -16,6 +17,8 @@ class Module_generic_filter_data extends WorkflowFilteringLogicModule
         'not_in' => 'Not in',
         'equals' => 'Equals',
         'not_equals' => 'Not equals',
+        'any_value' => 'Any value',
+        'in_or' => 'Any value from',
     ];
 
     public function __construct()
@@ -27,6 +30,7 @@ class Module_generic_filter_data extends WorkflowFilteringLogicModule
                 'label' => __('Filtering Label'),
                 'type' => 'select',
                 'options' => $this->_genFilteringLabels(),
+                'default' => array_keys($this->_genFilteringLabels())[0],
             ],
             [
                 'id' => 'selector',
@@ -39,6 +43,19 @@ class Module_generic_filter_data extends WorkflowFilteringLogicModule
                 'label' => __('Value'),
                 'type' => 'input',
                 'placeholder' => 'tlp:red',
+                'display_on' => [
+                    'operator' => ['in', 'not_in', 'equals', 'not_equals',],
+                ],
+            ],
+            [
+                'id' => 'value_list',
+                'label' => __('Value list'),
+                'type' => 'picker',
+                'picker_create_new' => true,
+                'placeholder' => '[\'ip-src\', \'ip-dst\']',
+                'display_on' => [
+                    'operator' => 'in_or',
+                ],
             ],
             [
                 'id' => 'operator',
@@ -50,7 +67,7 @@ class Module_generic_filter_data extends WorkflowFilteringLogicModule
             [
                 'id' => 'hash_path',
                 'label' => __('Hash path'),
-                'type' => 'input',
+                'type' => 'hashpath',
                 'placeholder' => 'Tag.name',
             ],
         ];
@@ -64,6 +81,8 @@ class Module_generic_filter_data extends WorkflowFilteringLogicModule
         $path = $params['hash_path']['value'];
         $operator = $params['operator']['value'];
         $value = $params['value']['value'];
+        $value_list = $params['value_list']['value'];
+        $valueToEvaluate = $operator == 'in_or' ? $value_list : $value;
         $filteringLabel = $params['filtering-label']['value'];
         $rData = $roamingData->getData();
 
@@ -75,7 +94,7 @@ class Module_generic_filter_data extends WorkflowFilteringLogicModule
             'selector' => $selector,
             'path' => $path,
             'operator' => $operator,
-            'value' => $value,
+            'value' => $valueToEvaluate,
         ];
 
         $roamingData->setData($newRData);

app/Model/WorkflowModules/logic/Module_generic_if.php

@@ -5,6 +5,7 @@ class Module_generic_if extends WorkflowBaseLogicModule
 {
     public $id = 'generic-if';
     public $name = 'IF :: Generic';
+    public $version = '0.2';
     public $description = 'Generic IF / ELSE condition block. The `then` output will be used if the encoded conditions is satisfied, otherwise the `else` output will be used.';
     public $icon = 'code-branch';
     public $inputs = 1;
@@ -17,6 +18,8 @@ class Module_generic_if extends WorkflowBaseLogicModule
         'not_in' => 'Not in',
         'equals' => 'Equals',
         'not_equals' => 'Not equals',
+        'any_value' => 'Any value',
+        'in_or' => 'Any value from',
     ];
 
     public function __construct()
@@ -28,6 +31,19 @@ class Module_generic_if extends WorkflowBaseLogicModule
                 'label' => 'Value',
                 'type' => 'input',
                 'placeholder' => 'tlp:red',
+                'display_on' => [
+                    'operator' => ['in', 'not_in', 'equals', 'not_equals',],
+                ],
+            ],
+            [
+                'id' => 'value_list',
+                'label' => __('Value list'),
+                'type' => 'picker',
+                'picker_create_new' => true,
+                'placeholder' => '[\'ip-src\', \'ip-dst\']',
+                'display_on' => [
+                    'operator' => 'in_or',
+                ],
             ],
             [
                 'id' => 'operator',
@@ -39,7 +55,7 @@ class Module_generic_if extends WorkflowBaseLogicModule
             [
                 'id' => 'hash_path',
                 'label' => 'Hash path',
-                'type' => 'input',
+                'type' => 'hashpath',
                 'placeholder' => 'Attribute.{n}.Tag',
             ],
         ];
@@ -52,6 +68,8 @@ class Module_generic_if extends WorkflowBaseLogicModule
         $path = $params['hash_path']['value'];
         $operator = $params['operator']['value'];
         $value = $params['value']['value'];
+        $value_list = $params['value_list']['value'];
+        $valueToEvaluate = $operator == 'in_or' ? $value_list : $value;
         $data = $roamingData->getData();
         $extracted = [];
         if ($operator == 'equals' || $operator == 'not_equals') {
@@ -59,7 +77,10 @@ class Module_generic_if extends WorkflowBaseLogicModule
         } else {
             $extracted = Hash::extract($data, $path);
         }
-        $eval = $this->evaluateCondition($extracted, $operator, $value);
+        if ($operator == 'any_value' && !empty($extracted)) {
+            return true;
+        }
+        $eval = $this->evaluateCondition($extracted, $operator, $valueToEvaluate);
         return !empty($eval);
     }
 }

app/View/Attributes/index.ctp

@@ -158,7 +158,7 @@ echo $this->element('/genericElements/IndexTable/index_table', [
             [
                 'title' => __('Propose enrichment'),
                 'icon' => 'asterisk',
-                'onclick' => 'simplePopup(\'' . $baseurl . '/events/queryEnrichment/[onclick_params_data_path]/ShadowAttribute\');',
+                'onclick' => 'simplePopup(\'' . $baseurl . '/events/queryEnrichment/[onclick_params_data_path]/Enrichment/ShadowAttribute\');',
                 'onclick_params_data_path' => 'Attribute.id',
                 'complex_requirement' => [
                     'function' => function ($object) use ($modules, $isSiteAdmin, $me) {
@@ -178,7 +178,7 @@ echo $this->element('/genericElements/IndexTable/index_table', [
             [
                 'title' => __('Propose enrichment through Cortex'),
                 'icon' => 'eye',
-                'onclick' => 'simplePopup(\'' . $baseurl . '/events/queryEnrichment/[onclick_params_data_path]/ShadowAttribute/Cortex\');',
+                'onclick' => 'simplePopup(\'' . $baseurl . '/events/queryEnrichment/[onclick_params_data_path]/Enrichment/ShadowAttribute/Cortex\');',
                 'onclick_params_data_path' => 'Attribute.id',
                 'complex_requirement' => [
                     'function' => function ($object) use ($cortex_modules, $isSiteAdmin, $me) {
@@ -202,7 +202,7 @@ echo $this->element('/genericElements/IndexTable/index_table', [
             [
                 'title' => __('Add enrichment'),
                 'icon' => 'asterisk',
-                'onclick' => 'simplePopup(\'' . $baseurl . '/events/queryEnrichment/[onclick_params_data_path]/Attribute\');',
+                'onclick' => 'simplePopup(\'' . $baseurl . '/events/queryEnrichment/[onclick_params_data_path]/Enrichment/Attribute\');',
                 'onclick_params_data_path' => 'Attribute.id',
                 'complex_requirement' => function ($object) use ($modules) {
                     return $this->Acl->canModifyEvent($object) &&
@@ -213,7 +213,7 @@ echo $this->element('/genericElements/IndexTable/index_table', [
             [
                 'title' => __('Add enrichment via Cortex'),
                 'icon' => 'eye',
-                'onclick' => 'simplePopup(\'' . $baseurl . '/events/queryEnrichment/[onclick_params_data_path]/Attribute/Cortex\');',
+                'onclick' => 'simplePopup(\'' . $baseurl . '/events/queryEnrichment/[onclick_params_data_path]/Enrichment/Attribute/Cortex\');',
                 'onclick_params_data_path' => 'Attribute.id',
                 'complex_requirement' => function ($object) use ($cortex_modules) {
                     return $this->Acl->canModifyEvent($object) &&
@@ -250,7 +250,8 @@ echo $this->element('/genericElements/IndexTable/index_table', [
                     return $this->Acl->canModifyEvent($object) && empty($object['Event']['publish_timestamp']);
                 },
             ]
-        ]
+        ],
+        'persistUrlParams' => ['results']
     ]
 ]);
 

app/View/AuditLogs/event_index.ctp

@@ -50,7 +50,8 @@ echo $this->element('genericElements/IndexTable/index_table', [
                 'element_path' => 'AuditLog/change'
             ]
         ],
-        'title' => __('Audit logs for event #%s', intval($event['Event']['id']))
+        'title' => __('Audit logs for event #%s', intval($event['Event']['id'])),
+        'persistUrlParams' => ['eventId', 'org']
     ]
 ]);
 echo '</div>';

app/View/Elements/Events/View/row_attribute.ctp

@@ -11,6 +11,9 @@
   } else {
       $attributeEvent = $event;
   }
+
+  $isNew = $object['timestamp'] > $event['Event']['publish_timestamp'];
+  
   $editScope = $mayModify ? 'Attribute' : 'ShadowAttribute';
   if (!empty($child)) {
       if ($child === 'last' && empty($object['ShadowAttribute'])) {
@@ -69,7 +72,7 @@
       <td class="short context hidden">
           <?php echo $this->element('/Events/View/seen_field', array('object' => $object)); ?>
       </td>
-      <td class="short timestamp"><?= $this->Time->date($object['timestamp']) ?></td>
+      <td class="short timestamp <?= $isNew ? 'bold red' : '' ?>" <?= $isNew ? 'title="' . __('Element or modification to an existing element has not been published yet.') . '"' : '' ?>><?= $this->Time->date($object['timestamp']) . ($isNew ? '*' : '') ?></td>
       <?php
         if (!empty($extended)):
       ?>
@@ -338,12 +341,12 @@
             if ($isAclAdd && ($isSiteAdmin || !$mayModify)):
               if (isset($modules) && isset($modules['types'][$object['type']])):
         ?>
-          <span class="fas fa-asterisk useCursorPointer" role="button" tabindex="0" aria-label="<?php echo __('Query enrichment');?>" onclick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/ShadowAttribute');" title="<?php echo __('Propose enrichment');?>">&nbsp;</span>
+          <span class="fas fa-asterisk useCursorPointer" role="button" tabindex="0" aria-label="<?php echo __('Query enrichment');?>" onclick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/0/Enrichment/ShadowAttribute');" title="<?php echo __('Propose enrichment');?>">&nbsp;</span>
         <?php
               endif;
               if (isset($cortex_modules) && isset($cortex_modules['types'][$object['type']])):
         ?>
-          <span class="icon-eye-open useCursorPointer" role="button" tabindex="0" aria-label="<?php echo __('Query Cortex');?>" onclick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/ShadowAttribute/Cortex');" title="<?php echo __('Propose enrichment through Cortex');?>"></span>
+          <span class="icon-eye-open useCursorPointer" role="button" tabindex="0" aria-label="<?php echo __('Query Cortex');?>" onclick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/Enrichment/0/ShadowAttribute/Cortex');" title="<?php echo __('Propose enrichment through Cortex');?>"></span>
         <?php
               endif;
         ?>
@@ -358,12 +361,12 @@
             if ($isSiteAdmin || $mayModify):
               if (isset($modules) && isset($modules['types'][$object['type']])):
         ?>
-          <span class="fas fa-asterisk useCursorPointer" onclick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/Attribute');" title="<?php echo __('Add enrichment');?>" role="button" tabindex="0" aria-label="<?php echo __('Add enrichment');?>">&nbsp;</span>
+          <span class="fas fa-asterisk useCursorPointer" onclick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/0/Enrichment/Attribute');" title="<?php echo __('Add enrichment');?>" role="button" tabindex="0" aria-label="<?php echo __('Add enrichment');?>">&nbsp;</span>
         <?php
               endif;
               if (isset($cortex_modules) && isset($cortex_modules['types'][$object['type']])):
         ?>
-          <span class="icon-eye-open useCursorPointer" onclick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/Attribute/Cortex');" title="<?php echo __('Add enrichment');?>" role="button" tabindex="0" aria-label="<?php echo __('Add enrichment via Cortex');?>"></span>
+          <span class="icon-eye-open useCursorPointer" onclick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/0/Enrichment/Attribute/Cortex');" title="<?php echo __('Add enrichment');?>" role="button" tabindex="0" aria-label="<?php echo __('Add enrichment via Cortex');?>"></span>
         <?php
               endif;
         ?>

app/View/Elements/Events/View/row_object.ctp

@@ -7,6 +7,7 @@
   } else {
       $objectEvent = $event;
   }
+  $isNew = $object['timestamp'] > $event['Event']['publish_timestamp'];
   if ($object['deleted']) $tr_class .= ' lightBlueRow';
   else $tr_class .= ' blueRow';
   if (!empty($k)) {
@@ -33,7 +34,7 @@ $objectId = intval($object['id']);
   <td class="short context hidden">
       <?php echo $this->element('/Events/View/seen_field', array('object' => $object)); ?>
   </td>
-  <td class="short timestamp"><?= $this->Time->date($object['timestamp']) ?></td>
+  <td class="short timestamp <?= $isNew ? 'bold red' : '' ?>" <?= $isNew ? 'title="' . __('Element or modification to an existing element has not been published yet.') . '"' : '' ?>><?= $this->Time->date($object['timestamp']) . ($isNew ? '*' : '') ?></td>
   <?php
     if ($extended):
   ?>
@@ -111,6 +112,17 @@ $objectId = intval($object['id']);
   <td class="short action-links">
     <?php
       if ($mayModify) {
+          if (Configure::read('Plugin.Enrichment_services_enable') && ($isSiteAdmin || $mayModify) && (isset($modules) && isset($modules['types'][$object['name']]))) {
+            echo sprintf(
+              '<span class="fa fa-asterisk white useCursorPointer" title="%1$s" role="button" tabindex="0" aria-label="%1$s" onclick="%2$s"></span> ',
+              __('Add enrichment'),
+              sprintf(
+                'simplePopup(\'%s/events/queryEnrichment/%s/0/Enrichment/Object\');',
+                  $baseurl, $objectId
+              )
+            );
+          }
+
           if (empty($object['deleted'])) {
             echo sprintf(
               '<a href="%s/objects/edit/%s" title="%s" aria-label="%s" class="fa fa-edit white"></a> ',

app/View/Elements/Workflows/infoModal.ctp

@@ -8,6 +8,7 @@
             <li class="active"><a href=" #modal-info-concept" data-toggle="tab"><?= __('Terminology & Concepts') ?></a></li>
             <li class=""><a href=" #modal-hash-path" data-toggle="tab"><?= __('Hash Path') ?></a></li>
             <li class=""><a href=" #modal-core-format" data-toggle="tab"><?= __('MISP Core Format') ?></a></li>
+            <li class=""><a href=" #modal-jinja2" data-toggle="tab"><?= __('Jinja2 Syntax') ?></a></li>
             <li class=""><a href=" #modal-blueprint" data-toggle="tab"><?= __('Blueprints') ?></a></li>
             <li class=""><a href=" #modal-debugging" data-toggle="tab"><?= __('Debugging') ?></a></li>
             <li><a href="#modal-info-usage" data-toggle="tab"><?= __('Usage & Shortcuts') ?></a></li>
@@ -128,12 +129,12 @@ $data_passed_to_if_module = [
                     <li><?= __('Attributes are always encapsulated in the Event or Object') ?></li>
                     <li><?= __('Additional key') ?> <code>_AttributeFlattened</code> containing all Attributes</li>
                     <li><?= __('Additional key') ?> <code>_allTags</code> containing all tags</li>
-                        <ul>
-                            <li><?= __('Additional key %s for Tags', '<code>inherited</code>') ?></li>
-                        </ul>
+                    <ul>
+                        <li><?= __('Additional key %s for Tags', '<code>inherited</code>') ?></li>
+                    </ul>
                 </ul>
                 <p><strong><?= __('Sample:') ?></strong></p>
-                <pre>
+                <pre id="misp-core-format-sample">
 {
     "Event": {
         "id": "64",
@@ -641,6 +642,50 @@ $data_passed_to_if_module = [
 
             </div>
 
+            <div class="tab-pane" id="modal-jinja2">
+                <h3><?= __('Jinja2 Syntax') ?></h3>
+                <p><i class="fa-fw <?= $this->FontAwesome->getClass('exclamation-triangle') ?>"></i> <?= __('For these examples, we consider the module received data under the MISP core format.') ?></p>
+                <h4><?= __('You can use the dot <code>`.` </code> notation or the subscript syntax <code>`[]`</code> to access attributes of a variable') ?></h4>
+                <ul>
+                    <li><code>{{ Event.info }}</code>: <?= __('Shows the title of the event') ?></li>
+                    <li><code>{{ Event['info'] }}</code>: <?= __('Shows the title of the event') ?></li>
+                </ul>
+                <h4><?= __('Jinja2 allows you to easily create list') ?></h4>
+<pre>
+{% for attribute in Event.Attribute %}
+- {{ attribute.value }}
+{% endfor %}
+</pre>
+
+                <h4><?= __('Jinja2 allows you to add logic') ?></h4>
+<pre>
+{% if "tlp:white" in Event.Tag %}
+    - This Event has the TLP:WHITE tag
+{% else %}
+    - This Event doesn't have the TLP:WHITE tag
+{% endif %}
+</pre>
+
+                <h4><?= __('Jinja2 allows you to modify variables by using filters') ?></h4>
+<pre>
+# The `reverse` filter
+- `{{ Event.info | reverse }}`
+-> The event title, but reversed
+
+# The `format` filter
+- `{{ "%s :: %s" | format(Event.Attribute[0].type, Event.Attribute[0].value) }}`
+-> Allow to format string. python `.format()`
+
+# The `groupby` filter
+{% for type, attributes in Event.Attribute|groupby("type") %}
+- {{ type }}{% for attribute in attributes %}
+    - {{ attribute.value }}
+    {% endfor %}
+{% endfor %}
+</pre>
+
+            </div>
+
             <div class="tab-pane" id="modal-blueprint">
                 <h3><?= __('Blueprints') ?></h3>
                 <ul>

app/View/Elements/dashboard/Widgets/BarChart.ctp

@@ -1,3 +1,8 @@
+<?php
+    // If you want to force log scale, set the `forceLogarithm` option to true in the widget config: `{"widget_config": {"forceLogarithm": "1"}}`
+    // Or the widget already contains data in log, set `logarithmic` to 1
+
+?>
 <table style="border-spacing:0px;">
 <?php
     if (!empty($data['logarithmic'])) {
@@ -7,6 +12,7 @@
             $max = 0;
         } else {
             $max = max($data['data']);
+            $max = !empty($config['widget_config']['forceLogarithm']) ? log10($max) : $max;
         }
     }
     if (!empty($max)) {
@@ -14,6 +20,8 @@
             $value = $count;
             if (!empty($data['logarithmic'])) {
                 $value = $data['logarithmic'][$entry];
+            } else if (!empty($config['widget_config']['forceLogarithm'])) {
+                $value = log10($count);
             }
             $shortlabel = $entry;
             if (mb_strlen($shortlabel) > 30) {

app/View/Elements/dashboard/Widgets/WorldMap.ctp

@@ -37,7 +37,13 @@
                     values: mapData,
                     scale:
                     <?= $data['colour_scale'] ?>, //  gradient blue->green->yellow->red
-                    normalizeFunction: 'polynomial'
+                    normalizeFunction: 'polynomial',
+                    legend: {
+                        vertical: false,
+                        labelRender: function(v){
+                            return Math.round(v);
+                        }
+                    }
                 }]
             },
             onRegionTipShow: function(e, el, code) {

app/View/Elements/dashboard/widget.ctp

@@ -12,9 +12,17 @@
         sprintf(
             '<div class="grid-stack-item-content"><div class="widgetTitle"><span class="widgetTitleText">%s</span> %s %s %s</div><div class="widgetContent">%s</div></div>',
             empty($widget['config']['alias']) ? h($widget['title']) : h($widget['config']['alias']),
-            sprintf(
-                '<span class="fas fa-download export-widget useCursorPointer" title="%s"></span>',
-                __('Export raw data')
+            sprintf('<span class="dropdown">
+                    <a class="dropdown-toggle" data-toggle="dropdown" href="#" title="%s"><i class="%s"></i></a>
+                    <ul class="dropdown-menu widget-export-menu" role="menu" aria-labelledby="dLabel">
+                        <li><a tabindex="-1" href="#" data-exporttype="json">%s</a></li>
+                        <li><a tabindex="-1" href="#" data-exporttype="csv">%s</a></li>
+                    </ul>
+                </span>',
+                __('Export raw data'),
+                $this->FontAwesome->getClass('download'),
+                __('Export as JSON'),
+                __('Export as CSV')
             ),
             sprintf(
                 '<span class="fas fa-edit edit-widget useCursorPointer" title="%s"></span>',

app/View/Elements/eventdiscussion.ctp

@@ -127,6 +127,7 @@
         </div>
     <?php endif; ?>
     <div class="comment">
+    <?php if ($this->Acl->canAccess('posts', 'add')): ?>
     <?php
         if (isset($currentEvent)) $url = $baseurl . '/posts/add/event/' . $currentEvent;
         else $url = $baseurl . '/posts/add/thread/' . $thread['Thread']['id'];
@@ -153,6 +154,7 @@
     <?php
         echo $this->Form->end();
     ?>
+    <?php endif; ?>
     </div>
 </div>
 <script type="text/javascript">

app/View/Elements/genericElements/Form/fieldScaffold.ctp

@@ -40,6 +40,9 @@
         } else {
             $params['class'] = '';
         }
+        if (!empty($fieldData['autofocus'])) {
+            $params['autofocus'] = 1;
+        }
         if (empty($fieldData['type']) || $fieldData['type'] !== 'checkbox' ) {
             $params['class'] .= ' form-control';
         }

app/View/Elements/global_menu.ctp

@@ -465,7 +465,7 @@
                     array(
                         'text' => __('Audit Logs'),
                         'url' => $baseurl . '/admin/audit_logs/index',
-                        'requirement' => Configure::read('MISP.log_new_audit') && $isAdmin,
+                        'requirement' => Configure::read('MISP.log_new_audit') && $this->Acl->canAccess('auditLogs', 'admin_index'),
                     ),
                     array(
                         'text' => __('Access Logs'),
@@ -475,7 +475,7 @@
                     array(
                         'text' => __('Search Logs'),
                         'url' => $baseurl . '/admin/logs/search',
-                        'requirement' => $isAdmin
+                        'requirement' => $this->Acl->canAccess('logs', 'admin_search')
                     )
                 )
             ),

app/View/Elements/healthElements/diagnostics.ctp

@@ -7,33 +7,48 @@ $humanReadableFilesize = function ($bytes, $dec = 2) {
 ?>
 <div style="border:1px solid #dddddd; margin-top:1px; width:95%; padding:10px">
     <?php if (!$dbEncodingStatus):?>
-    <div style="font-size:12pt;padding-left:3px;width:100%;background-color:red;color:white;font-weight:bold;"><?= __('Incorrect database encoding setting: Your database connection is currently NOT set to UTF-8. Please make sure to uncomment the \'encoding\' => \'utf8\' line in ') . APP; ?>Config/database.php</div>
+    <div style="font-size:12pt;padding-left:3px;width:100%;background-color:red;color:white;font-weight:bold;">
+        <?= __('Incorrect database encoding setting: Your database connection is currently NOT set to UTF-8. Please make sure to uncomment the \'encoding\' => \'utf8\' line in ') . APP; ?>Config/database.php
+    </div>
     <?php endif; ?>
     <h3><?= __('MISP version');?></h3>
-    <p><?= __('Every version of MISP includes a JSON file with the current version. This is checked against the latest tag on GitHub, if there is a version mismatch the tool will warn you about it. Make sure that you update MISP regularly.');?></p>
+    <p><?php
+        echo __('Every version of MISP includes a JSON file with the current version.') . " ";
+        if (Configure::read('MISP.online_version_check') || !Configure::check('MISP.online_version_check')) {
+            echo _('This is checked against the latest tag on GitHub, if there is a version mismatch the tool will warn you about it.') . " ";
+        } else {
+            echo "<b>" . __('The online version check is disabled, so you will not be warned about an outdated MISP version.') . "</b> ";
+        }
+        echo __('Make sure that you update MISP regularly.');
+    ?></p>
     <div class="diagnostics-box" style="width:100%">
         <span><?= __('Currently installed version…');?>
             <?php
                 $upToDate = isset($version['upToDate']) ? $version['upToDate'] : null;
                 switch ($upToDate) {
                     case 'newer':
-                        $fontColour = 'orange';
+                        $colourClass = 'orange';
                         $versionText = __('Upcoming development version');
                         break;
                     case 'older':
-                        $fontColour = 'red';
+                        $colourClass = 'red';
                         $versionText = __('Outdated version');
                         break;
                     case 'same':
-                        $fontColour = 'green';
+                        $colourClass = 'green';
                         $versionText = __('OK');
                         break;
+                    case 'disabled':
+                        $colourClass = 'bold';
+                        $versionText = __('Online version checks are disabled');
+                    break;
+                    case 'error':
                     default:
-                        $fontColour = 'red';
+                        $colourClass = 'red';
                         $versionText = __('Could not retrieve version from GitHub');
                 }
             ?>
-            <span style="color:<?php echo $fontColour; ?>;">
+            <span class="<?= $colourClass; ?>">
                 <?= (isset($version['current']) ? $version['current'] : __('Unknown')) . ' (' . ($commit ? h($commit) : __('Unknown')) . ')';
                 ?>
                 <?php if ($commit === ''): ?>
@@ -45,25 +60,45 @@ $humanReadableFilesize = function ($bytes, $dec = 2) {
             </span>
         </span><br>
         <span><?php echo __('Latest available version…');?>
-            <span style="color:<?php echo $fontColour; ?>;">
-                <?= (isset($version['newest']) ? $version['newest'] : __('Unknown')) . ' (' . ($latestCommit ? $latestCommit : __('Unknown')) . ')' ?>
+            <span class="<?= $colourClass; ?>">
+                <?php
+                if (!Configure::read('MISP.online_version_check') && Configure::check('MISP.online_version_check')) {
+                    echo __('Online version checks are disabled');
+                } else {
+                    echo (isset($version['newest']) ? $version['newest'] : __('Unknown')) . ' (' . ($latestCommit ? $latestCommit : __('Unknown')) . ')';
+                }
+                ?>
             </span>
         </span><br>
         <span><?php echo __('Status…');?>
-            <span style="color:<?= $fontColour; ?>;"><?= $versionText ?></span>
+            <span class="<?= $colourClass; ?>"><?= $versionText ?></span>
         </span><br>
         <span><?php echo __('Current branch…');?>
             <?php
-                $branchColour = $branch == '2.4' ? 'green' : 'red bold';
+                $branchColour = $branch == '2.4' ? 'green' : (!Configure::read('MISP.self_update') && Configure::check('MISP.self_update')) ? 'bold' : 'red bold';
             ?>
             <span class="<?php echo h($branchColour); ?>">
-                <?= $branch == '2.4' ? h($branch) : __('You are not on a branch, Update MISP will fail'); ?>
+                <?php
+                  if ($branch == '2.4') {
+                      h($branch);
+                  } elseif (!Configure::read('MISP.self_update') && Configure::check('MISP.self_update')) {
+                      echo __('You are not on a branch, but since MISP self-update is disabled this is expected.');
+                  } else {
+                      echo __('You are not on a branch, Update MISP will fail');
+                  }
+                ?>
             </span>
         </span><br>
         <pre class="hidden green bold" id="gitResult"></pre>
+
+    </div>
+
+    <h3><?php echo __('Update MISP');?></h3>
+    <?php if (Configure::read('MISP.self_update') || !Configure::check('MISP.self_update')): ?>
+    <p>
         <button title="<?php echo __('Pull the latest MISP version from GitHub');?>" class="btn btn-inverse" style="padding-top:1px;padding-bottom:1px;" onClick = "updateMISP();"><?php echo __('Update MISP');?></button>
         <a title="<?php echo __('Click the following button to go to the update progress page. This page lists all updates that are currently queued and executed.'); ?>" style="margin-left: 5px;" href="<?php echo $baseurl; ?>/servers/updateProgress/"><i class="fas fa-tasks"></i> <?php echo __('View Update Progress');?></a>
-    </div>
+    </p>
     <h3><?php echo __('Submodules version');?>
         <it id="refreshSubmoduleStatus" class="fas fa-sync useCursorPointer" style="font-size: small; margin-left: 5px;" title="<?php echo __('Refresh submodules version.'); ?>"></it>
     </h3>
@@ -71,6 +106,9 @@ $humanReadableFilesize = function ($bytes, $dec = 2) {
     <span id="updateAllJson" class="btn btn-inverse" title="<?php echo __('Load all JSON into the database.'); ?>">
         <it class="fas fa-file-upload"></it> <?php echo __("Load JSON into database"); ?>
     </span>
+    <?php else: ?>
+      You are using a MISP installation method that does not support or recommend using the MISP self-update, such as a Docker container. Please update using the appropriate update mechanism.
+    <?php endif; ?>
 
     <h3><?php echo __('Writeable Directories and files');?></h3>
     <p><?php echo __('The following directories and files have to be writeable for MISP to function properly. Make sure that the apache user has write privileges for the directories below.');?></p>
@@ -565,13 +603,13 @@ $humanReadableFilesize = function ($bytes, $dec = 2) {
     <span class="btn btn-inverse" role="button" tabindex="0" aria-label="<?php echo __('Recover deleted events');?>" title="<?php echo __('Recover deleted events');?>" style="padding-top:1px;padding-bottom:1px;" onClick="location.href = '<?php echo $baseurl; ?>/events/restoreDeletedEvents';"><?php echo __('Recover deleted events');?></span>
 </div>
 
+<?php if (Configure::read('MISP.self_update') || !Configure::check('MISP.self_update')): ?>
 <script>
     $(function() {
         updateSubModulesStatus();
         $('#refreshSubmoduleStatus').click(function() { updateSubModulesStatus(); });
         $('#updateAllJson').click(function() { updateAllJson(); });
     });
-
     function updateSubModulesStatus(message, job_sent, sync_result) {
         job_sent = job_sent === undefined ? false : job_sent;
         sync_result = sync_result === undefined ? '' : sync_result;
@@ -616,3 +654,4 @@ $humanReadableFilesize = function ($bytes, $dec = 2) {
         });
     }
 </script>
+<?php endif; ?>