From 4b480055b9d8c393ee76492b2950256dec577001 Mon Sep 17 00:00:00 2001
From: Christian Studer <christian.studer@circl.lu>
Date: Fri, 28 Apr 2023 09:54:41 +0200
Subject: [PATCH 01/96] fix: [taxii_push] The path `resolve` method needs to be
 called

---
 app/files/scripts/taxii/taxii_push.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/scripts/taxii/taxii_push.py b/app/files/scripts/taxii/taxii_push.py
index 948f42ebd..9276e9102 100644
--- a/app/files/scripts/taxii/taxii_push.py
+++ b/app/files/scripts/taxii/taxii_push.py
@@ -12,7 +12,7 @@ from base64 import b64decode
 from pathlib import Path
 from requests.auth import HTTPBasicAuth
 
-_script_path = Path(__file__).resolve.parents[1]
+_script_path = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(_script_path / 'misp-stix'))
 import misp_stix_converter
 

From 26ad0ef607f182ae93fe5a074c1742e95112fe28 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Thu, 29 Jun 2023 12:38:29 +0200
Subject: [PATCH 02/96] fix: [customauth] Don't renew the session with each
 query

- Leave the session handling to the normal life-cycle management
- should solve the issues where CSRF keeps kicking users off
---
 app/Controller/AppController.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php
index 19f01ac6e..6ae7b001f 100755
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@@ -1112,7 +1112,7 @@ class AppController extends Controller
                 $user['User'] = $temp;
                 if ($user['User']) {
                     $this->User->updateLoginTimes($user['User']);
-                    $this->Session->renew();
+                    //$this->Session->renew();
                     $this->Session->write(AuthComponent::$sessionKey, $user['User']);
                     if (Configure::read('MISP.log_auth')) {
                         $this->Log = ClassRegistry::init('Log');

From 4a2734bb115fc99fdbdbb5c6566017361b4c3e7b Mon Sep 17 00:00:00 2001
From: Alex Jarvis-Blanks <45558436+ajb3932@users.noreply.github.com>
Date: Thu, 29 Jun 2023 16:33:59 +0100
Subject: [PATCH 03/96] Update INSTALL.sh

The current command adds the line "Listen 443" after the line containing "Listen 80" even if "Listen 443" already exists.

In my update, the "Listen 443" line will only be added if it doesn't already exist in the file.
---
 INSTALL/INSTALL.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/INSTALL/INSTALL.sh b/INSTALL/INSTALL.sh
index ce2132009..46f5d4739 100755
--- a/INSTALL/INSTALL.sh
+++ b/INSTALL/INSTALL.sh
@@ -2543,7 +2543,7 @@ apacheConfig_RHEL7 () {
   #sudo sed -i "s/SetHandler/\#SetHandler/g" /etc/httpd/conf.d/misp.ssl.conf
   sudo rm /etc/httpd/conf.d/ssl.conf
   sudo chmod 644 /etc/httpd/conf.d/misp.ssl.conf
-  sudo sed -i '/Listen 80/a Listen 443' /etc/httpd/conf/httpd.conf
+  sudo sed -i '/Listen 443/!s/Listen 80/a Listen 443/' /etc/httpd/conf/httpd.conf
 
   # If a valid SSL certificate is not already created for the server, create a self-signed certificate:
   echo "The Common Name used below will be: ${OPENSSL_CN}"
@@ -2591,7 +2591,7 @@ apacheConfig_RHEL8 () {
   #sudo sed -i "s/SetHandler/\#SetHandler/g" /etc/httpd/conf.d/misp.ssl.conf
   sudo rm /etc/httpd/conf.d/ssl.conf
   sudo chmod 644 /etc/httpd/conf.d/misp.ssl.conf
-  sudo sed -i '/Listen 80/a Listen 443' /etc/httpd/conf/httpd.conf
+  sudo sed -i '/Listen 443/!s/Listen 80/a Listen 443/' /etc/httpd/conf/httpd.conf
 
   # If a valid SSL certificate is not already created for the server, create a self-signed certificate:
   echo "The Common Name used below will be: ${OPENSSL_CN}"

From f3a30ac38c4d75f294329c8eec379211a399def0 Mon Sep 17 00:00:00 2001
From: Steve Clement <steve@localhost.lu>
Date: Sat, 1 Jul 2023 17:10:16 +0200
Subject: [PATCH 04/96] chg: [doc] "Listen 443" line will only be added if it
 doesn't already exist in the file."

---
 docs/INSTALL.rhel7.md | 2 +-
 docs/INSTALL.rhel8.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/docs/INSTALL.rhel7.md b/docs/INSTALL.rhel7.md
index 003e36713..b392da793 100644
--- a/docs/INSTALL.rhel7.md
+++ b/docs/INSTALL.rhel7.md
@@ -411,7 +411,7 @@ apacheConfig_RHEL7 () {
   #sudo sed -i "s/SetHandler/\#SetHandler/g" /etc/httpd/conf.d/misp.ssl.conf
   sudo rm /etc/httpd/conf.d/ssl.conf
   sudo chmod 644 /etc/httpd/conf.d/misp.ssl.conf
-  sudo sed -i '/Listen 80/a Listen 443' /etc/httpd/conf/httpd.conf
+  sudo sed -i '/Listen 443/!s/Listen 80/a Listen 443/' /etc/httpd/conf/httpd.conf
 
   # If a valid SSL certificate is not already created for the server, create a self-signed certificate:
   echo "The Common Name used below will be: ${OPENSSL_CN}"
diff --git a/docs/INSTALL.rhel8.md b/docs/INSTALL.rhel8.md
index 5db35de42..3e8243ee8 100644
--- a/docs/INSTALL.rhel8.md
+++ b/docs/INSTALL.rhel8.md
@@ -452,7 +452,7 @@ apacheConfig_RHEL8 () {
   #sudo sed -i "s/SetHandler/\#SetHandler/g" /etc/httpd/conf.d/misp.ssl.conf
   sudo rm /etc/httpd/conf.d/ssl.conf
   sudo chmod 644 /etc/httpd/conf.d/misp.ssl.conf
-  sudo sed -i '/Listen 80/a Listen 443' /etc/httpd/conf/httpd.conf
+  sudo sed -i '/Listen 443/!s/Listen 80/a Listen 443/' /etc/httpd/conf/httpd.conf
 
   # If a valid SSL certificate is not already created for the server, create a self-signed certificate:
   echo "The Common Name used below will be: ${OPENSSL_CN}"

From 648c1c9ea21b85e2d360033a65882fe33c8b9bbe Mon Sep 17 00:00:00 2001
From: Steve Clement <steve@localhost.lu>
Date: Sat, 1 Jul 2023 17:15:15 +0200
Subject: [PATCH 05/96] chg: [installer] Updated installer to latest version.

---
 INSTALL/INSTALL.sh        | 10 +++++++++-
 INSTALL/INSTALL.sh.sfv    |  6 +++---
 INSTALL/INSTALL.sh.sha1   |  2 +-
 INSTALL/INSTALL.sh.sha256 |  2 +-
 INSTALL/INSTALL.sh.sha384 |  2 +-
 INSTALL/INSTALL.sh.sha512 |  2 +-
 6 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/INSTALL/INSTALL.sh b/INSTALL/INSTALL.sh
index 46f5d4739..4e01e2c20 100755
--- a/INSTALL/INSTALL.sh
+++ b/INSTALL/INSTALL.sh
@@ -1509,9 +1509,17 @@ coreCAKE () {
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "GnuPG.homedir" "${PATH_TO_MISP}/.gnupg"
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "GnuPG.password" "${GPG_PASSPHRASE}"
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "GnuPG.obscure_subject" true
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "GnuPG.key_fetching_disabled" false
   # FIXME: what if we have not gpg binary but a gpg2 one?
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "GnuPG.binary" "$(which gpg)"
 
+  # LinOTP
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "LinOTPAuth.enabled" false
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "LinOTPAuth.baseUrl" "https://<your-linotp-baseUrl>"
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "LinOTPAuth.realm" "lino"
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "LinOTPAuth.verifyssl" true
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "LinOTPAuth.mixedauth" false
+
   # Enable installer org and tune some configurables
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.host_org_id" 1
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "MISP.email" "info@admin.test"
@@ -1870,7 +1878,7 @@ mispmodules () {
 modulesCAKE () {
   # Enable Enrichment, set better timeouts
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_services_enable" true
-  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_hover_enable" true
+  ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_hover_enable" false
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_hover_popover_only" false
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_hover_timeout" 150
   ${SUDO_WWW} ${RUN_PHP} -- ${CAKE} Admin setSetting "Plugin.Enrichment_timeout" 300
diff --git a/INSTALL/INSTALL.sh.sfv b/INSTALL/INSTALL.sh.sfv
index 6f53f2312..1e88b1879 100644
--- a/INSTALL/INSTALL.sh.sfv
+++ b/INSTALL/INSTALL.sh.sfv
@@ -1,5 +1,5 @@
-; Generated by RHash v1.4.2 on 2022-05-23 at 12:45.34
+; Generated by RHash v1.4.2 on 2023-07-01 at 17:15.04
 ; Written by Kravchenko Aleksey (Akademgorodok) - http://rhash.sf.net/
 ;
-;       160126  12:45.34 2022-05-23 INSTALL.sh
-INSTALL.sh 4296D40B11B3002DF3FDFD69A508ED5ECACB8C13 D32E5A4B0F37F4C937CD4F85927E998D917BCBE89E4E0E864FFD7EA09E29ADEF BD093D8018C351E3D3722646E269C4B60E6DA19F42150338CE6FD72FEE293B8B89AA69D48A84B19D3EFDDAE25EC9E646 ECACC3071E130058C3DDECC86E1CBF27DD4F11389D10F43B14293B1915F7A24F02D0DA51E299706A38C00F2D2A7505B0FE46E33B705E53594383CE65461F2B08
+;       160686  17:15.04 2023-07-01 INSTALL.sh
+INSTALL.sh 9576C31EC5BD942E1C9B12413E6408E4623252F7 78B708FE1FC6B39BE081B9F05C6AA5E1478F8762CAF5A8A7671A12EBA4D3C1C5 27991471FB5788F42AF3BBF86FC80A95341AA17AE9487016EEC94961A48437172702EB8E2D6CB300387E87D9E8E0E3E5 C1C21FD491AEFD662C87C3EF62837D769E63E9CF2446B9BD607CCEF8AFD72528824A8F408C6892FD51109390104010EF90DA7F4828950A8671D2986A6B8E216F
diff --git a/INSTALL/INSTALL.sh.sha1 b/INSTALL/INSTALL.sh.sha1
index 90e04e1ef..e1db6c05b 100644
--- a/INSTALL/INSTALL.sh.sha1
+++ b/INSTALL/INSTALL.sh.sha1
@@ -1 +1 @@
-4296d40b11b3002df3fdfd69a508ed5ecacb8c13  INSTALL.sh
+9576c31ec5bd942e1c9b12413e6408e4623252f7  INSTALL.sh
diff --git a/INSTALL/INSTALL.sh.sha256 b/INSTALL/INSTALL.sh.sha256
index 80d0ca800..6622f0558 100644
--- a/INSTALL/INSTALL.sh.sha256
+++ b/INSTALL/INSTALL.sh.sha256
@@ -1 +1 @@
-d32e5a4b0f37f4c937cd4f85927e998d917bcbe89e4e0e864ffd7ea09e29adef  INSTALL.sh
+78b708fe1fc6b39be081b9f05c6aa5e1478f8762caf5a8a7671a12eba4d3c1c5  INSTALL.sh
diff --git a/INSTALL/INSTALL.sh.sha384 b/INSTALL/INSTALL.sh.sha384
index 58d22834f..0f9ebbe68 100644
--- a/INSTALL/INSTALL.sh.sha384
+++ b/INSTALL/INSTALL.sh.sha384
@@ -1 +1 @@
-bd093d8018c351e3d3722646e269c4b60e6da19f42150338ce6fd72fee293b8b89aa69d48a84b19d3efddae25ec9e646  INSTALL.sh
+27991471fb5788f42af3bbf86fc80a95341aa17ae9487016eec94961a48437172702eb8e2d6cb300387e87d9e8e0e3e5  INSTALL.sh
diff --git a/INSTALL/INSTALL.sh.sha512 b/INSTALL/INSTALL.sh.sha512
index e83897162..fa8fc6529 100644
--- a/INSTALL/INSTALL.sh.sha512
+++ b/INSTALL/INSTALL.sh.sha512
@@ -1 +1 @@
-ecacc3071e130058c3ddecc86e1cbf27dd4f11389d10f43b14293b1915f7a24f02d0da51e299706a38c00f2d2a7505b0fe46e33b705e53594383ce65461f2b08  INSTALL.sh
+c1c21fd491aefd662c87c3ef62837d769e63e9cf2446b9bd607ccef8afd72528824a8f408c6892fd51109390104010ef90da7f4828950a8671d2986a6b8e216f  INSTALL.sh

From 3cc3549bac9ea42da0202bbed42ad914c0f0fcd7 Mon Sep 17 00:00:00 2001
From: vincenzocaputo <32276363+vincenzocaputo@users.noreply.github.com>
Date: Sun, 2 Jul 2023 22:36:17 +0200
Subject: [PATCH 06/96] Add dashboard widget for monthly number of events per
 org

---
 app/Lib/Dashboard/OrgEventsWidget.php | 121 ++++++++++++++++++++++++++
 1 file changed, 121 insertions(+)
 create mode 100644 app/Lib/Dashboard/OrgEventsWidget.php

diff --git a/app/Lib/Dashboard/OrgEventsWidget.php b/app/Lib/Dashboard/OrgEventsWidget.php
new file mode 100644
index 000000000..cf74e9a88
--- /dev/null
+++ b/app/Lib/Dashboard/OrgEventsWidget.php
@@ -0,0 +1,121 @@
+<?php
+/**
+* Org Events widget which reportes the number of events created monthly by each local organizations
+*
+*/
+class OrgEventsWidget
+{
+    public $title = 'Org Events';
+    public $render = 'MultiLineChart';
+    public $width = 8;
+    public $height = 6;
+    public $description = 'A graph to show the monthly number of events per organisation';
+    public $cacheLifetime = 10;
+    public $autoRefreshDelay = false;
+    public $params = array (
+        'blocklist_orgs' => 'A list of organisation names to filter out',
+        'months' => 'Number of past months to consider for the graph',
+        'logarithmic' => 'Visualize data on logarithmic scale'
+    );
+
+    public $placeholder =
+'{
+    "blocklist_orgs": ["Orgs to filter"],
+    "months": "6",
+    "logarithmic": "true"
+}';
+
+
+
+
+
+    /*
+    * Target_month must be from 1 to 12
+    * Target year must be 4 digits
+    */
+    private function org_events_count($user, $org, $target_month, $target_year) {
+        $events_count = 0;
+
+        $start_date = $target_year.'-'.$target_month.'-01';
+        if($target_month == 12) {
+            $end_date = ($target_year+1).'-01-01';
+        } else {
+            $end_date = $target_year.'-'.($target_month+1).'-01';
+        }
+        $conditions = array('Event.orgc_id' => $org['Organisation']['id'], 'Event.date >=' => $start_date, 'Event.date <' => $end_date);
+
+        //This is required to enforce the ACL (not pull directly from the DB)
+        $eventIds = $this->Event->fetchSimpleEventIds($user, array('conditions' => $conditions));
+
+        if(!empty($eventIds)) {
+            $params = array('Event.id' => $eventIds);
+            $events = $this->Event->find('all', array('conditions' => array('AND' => $params)));
+            foreach($events as $event) {
+                $events_count+= 1;
+            }
+        }
+        return $events_count;
+    }
+
+    private function filter_ghost_orgs(&$data, $orgs){
+        foreach ($data['data'] as &$item) {
+            foreach(array_keys($orgs) as $org_name) {
+                unset($item[$org_name]);
+            }
+        }
+    }
+
+    public function handler($user, $options = array())
+    {
+        $this->Log = ClassRegistry::init('Log');
+        $this->Org = ClassRegistry::init('Organisation');
+        $this->Event = ClassRegistry::init('Event');
+        $orgs = $this->Org->find('all', array( 'conditions' => array('Organisation.local' => 1)));
+        $current_month = date('n');
+        $current_year = date('Y');
+        $limit = 6; // months
+        if(!empty($options['months'])) {
+            $limit = (int) ($options['months']);
+        }
+        $offset = 0;
+        $ghost_orgs = array(); // track orgs without any contribution
+        // We start by putting all orgs_id in there:
+        foreach($orgs as $org) {
+            // We check for blocklisted orgs
+            if(!empty($options['blocklist_orgs']) && in_array($org['Organisation']['name'], $options['blocklist_orgs'])) {
+                unset($orgs[$offset]);
+            } else {
+                $ghost_orgs[$org['Organisation']['name']] = true;
+            }
+            $offset++;
+        }
+        $data = array();
+        $data['data'] = array();
+        for ($i=0; $i < $limit; $i++) {
+            $target_month = $current_month - $i;
+            $target_year = $current_year;
+            if ($target_month < 1) {
+                $target_month += 12;
+                $target_year -= 1;
+            }
+            $item = array();
+            $item ['date'] = $target_year.'-'.$target_month.'-01';
+            foreach($orgs as $org) {
+                    $count = $this->org_events_count($user, $org, $target_month, $target_year);
+                    if($options['logarithmic'] === "true" || $options['logarithmic'] === "1") {
+                        $item[$org['Organisation']['name']] = (int) round(log($count, 1.1));  // taking the logarithmic view
+                    } else if(empty($options['logarithmic']) || $options['logarithmic'] === "true" || $options['logarithmic'] === "1"){
+                        $item[$org['Organisation']['name']] = $count;
+                    }
+                    // if a positive score is detected at least once it's enough to be
+                    // considered for the graph
+                    if($count > 0) {
+                        unset($ghost_orgs[$org['Organisation']['name']]);
+                    }
+                }
+            $data['data'][] = $item;
+        }
+        $this->filter_ghost_orgs($data, $ghost_orgs);
+        return $data;
+    }
+}

From 6d7d2cbb453aec8c21e4e5af1971067abbab7957 Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Tue, 4 Jul 2023 14:59:59 +0200
Subject: [PATCH 07/96] chg: [misp-warninglists] updated

---
 app/files/warninglists | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/warninglists b/app/files/warninglists
index 911aafb91..1a94fcd66 160000
--- a/app/files/warninglists
+++ b/app/files/warninglists
@@ -1 +1 @@
-Subproject commit 911aafb91a38a68bbf6f5474c06e77a039469c93
+Subproject commit 1a94fcd666bbf7eb505d4fbbc47ef6170c375706

From 1e99d7022f1d7802f2cd98dbbada56e66ec935e3 Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Tue, 4 Jul 2023 15:01:27 +0200
Subject: [PATCH 08/96] chg: [misp-objects] updated

---
 app/files/misp-objects | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/misp-objects b/app/files/misp-objects
index 2ca2667d7..da801ab14 160000
--- a/app/files/misp-objects
+++ b/app/files/misp-objects
@@ -1 +1 @@
-Subproject commit 2ca2667d7668067f906e9601e0c97a79d4c7ccf1
+Subproject commit da801ab146fb622a6447c8d2922a95b6049bb70a

From 2962ecbe824acc5f23fc6bcfd6a44b0b37b8077d Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Tue, 4 Jul 2023 15:02:30 +0200
Subject: [PATCH 09/96] chg: [misp-galaxy] updated

---
 app/files/misp-galaxy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy
index 734d57edf..7028860c0 160000
--- a/app/files/misp-galaxy
+++ b/app/files/misp-galaxy
@@ -1 +1 @@
-Subproject commit 734d57edf5e76900cd0c8d5d48d6f5910e29b84e
+Subproject commit 7028860c0aa8c471324008d3dc651b7ea9e07c0a

From 297f0f73a652369a33bffb75e817a1ebcc652698 Mon Sep 17 00:00:00 2001
From: Sura De Silva <sura.silva@outlook.com>
Date: Fri, 7 Jul 2023 12:30:03 +1000
Subject: [PATCH 10/96] fix: localisation workflow typo

---
 app/View/Workflows/editor.ctp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/View/Workflows/editor.ctp b/app/View/Workflows/editor.ctp
index d5aec5529..566bacee0 100644
--- a/app/View/Workflows/editor.ctp
+++ b/app/View/Workflows/editor.ctp
@@ -184,7 +184,7 @@ $debugEnabled = !empty($selectedWorkflow['Workflow']['debug_enabled']);
                             <a href="#"><i class="fa-fw <?= $this->FontAwesome->getClass('file-import') ?>"></i> <?= __('Import blueprint') ?></a>
                             <ul class="dropdown-menu pull-right">
                                 <?php if (empty($workflowBlueprints)) : ?>
-                                    <li><a href="#"><?= _('No workflow blueprints saved') ?></a></li>
+                                    <li><a href="#"><?= __('No workflow blueprints saved') ?></a></li>
                                 <?php endif; ?>
                                 <?php foreach ($workflowBlueprints as $workflowBlueprint) : ?>
                                     <li>
@@ -202,7 +202,7 @@ $debugEnabled = !empty($selectedWorkflow['Workflow']['debug_enabled']);
                             <a href="#"><i class="fa-fw <?= $this->FontAwesome->getClass('edit') ?>"></i> <?= __('Edit existing blueprint') ?></a>
                             <ul class="dropdown-menu pull-right disabled">
                                 <?php if (empty($workflowBlueprints)) : ?>
-                                    <li><a href="#"><?= _('No workflow blueprints saved') ?></a></li>
+                                    <li><a href="#"><?= __('No workflow blueprints saved') ?></a></li>
                                 <?php endif; ?>
                                 <?php foreach ($workflowBlueprints as $workflowBlueprint) : ?>
                                     <li class="control-edit-bp-blocks">

From b86456031bab9f67953e1a9120557ecec2388cbc Mon Sep 17 00:00:00 2001
From: 417190e5c48babc7 <417190e5c48babc7@proton.me>
Date: Mon, 10 Jul 2023 08:50:10 +0300
Subject: [PATCH 11/96] fix: [UI] Preserve linebreaks in comments in enrichment
 results

---
 app/View/Events/resolved_misp_format.ctp | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/app/View/Events/resolved_misp_format.ctp b/app/View/Events/resolved_misp_format.ctp
index edaf27f1a..968629b8c 100644
--- a/app/View/Events/resolved_misp_format.ctp
+++ b/app/View/Events/resolved_misp_format.ctp
@@ -229,8 +229,8 @@
                 }
               ?>
             </td>
-            <td class="short">
-              <input type="text" class="ObjectComment" style="padding:0;height:20px;margin-bottom:0;" placeholder="<?php echo h($importComment); ?>"<?php if (!empty($object['comment'])) echo ' value="' . h($object['comment']) . '"';?>>
+            <td class="bitwider">
+              <textarea class="ObjectComment inline-input" cols="30" rows="6" placeholder="<?php echo h($importComment); ?>" oninput="autoresize(this)"><?php if (!empty($object['comment'])) echo h($object['comment']);?></textarea>
             </td>
             <td style="width:60px;text-align:center;">
               <select class="ObjectDistribution" style="padding:0;height:20px;margin-bottom:0;">
@@ -302,8 +302,8 @@
             <td class="short" style="width:40px;text-align:center;">
               <input type="checkbox" class="AttributeDisableCorrelation"<?php if (!empty($attribute['disable_correlation'])) echo ' checked'; ?>>
             </td>
-            <td class="short">
-              <input type="text" class="AttributeComment" style="padding:0;height:20px;margin-bottom:0;"<?php if (!empty($attribute['comment'])) echo ' value="' . h($attribute['comment']) . '"';?>>
+            <td class="bitwider">
+              <textarea class="AttributeComment inline-input" rows="1" oninput="autoresize(this)"><?php if (!empty($attribute['comment'])) echo h($attribute['comment']);?></textarea>
             </td>
             <td class="short" style="width:40px;text-align:center;">
               <select class="AttributeDistribution" style="padding:0;height:20px;margin-bottom:0;">
@@ -412,8 +412,8 @@
           <td class="short" style="width:40px;text-align:center;">
             <input type="checkbox" class="AttributeDisableCorrelation"<?php if (isset($attribute['disable_correlation']) && $attribute['disable_correlation']) echo ' checked'; ?>>
           </td>
-          <td class="short">
-            <input type="text" class="AttributeComment" style="padding:0;height:20px;margin-bottom:0;" placeholder="<?php echo h($importComment); ?>"<?php if (!empty($attribute['comment'])) echo ' value="' . h($attribute['comment']) . '"';?>>
+          <td class="bitwider">
+            <textarea class="AttributeComment inline-input" rows="1" oninput="autoresize(this)" placeholder="<?php echo h($importComment); ?>"><?php if (!empty($attribute['comment'])) echo h($attribute['comment']);?></textarea>
           </td>
           <td class="short" style="width:40px;text-align:center;">
             <select class="AttributeDistribution" style="padding:0;height:20px;margin-bottom:0;">

From 8ff6dc2ea15edbedc9e53ab4c08975c98576a630 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 10 Jul 2023 15:30:28 +0200
Subject: [PATCH 12/96] new: [forgotten password] optional feature added

---
 app/Console/Command/AdminShell.php        |  12 ++
 app/Controller/AppController.php          |  10 +-
 app/Controller/Component/ACLComponent.php |   2 +
 app/Controller/UsersController.php        | 199 +++++++++++++++-------
 app/Model/Log.php                         |   2 +
 app/Model/Server.php                      |   8 +
 app/Model/User.php                        |  87 ++++++++++
 app/View/Users/forgot.ctp                 |  18 ++
 app/View/Users/login.ctp                  |   6 +
 app/View/Users/password_reset.ctp         |  25 +++
 10 files changed, 305 insertions(+), 64 deletions(-)
 create mode 100644 app/View/Users/forgot.ctp
 create mode 100644 app/View/Users/password_reset.ctp

diff --git a/app/Console/Command/AdminShell.php b/app/Console/Command/AdminShell.php
index 803fdebd0..c7d842f1c 100644
--- a/app/Console/Command/AdminShell.php
+++ b/app/Console/Command/AdminShell.php
@@ -112,6 +112,18 @@ class AdminShell extends AppShell
         return $parser;
     }
 
+    public function jobForgot()
+    {
+        if (empty($this->args[0])) {
+            die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Forgot'] . PHP_EOL);
+        }
+
+        $email = $this->args[0];
+        $ip = empty($this->args[1]) ? null : $this->args[1];
+        $jobId = empty($this->args[2]) ? null : $this->args[2];
+        $this->User->forgot($email, $ip, $jobId);
+    }
+
     public function jobGenerateCorrelation()
     {
         if (empty($this->args[0])) {
diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php
index 38d0bd8f3..14b6b9336 100755
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@@ -222,8 +222,10 @@ class AppController extends Controller
             !$userLoggedIn &&
             (
                 $controller !== 'users' ||
-                $action !== 'register' ||
-                empty(Configure::read('Security.allow_self_registration'))
+                (
+                    ($action !== 'register' || empty(Configure::read('Security.allow_self_registration'))) &&
+                    (!in_array($action, ['forgot', 'password_reset']) || empty(Configure::read('Security.allow_password_forgotten')))
+                )
             )
         ) {
             // REST authentication
@@ -314,6 +316,10 @@ class AppController extends Controller
             if (!empty(Configure::read('Security.email_otp_enabled'))) {
                 $preAuthActions[] = 'email_otp';
             }
+            if (!empty(Configure::read('Security.allow_password_forgotten'))) {
+                $preAuthActions[] = 'forgot';
+                $preAuthActions[] = 'password_reset';
+            }
             if (!$this->_isControllerAction(['users' => $preAuthActions, 'servers' => ['cspReport']])) {
                 if ($isAjax) {
                     $response = $this->RestResponse->throwException(401, "Unauthorized");
diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php
index a5f5df130..5bc70675b 100644
--- a/app/Controller/Component/ACLComponent.php
+++ b/app/Controller/Component/ACLComponent.php
@@ -748,6 +748,7 @@ class ACLComponent extends Component
                 'downloadTerms' => array('*'),
                 'edit' => array('self_management_enabled'),
                 'email_otp' => array('*'),
+                'forgot' => array('*'),
                 'otp' => array('*'),
                 'hotp' => array('*'),
                 'totp_new' => array('*'),
@@ -760,6 +761,7 @@ class ACLComponent extends Component
                 'logout' => array('*'),
                 'logout401' => array('*'),
                 'notificationSettings' => ['*'],
+                'password_reset' => array('*'),
                 'register' => array('*'),
                 'registrations' => array(),
                 'resetAllSyncAuthKeys' => array(),
diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php
index e61b61923..c4f2b8e8e 100644
--- a/app/Controller/UsersController.php
+++ b/app/Controller/UsersController.php
@@ -30,6 +30,10 @@ class UsersController extends AppController
 
         // what pages are allowed for non-logged-in users
         $allowedActions = array('login', 'logout', 'getGpgPublicKey', 'logout401', 'otp');
+        if (!empty(Configure::read('Security.allow_password_forgotten'))) {
+            $allowedActions[] = 'forgot';
+            $allowedActions[] = 'password_reset';
+        }
         if(!empty(Configure::read('Security.email_otp_enabled'))) {
           $allowedActions[] = 'email_otp';
         }
@@ -262,6 +266,74 @@ class UsersController extends AppController
         $this->set('canFetchPgpKey', $this->__canFetchPgpKey());
     }
 
+    private function __pw_change($user, $source, &$abortPost)
+    {
+        if (!isset($this->request->data['User'])) {
+            $this->request->data = array('User' => $this->request->data);
+        }
+        if (Configure::read('Security.require_password_confirmation')) {
+            if (!empty($this->request->data['User']['current_password'])) {
+                $hashed = $this->User->verifyPassword($this->Auth->user('id'), $this->request->data['User']['current_password']);
+                if (!$hashed) {
+                    $message = __('Invalid password. Please enter your current password to continue.');
+                    if ($this->_isRest()) {
+                        return $this->RestResponse->saveFailResponse('Users', $source, false, $message, $this->response->type());
+                    }
+                    $abortPost = true;
+                    $this->Flash->error($message);
+                }
+                unset($this->request->data['User']['current_password']);
+            } else if (!$this->_isRest()) {
+                $message = __('Please enter your current password to continue.');
+                if ($this->_isRest()) {
+                    return $this->RestResponse->saveFailResponse('Users', $source, false, $message, $this->response->type());
+                }
+                $abortPost = true;
+                $this->Flash->info($message);
+            }
+        }
+        $hashed = $this->User->verifyPassword($this->Auth->user('id'), $this->request->data['User']['password']);
+        if ($hashed) {
+            $message = __('Submitted new password cannot be the same as the current one');
+            $abortPost = true;
+        }
+        if (!$abortPost) {
+            // What fields should be saved (allowed to be saved)
+            $user['User']['change_pw'] = 0;
+            $user['User']['password'] = $this->request->data['User']['password'];
+            if ($this->_isRest()) {
+                $user['User']['confirm_password'] = $this->request->data['User']['password'];
+            } else {
+                $user['User']['confirm_password'] = $this->request->data['User']['confirm_password'];
+            }
+            $temp = $user['User']['password'];
+            // Save the data
+            if ($this->User->save($user)) {
+                $message = __('Password Changed.');
+                // log as System if the reset comes from an unauthed user using password_reset tokens
+                $logUser = empty($this->Auth->user()) ? 'SYSTEM' : $this->Auth->user();
+                $this->User->extralog($logUser, $source, null, null, $user);
+                if ($this->_isRest()) {
+                    return $this->RestResponse->saveSuccessResponse('User', $source, false, $this->response->type(), $message);
+                }
+                $this->Flash->success($message);
+                $this->redirect(array('action' => 'view', $user['User']['id']));
+            } else {
+                $message = __('The password could not be updated. Make sure you meet the minimum password length / complexity requirements.');
+                if ($this->_isRest()) {
+                    return $this->RestResponse->saveFailResponse('Users', $source, false, $message, $this->response->type());
+                }
+                $this->Flash->error($message);
+            }
+        } else {
+            if ($this->_isRest()) {
+                return $this->RestResponse->saveFailResponse('Users', $source, false, $message, $this->response->type());
+            } else {
+                $this->Flash->error($message);
+            }
+        }
+    }
+
     public function change_pw()
     {
         $id = $this->Auth->user('id');
@@ -270,69 +342,8 @@ class UsersController extends AppController
             'recursive' => -1
         ));
         if ($this->request->is('post') || $this->request->is('put')) {
-            if (!isset($this->request->data['User'])) {
-                $this->request->data = array('User' => $this->request->data);
-            }
             $abortPost = false;
-            if (Configure::read('Security.require_password_confirmation')) {
-                if (!empty($this->request->data['User']['current_password'])) {
-                    $hashed = $this->User->verifyPassword($this->Auth->user('id'), $this->request->data['User']['current_password']);
-                    if (!$hashed) {
-                        $message = __('Invalid password. Please enter your current password to continue.');
-                        if ($this->_isRest()) {
-                            return $this->RestResponse->saveFailResponse('Users', 'change_pw', false, $message, $this->response->type());
-                        }
-                        $abortPost = true;
-                        $this->Flash->error($message);
-                    }
-                    unset($this->request->data['User']['current_password']);
-                } else if (!$this->_isRest()) {
-                    $message = __('Please enter your current password to continue.');
-                    if ($this->_isRest()) {
-                        return $this->RestResponse->saveFailResponse('Users', 'change_pw', false, $message, $this->response->type());
-                    }
-                    $abortPost = true;
-                    $this->Flash->info($message);
-                }
-            }
-            $hashed = $this->User->verifyPassword($this->Auth->user('id'), $this->request->data['User']['password']);
-            if ($hashed) {
-                $message = __('Submitted new password cannot be the same as the current one');
-                $abortPost = true;
-            }
-            if (!$abortPost) {
-                // What fields should be saved (allowed to be saved)
-                $user['User']['change_pw'] = 0;
-                $user['User']['password'] = $this->request->data['User']['password'];
-                if ($this->_isRest()) {
-                    $user['User']['confirm_password'] = $this->request->data['User']['password'];
-                } else {
-                    $user['User']['confirm_password'] = $this->request->data['User']['confirm_password'];
-                }
-                $temp = $user['User']['password'];
-                // Save the data
-                if ($this->User->save($user)) {
-                    $message = __('Password Changed.');
-                    $this->User->extralog($this->Auth->user(), "change_pw", null, null, $user);
-                    if ($this->_isRest()) {
-                        return $this->RestResponse->saveSuccessResponse('User', 'change_pw', false, $this->response->type(), $message);
-                    }
-                    $this->Flash->success($message);
-                    $this->redirect(array('action' => 'view', $id));
-                } else {
-                    $message = __('The password could not be updated. Make sure you meet the minimum password length / complexity requirements.');
-                    if ($this->_isRest()) {
-                        return $this->RestResponse->saveFailResponse('Users', 'change_pw', false, $message, $this->response->type());
-                    }
-                    $this->Flash->error($message);
-                }
-            } else {
-                if ($this->_isRest()) {
-                    return $this->RestResponse->saveFailResponse('Users', 'change_pw', false, $message, $this->response->type());
-                } else {
-                    $this->Flash->error($message);
-                }
-            }
+            return $this->__pw_change($user, 'change_pw', $abortPost);
         }
         if ($this->_isRest()) {
             return $this->RestResponse->describe('Users', 'change_pw', false, $this->response->type());
@@ -3080,4 +3091,68 @@ class UsersController extends AppController
         # To use this, set Plugin.CustomAuth_custom_logout to /users/logout401
         $this->response->statusCode(401);
     }
+
+    public function forgot()
+    {
+        if (empty(Configure::read('Security.allow_password_forgotten'))) {
+            $this->Flash->error(__('This feature is disabled.'));
+            $this->redirect('/');
+        }
+        if (!empty($this->Auth->user()) && !$this->_isRest()) {
+            $this->Flash->info(__('You are already logged in, no need to ask for a password reset. Log out first.'));
+            $this->redirect('/');
+        }
+        if ($this->request->is('post')) {
+            if (empty($this->request->data['User'])) {
+                $this->request->data = ['User' => $this->request->data];
+            }
+            if (empty($this->request->data['User']['email'])) {
+                throw new MethodNotAllowedException(__('No email provided, cannot generate password reset message.'));
+            }
+            $user = [
+                'id' => 0,
+                'email' => 'SYSTEM',
+                'Organisation' => [
+                    'name' => 'SYSTEM'
+                ]
+            ];
+            $this->loadModel('Log');
+            $this->Log->createLogEntry($user, 'forgot', 'User', 0, 'Password reset requested for: ' . $this->request->data['User']['email']);
+            $this->User->forgotRouter($this->request->data['User']['email'], $this->_remoteIp());
+            $message = __('Password reset request submitted. If a valid user is found, you should receive an e-mail with a temporary reset link momentarily. Please be advised that this link is only valid for 10 minutes.');
+            if ($this->_isRest()) {
+                return $this->RestResponse->saveSuccessResponse('User', 'forgot', false, $this->response->type(), $message);
+            }
+            $this->Flash->info($message);
+            $this->redirect('/');
+        }
+    }
+
+    public function password_reset($token)
+    {
+        if (empty(Configure::read('Security.allow_password_forgotten'))) {
+            $this->Flash->error(__('This feature is disabled.'));
+            $this->redirect('/');
+        }
+        $this->loadModel('Server');
+        $this->set('complexity', !empty(Configure::read('Security.password_policy_complexity')) ? Configure::read('Security.password_policy_complexity') : $this->Server->serverSettings['Security']['password_policy_complexity']['value']);
+        $this->set('length', !empty(Configure::read('Security.password_policy_length')) ? Configure::read('Security.password_policy_length') : $this->Server->serverSettings['Security']['password_policy_length']['value']);
+        if (!empty($this->Auth->user()) && !$this->_isRest()) {
+            $this->redirect('/');
+        }
+        $user = $this->User->fetchForgottenPasswordUser($token);
+        if (empty($user)) {
+            $message = __('Invalid token, or password request token already expired.');
+            if ($this->_isRest()) {
+                throw new MethodNotAllowedException($message);
+            } else {
+                $this->Flash->error($message);
+            }
+        }
+        if ($this->request->is('post') || $this->request->is('put')) {
+            $abortPost = false;
+            return $this->__pw_change(['User' => $user], 'password_reset', $abortPost);
+        }
+    }
+
 }
diff --git a/app/Model/Log.php b/app/Model/Log.php
index a2ce484cf..cd0130432 100644
--- a/app/Model/Log.php
+++ b/app/Model/Log.php
@@ -44,6 +44,7 @@ class Log extends AppModel
                     'export',
                     'fetchEvent',
                     'file_upload',
+                    'forgot',
                     'galaxy',
                     'include_formula',
                     'load_module',
@@ -51,6 +52,7 @@ class Log extends AppModel
                     'login_fail',
                     'logout',
                     'merge',
+                    'password_reset',
                     'pruneUpdateLogs',
                     'publish',
                     'publish_sightings',
diff --git a/app/Model/Server.php b/app/Model/Server.php
index 0c5d4ba59..f7167c1e4 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -6457,6 +6457,14 @@ class Server extends AppModel
                     'type' => 'boolean',
                     'null' => true
                 ),
+                'allow_password_forgotten' => array(
+                    'level' => 1,
+                    'description' => __('Enabling this setting will allow users to request automated password reset tokens via mail and initiate a reset themselves. Users with no encryption keys will not be able to use this feature.'),
+                    'value' => false,
+                    'test' => 'testBool',
+                    'type' => 'boolean',
+                    'null' => true
+                ),
                 'self_registration_message' => array(
                     'level' => 1,
                     'bigField' => true,
diff --git a/app/Model/User.php b/app/Model/User.php
index c28dde9e6..7adaccbd5 100644
--- a/app/Model/User.php
+++ b/app/Model/User.php
@@ -1229,6 +1229,15 @@ class User extends AppModel
 
     public function extralog($user, $action = null, $description = null, $fieldsResult = null, $modifiedUser = null)
     {
+        if (!is_array($user) && $user === 'SYSTEM') {
+            $user = [
+                'id' => 0,
+                'email' => 'SYSTEM',
+                'Organisation' => [
+                    'name' => 'SYSTEM'
+                ]
+            ];
+        }
         // new data
         $model = 'User';
         $modelId = $user['id'];
@@ -2007,4 +2016,82 @@ class User extends AppModel
         }
         return false;
     }
+
+    public function forgotRouter($email, $ip)
+    {
+        if (Configure::read('MISP.background_jobs')) {
+            /** @var Job $job */
+            $job = ClassRegistry::init('Job');
+            $dummyUser = [
+                'email' => 'SYSTEM',
+                'org_id' => 0,
+                'role_id' => 0
+            ];
+            $jobId = $job->createJob($dummyUser, Job::WORKER_EMAIL, 'forgot_password', $email, 'Sending...');
+
+            $args = [
+                'jobForgot',
+                $email,
+                $ip,
+                $jobId,
+            ];
+
+            $this->getBackgroundJobsTool()->enqueue(
+                BackgroundJobsTool::EMAIL_QUEUE,
+                BackgroundJobsTool::CMD_ADMIN,
+                $args,
+                true,
+                $jobId
+            );
+
+            return true;
+        } else {
+            return $this->forgot($email);
+        }
+    }
+
+    public function forgot($email, $ip, $jobId = null)
+    {
+        $user = $this->find('first', [
+            'recursive' => -1,
+            'conditions' => [
+                'User.email' => $email,
+                'User.disabled' => 0
+            ]
+        ]);
+        if (empty($user)) {
+            return false;
+        }
+        $redis = $this->setupRedis();
+        $token = RandomTool::random_str(true, 40, '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ');
+        $redis->set('misp:forgot:' . $token, $user['User']['id'], ['nx', 'ex' => 600]);
+        $baseurl = Configure::check('MISP.external_baseurl') ? Configure::read('MISP.external_baseurl') : Configure::read('MISP.baseurl');
+        $body = __(
+            "Dear MISP user,\n\nyou have requested a password reset on the MISP instane at %s. Click the link below to change your password.\n\n%s\n\nThe link above is only valid for 10 minutes, feel free to request a new one if it has expired.\n\nIf you haven't requested a password reset, feel free to ignore this. The IP used to trigger the request was: %s\n\nBest regards,\nYour MISP admin team",
+            $baseurl,
+            $baseurl . '/users/password_reset/' . $token,
+            $ip
+        );
+        $bodyNoEnc = __(
+            "Dear MISP user,\n\nyou have requested a password reset on the MISP instance at %s, however, no valid encryption key was found for your user and thus we cannot deliver your reset token. Please get in touch with your org admin / with an instance site admin to ask for a reset.\n\nThe IP used to trigger the request was: %s\n\nBest regards,\nYour MISP admin team",
+            $baseurl,
+            $ip
+        );
+        $this->sendEmail($user, $body, $bodyNoEnc, __('MISP password reset'));
+        return true;
+    }
+
+    public function fetchForgottenPasswordUser($token)
+    {
+        if (!ctype_alnum($token)) {
+            return false;
+        }
+        $redis = $this->setupRedis();
+        $userId = $redis->get('misp:forgot:' . $token);
+        if (empty($userId)) {
+            return false;
+        }
+        $user = $this->getAuthUser($userId, true);
+        return $user;
+    }
 }
diff --git a/app/View/Users/forgot.ctp b/app/View/Users/forgot.ctp
new file mode 100644
index 000000000..761881a67
--- /dev/null
+++ b/app/View/Users/forgot.ctp
@@ -0,0 +1,18 @@
+<?php
+echo $this->element('genericElements/Form/genericForm', [
+    'data' => [
+        'description' => __("If you already are enrolled on this instance, but forgot your password, you can request a new password below.") . '<br />' . __("An e-mail containing URL with an embedded token will be sent to you that you can use to reset the password within 10 minutes."),
+        'model' => 'User',
+        'title' => __('Forgotten password'),
+        'fields' => [
+            [
+                'field' => 'email',
+                'class' => 'span6'
+            ],
+        ],
+        'submit' => [
+            'action' => $this->request->params['action'],
+            'ajaxSubmit' => 'submitGenericFormInPlace();'
+        ]
+    ]
+]);
\ No newline at end of file
diff --git a/app/View/Users/login.ctp b/app/View/Users/login.ctp
index b8433df97..b1e8b015f 100644
--- a/app/View/Users/login.ctp
+++ b/app/View/Users/login.ctp
@@ -58,6 +58,12 @@
                     __('Registration will be sent to the administrators of the instance for consideration.'),
                     __('No account yet? Register now!')
                 );
+                echo empty(Configure::read('Security.allow_password_forgotten')) ? '' : sprintf(
+                    '<a href="%s/users/forgot" title="%s">%s</a>',
+                    $baseurl,
+                    __('Initiate a password reset.'),
+                    __('I have forgotten my password')
+                );
             ?>
             </div>
             <?= $this->Form->button(__('Login'), array('class' => 'btn btn-primary')); ?>
diff --git a/app/View/Users/password_reset.ctp b/app/View/Users/password_reset.ctp
new file mode 100644
index 000000000..90fc5647e
--- /dev/null
+++ b/app/View/Users/password_reset.ctp
@@ -0,0 +1,25 @@
+<div class="users form">
+    <?php echo $this->Form->create('User');?>
+    <fieldset>
+        <legend><?php echo __('Reset password'); ?></legend>
+    <?php
+        $passwordPopover = '<span class="blue bold">' . __('Minimal length') . '</span>: ' . h($length) . '<br>';
+        $passwordPopover .= '<span class="blue bold">' . __('Complexity') . '</span>: ' . h($complexity);
+        echo $this->Form->input('password', array(
+            'label' => __('New password') . ' <span id="PasswordPopover" data-content="' . h($passwordPopover) . '" class="fas fa-info-circle"></span>', 'autofocus'
+        ));
+        echo $this->Form->input('confirm_password', [
+            'type' => 'password',
+            'label' => __('Confirm new password'),
+            'div' => array('class' => 'input password required'),
+        ]);
+    ?>
+    </fieldset>
+    <div style="border-bottom: 1px solid #e5e5e5;width:100%;">&nbsp;</div>
+    <div class="clear" style="margin-top:10px;">
+    </div>
+<?php
+echo $this->Form->button(__('Submit'), array('class' => 'btn btn-primary'));
+echo $this->Form->end();
+?>
+</div>

From 3028132fcf24b5cb86da3514fced5ef24c722605 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 10 Jul 2023 15:39:28 +0200
Subject: [PATCH 13/96] fix: [login] screen small visual fix

---
 app/View/Users/login.ctp | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/app/View/Users/login.ctp b/app/View/Users/login.ctp
index b1e8b015f..077d83aa9 100644
--- a/app/View/Users/login.ctp
+++ b/app/View/Users/login.ctp
@@ -58,6 +58,9 @@
                     __('Registration will be sent to the administrators of the instance for consideration.'),
                     __('No account yet? Register now!')
                 );
+            ?>
+            <div class="clear">
+            <?php
                 echo empty(Configure::read('Security.allow_password_forgotten')) ? '' : sprintf(
                     '<a href="%s/users/forgot" title="%s">%s</a>',
                     $baseurl,

From fb15d48c3401f7362d8181638f10dbae2c1a1cb3 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 10 Jul 2023 15:46:08 +0200
Subject: [PATCH 14/96] chg: [forgotten password] reset text clarifications

- to avoid dumdum users from sharing their quasi-passwords
---
 app/Model/User.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Model/User.php b/app/Model/User.php
index 7adaccbd5..5df4e1746 100644
--- a/app/Model/User.php
+++ b/app/Model/User.php
@@ -2067,7 +2067,7 @@ class User extends AppModel
         $redis->set('misp:forgot:' . $token, $user['User']['id'], ['nx', 'ex' => 600]);
         $baseurl = Configure::check('MISP.external_baseurl') ? Configure::read('MISP.external_baseurl') : Configure::read('MISP.baseurl');
         $body = __(
-            "Dear MISP user,\n\nyou have requested a password reset on the MISP instane at %s. Click the link below to change your password.\n\n%s\n\nThe link above is only valid for 10 minutes, feel free to request a new one if it has expired.\n\nIf you haven't requested a password reset, feel free to ignore this. The IP used to trigger the request was: %s\n\nBest regards,\nYour MISP admin team",
+            "Dear MISP user,\n\nyou have requested a password reset on the MISP instance at %s. Click the link below to change your password.\n\n%s\n\nThe link above is only valid for 10 minutes, feel free to request a new one if it has expired.\n\nIf you haven't requested a password reset, reach out to your admin team and let them know that someone has attempted it in your stead.\n\nMake sure you keep the contents of this e-mail confidential, do NOT ever forward it as it contains a reset token that is equivalent of a password if acted upon. The IP used to trigger the request was: %s\n\nBest regards,\nYour MISP admin team",
             $baseurl,
             $baseurl . '/users/password_reset/' . $token,
             $ip

From 68cb56037fb1d146c7609d6b123b7e99d847b04f Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 10 Jul 2023 15:58:28 +0200
Subject: [PATCH 15/96] fix: [password reset] various issues

---
 app/Controller/UsersController.php | 7 ++++++-
 app/Model/User.php                 | 7 +++++++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php
index 91cf0d5de..2ab3f1f81 100644
--- a/app/Controller/UsersController.php
+++ b/app/Controller/UsersController.php
@@ -3151,11 +3151,16 @@ class UsersController extends AppController
                 throw new MethodNotAllowedException($message);
             } else {
                 $this->Flash->error($message);
+                $this->redirect('/');
             }
         }
         if ($this->request->is('post') || $this->request->is('put')) {
             $abortPost = false;
-            return $this->__pw_change(['User' => $user], 'password_reset', $abortPost);
+            $result = $this->__pw_change(['User' => $user], 'password_reset', $abortPost);
+            if (!$abortPost) {
+                $this->User->purgeForgetToken($token);
+            }
+            return $result;
         }
     }
 
diff --git a/app/Model/User.php b/app/Model/User.php
index 5df4e1746..dee18d411 100644
--- a/app/Model/User.php
+++ b/app/Model/User.php
@@ -2094,4 +2094,11 @@ class User extends AppModel
         $user = $this->getAuthUser($userId, true);
         return $user;
     }
+
+    public function purgeForgetToken($token)
+    {
+        $redis = $this->setupRedis();
+        $userId = $redis->del('misp:forgot:' . $token);
+        return true;
+    }
 }

From 71fdd9ac20dbb46ce65b05d46f0a8c3abc2b01da Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 10 Jul 2023 16:02:34 +0200
Subject: [PATCH 16/96] fix: [forgotten password] fixed

---
 app/Controller/UsersController.php | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php
index 2ab3f1f81..442fa07f8 100644
--- a/app/Controller/UsersController.php
+++ b/app/Controller/UsersController.php
@@ -266,7 +266,7 @@ class UsersController extends AppController
         $this->set('canFetchPgpKey', $this->__canFetchPgpKey());
     }
 
-    private function __pw_change($user, $source, &$abortPost)
+    private function __pw_change($user, $source, &$abortPost, $token = false)
     {
         if (!isset($this->request->data['User'])) {
             $this->request->data = array('User' => $this->request->data);
@@ -309,6 +309,9 @@ class UsersController extends AppController
             $temp = $user['User']['password'];
             // Save the data
             if ($this->User->save($user)) {
+                if ($token) {
+                    $this->User->purgeForgetToken($token);
+                }
                 $message = __('Password Changed.');
                 // log as System if the reset comes from an unauthed user using password_reset tokens
                 $logUser = empty($this->Auth->user()) ? 'SYSTEM' : $this->Auth->user();
@@ -3156,11 +3159,7 @@ class UsersController extends AppController
         }
         if ($this->request->is('post') || $this->request->is('put')) {
             $abortPost = false;
-            $result = $this->__pw_change(['User' => $user], 'password_reset', $abortPost);
-            if (!$abortPost) {
-                $this->User->purgeForgetToken($token);
-            }
-            return $result;
+            return $this->__pw_change(['User' => $user], 'password_reset', $abortPost);
         }
     }
 

From be28fdf53cdffb75ea21cfd220c3b168a16ec2ea Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 10 Jul 2023 16:08:07 +0200
Subject: [PATCH 17/96] fix: [pw reset] fix (pass the token for deletion)

---
 app/Controller/UsersController.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php
index 442fa07f8..e63ce7923 100644
--- a/app/Controller/UsersController.php
+++ b/app/Controller/UsersController.php
@@ -3159,7 +3159,7 @@ class UsersController extends AppController
         }
         if ($this->request->is('post') || $this->request->is('put')) {
             $abortPost = false;
-            return $this->__pw_change(['User' => $user], 'password_reset', $abortPost);
+            return $this->__pw_change(['User' => $user], 'password_reset', $abortPost, $token);
         }
     }
 

From 77f9cabce45fa659a2d2a48365063aa34a6722cb Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 10 Jul 2023 16:10:42 +0200
Subject: [PATCH 18/96] chg: [version] bump

---
 VERSION.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION.json b/VERSION.json
index 5696da9c9..7a0dab149 100644
--- a/VERSION.json
+++ b/VERSION.json
@@ -1 +1 @@
-{"major":2, "minor":4, "hotfix":172}
+{"major":2, "minor":4, "hotfix":173}

From fccbc08185a3ddb92202c2140f598b182378599a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Mon, 10 Jul 2023 16:18:19 +0200
Subject: [PATCH 19/96] chg: [PyMISP] Bump version

---
 PyMISP                           | 2 +-
 app/Controller/AppController.php | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/PyMISP b/PyMISP
index 7d1d8b6f3..ccae32ae7 160000
--- a/PyMISP
+++ b/PyMISP
@@ -1 +1 @@
-Subproject commit 7d1d8b6f38f210b28934a206f9c1470542e9da7e
+Subproject commit ccae32ae716c143bea09954e860238e193bc78c6
diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php
index df00ef05f..1142b9015 100755
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@@ -34,7 +34,7 @@ class AppController extends Controller
     public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
 
     private $__queryVersion = '152';
-    public $pyMispVersion = '2.4.172';
+    public $pyMispVersion = '2.4.173';
     public $phpmin = '7.2';
     public $phprec = '7.4';
     public $phptoonew = '8.0';

From d1910c7697610155b6e459a477bdd3d435a4a5c4 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Mon, 10 Jul 2023 11:14:16 -0400
Subject: [PATCH 20/96] chg: [workflow-modules:attach_enrichment] Enable
 selection of multiple modules and added support of module not accepting
 misp_format

---
 .../action/Module_attach_enrichment.php       | 57 ++++++++++++++++---
 1 file changed, 50 insertions(+), 7 deletions(-)

diff --git a/app/Model/WorkflowModules/action/Module_attach_enrichment.php b/app/Model/WorkflowModules/action/Module_attach_enrichment.php
index b7c6e55b7..96be52fa6 100644
--- a/app/Model/WorkflowModules/action/Module_attach_enrichment.php
+++ b/app/Model/WorkflowModules/action/Module_attach_enrichment.php
@@ -5,6 +5,7 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
 {
     public $id = 'attach-enrichment';
     public $name = 'Attach enrichment';
+    public $version = '0.2';
     public $description = 'Attach selected enrichment result to Attributes.';
     public $icon = 'asterisk';
     public $inputs = 1;
@@ -16,6 +17,7 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
     private $Module;
     private $fastLookupArrayMispFormat = [];
     private $fastLookupArrayFlattened = [];
+    private $allModulesByName = [];
 
 
     public function __construct()
@@ -23,7 +25,7 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
         parent::__construct();
         $this->Module = ClassRegistry::init('Module');
         $modules = $this->Module->getModules('Enrichment');
-        $moduleOptions = [];
+        $this->allModulesByName = Hash::combine($modules, '{n}.name', '{n}');
         if (is_array($modules)) {
             $moduleOptions = array_merge([''], Hash::combine($modules, '{n}.name', '{n}.name'));
         } else {
@@ -34,7 +36,8 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
             [
                 'id' => 'modules',
                 'label' => 'Modules',
-                'type' => 'select',
+                'type' => 'picker',
+                'multiple' => true,
                 'options' => $moduleOptions,
             ],
         ];
@@ -48,12 +51,14 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
             $errors[] = __('No enrichmnent module selected');
             return false;
         }
+        $selectedModules = array_filter($params['modules']['value'], function($module) {
+            return $module !== '';
+        });
         $rData = $roamingData->getData();
         $event_id = $rData['Event']['id'];
         $options = [
             'user' => $roamingData->getUser(),
             'event_id' => $event_id,
-            'module' => $params['modules']['value'],
             'config' => ['_' => '_'], // avoid casting empty associative array in to empty list
         ];
 
@@ -64,10 +69,21 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
         $this->_buildFastLookupForRoamingData($rData);
 
         foreach ($matchingItems as $attribute) {
-            $moduleData = $options;
-            $moduleData['attribute'] = $attribute;
-            $queryResult = $this->_queryModules($moduleData, $attribute, $rData);
-            $rData = $this->_attachEnrichmentData($attribute, $queryResult, $rData);
+            foreach ($selectedModules as $selectedModule) {
+                $moduleConfig = $this->allModulesByName[$selectedModule];
+                $moduleData = $options;
+                $moduleData['module'] = $selectedModule;
+                $moduleData['attribute'] = $attribute;
+                if (!$this->_checkIfInputSupported($attribute, $moduleConfig)) { // Queried module doesn't support the Attribute's type
+                    continue;
+                }
+                if (empty($moduleConfig['mispattributes']['format'])) { // Adapt payload if modules doesn't support the misp-format
+                    $moduleData = $this->_convertPayloadToModuleFormat($moduleData, $moduleConfig);
+                }
+                $queryResult = $this->_queryModules($moduleData, $attribute, $rData);
+                $queryResult = $this->_handleModuleResult($queryResult, $moduleConfig);
+                $rData = $this->_attachEnrichmentData($attribute, $queryResult, $rData);
+            }
         }
         $roamingData->setData($rData);
         return true;
@@ -82,6 +98,33 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
         return $result;
     }
 
+    protected function _convertPayloadToModuleFormat(array $options, array $moduleConfig): array
+    {
+        $attribute = $options['attribute'];
+        unset($options['attribute']);
+        foreach ($moduleConfig['mispattributes']['input'] as $supportedAttributeType) {
+            if ($supportedAttributeType == $attribute['type']) {
+                $options[$supportedAttributeType] = $attribute['value'];
+            }
+        }
+        return $options;
+    }
+
+    protected function _checkIfInputSupported(array $attribute, array $moduleConfig): bool
+    {
+        foreach ($moduleConfig['mispattributes']['input'] as $supportedAttributeType) {
+            if ($supportedAttributeType == $attribute['type']) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    protected function _handleModuleResult(array $queryResult, array $moduleConfig): array
+    {
+        return $queryResult;
+    }
+
     protected function _buildFastLookupForRoamingData($rData): void
     {
         foreach ($rData['Event']['Attribute'] as $i => $attribute) {

From 6f5bffcb4b062f6b8ee356c79ebed06f99b2ba6d Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 10 Jul 2023 23:32:58 +0200
Subject: [PATCH 21/96] fix: [db_schema] bumped

---
 db_schema.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/db_schema.json b/db_schema.json
index 346c849a9..1fca6ae7c 100644
--- a/db_schema.json
+++ b/db_schema.json
@@ -9549,5 +9549,5 @@
             "uuid": false
         }
     },
-    "db_version": "113"
+    "db_version": "114"
 }
\ No newline at end of file

From c45734ce5d1abf90047ea1fba10ca2d27b812e7c Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Tue, 11 Jul 2023 13:11:16 +0200
Subject: [PATCH 22/96] fix: [otp] autofocus added

---
 app/View/Elements/genericElements/Form/fieldScaffold.ctp | 3 +++
 app/View/Users/otp.ctp                                   | 1 +
 2 files changed, 4 insertions(+)

diff --git a/app/View/Elements/genericElements/Form/fieldScaffold.ctp b/app/View/Elements/genericElements/Form/fieldScaffold.ctp
index dc20a7968..bac7565ac 100644
--- a/app/View/Elements/genericElements/Form/fieldScaffold.ctp
+++ b/app/View/Elements/genericElements/Form/fieldScaffold.ctp
@@ -40,6 +40,9 @@
         } else {
             $params['class'] = '';
         }
+        if (!empty($fieldData['autofocus'])) {
+            $params['autofocus'] = 1;
+        }
         if (empty($fieldData['type']) || $fieldData['type'] !== 'checkbox' ) {
             $params['class'] .= ' form-control';
         }
diff --git a/app/View/Users/otp.ctp b/app/View/Users/otp.ctp
index eb44dd5f4..019b28373 100644
--- a/app/View/Users/otp.ctp
+++ b/app/View/Users/otp.ctp
@@ -19,6 +19,7 @@ echo $this->element('/genericElements/Form/genericForm', array(
         "label" => $label,
         "type" => "text",
         "placeholder" => __("Enter your OTP here"),
+        "autofocus" => 1
       )
     ),
     "submit" => array (

From 16a9b24c731b693bd5ef1f3332b6d912d9df5d61 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 12 Jul 2023 09:46:58 -0400
Subject: [PATCH 23/96] chg: [workflow-modules:edition] General improvements,
 fixed modified data not being reflected in rData and small refactoring

---
 .../WorkflowModules/WorkflowBaseModule.php    | 45 ++++++++++++++++++-
 .../action/Module_attach_enrichment.php       | 19 +-------
 .../action/Module_attach_warninglist.php      | 33 --------------
 .../Module_attribute_edition_operation.php    | 16 ++++---
 .../Module_attribute_ids_flag_operation.php   |  7 ++-
 .../action/Module_tag_operation.php           |  8 ++--
 6 files changed, 65 insertions(+), 63 deletions(-)

diff --git a/app/Model/WorkflowModules/WorkflowBaseModule.php b/app/Model/WorkflowModules/WorkflowBaseModule.php
index cc69cd600..771032ed5 100644
--- a/app/Model/WorkflowModules/WorkflowBaseModule.php
+++ b/app/Model/WorkflowModules/WorkflowBaseModule.php
@@ -213,11 +213,13 @@ class WorkflowBaseModule
     public function getItemsMatchingCondition($items, $value, $operator, $path)
     {
         foreach ($items as $i => $item) {
-            $subItem = $this->extractData($item, $path, $operator);
+            $subItem = $this->extractData($item, $path);
             if (in_array($operator, ['equals', 'not_equals'])) {
                 $subItem = !empty($subItem) ? $subItem[0] : $subItem;
             }
-            if (!$this->evaluateCondition($subItem, $operator, $value)) {
+            if ($operator == 'any_value_in' && !empty($subItem)) {
+                continue;
+            } else if (!$this->evaluateCondition($subItem, $operator, $value)) {
                 unset($items[$i]);
             }
         }
@@ -296,6 +298,45 @@ class WorkflowBaseLogicModule extends WorkflowBaseModule
 
 class WorkflowBaseActionModule extends WorkflowBaseModule
 {
+    protected $fastLookupArrayMispFormat = [];
+    protected $fastLookupArrayFlattened = [];
+
+    public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
+    {
+        $rData = $roamingData->getData();
+        $this->_buildFastLookupForRoamingData($rData);
+        return true;
+    }
+
+    protected function _buildFastLookupForRoamingData($rData): void
+    {
+        foreach ($rData['Event']['Attribute'] as $i => $attribute) {
+            $this->fastLookupArrayMispFormat[$attribute['id']] = $i;
+        }
+        foreach ($rData['Event']['Object'] as $j => $object) {
+            foreach ($object['Attribute'] as $i => $attribute) {
+                $this->fastLookupArrayMispFormat[$attribute['id']] = [$j, $i];
+            }
+        }
+        foreach ($rData['Event']['_AttributeFlattened'] as $i => $attribute) {
+            $this->fastLookupArrayFlattened[$attribute['id']] = $i;
+        }
+    }
+
+    protected function _overrideAttribute(array $oldAttribute, array $newAttribute, array $rData): array
+    {
+        $attributeID = $oldAttribute['id'];
+        $rData['Event']['_AttributeFlattened'][$this->fastLookupArrayFlattened[$attributeID]] = $newAttribute;
+        if (is_array($this->fastLookupArrayMispFormat[$attributeID])) {
+            $objectID = $this->fastLookupArrayMispFormat[$attributeID][0];
+            $attributeID = $this->fastLookupArrayMispFormat[$attributeID][1];
+            $rData['Event']['Object'][$objectID]['Attribute'][$attributeID] = $newAttribute;
+        } else {
+            $attributeID = $this->fastLookupArrayMispFormat[$attributeID];
+            $rData['Event']['Attribute'][$attributeID] = $newAttribute;
+        }
+        return $rData;
+    }
 }
 
 class WorkflowFilteringLogicModule extends WorkflowBaseLogicModule
diff --git a/app/Model/WorkflowModules/action/Module_attach_enrichment.php b/app/Model/WorkflowModules/action/Module_attach_enrichment.php
index 96be52fa6..cc7347129 100644
--- a/app/Model/WorkflowModules/action/Module_attach_enrichment.php
+++ b/app/Model/WorkflowModules/action/Module_attach_enrichment.php
@@ -15,8 +15,6 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
     public $params = [];
 
     private $Module;
-    private $fastLookupArrayMispFormat = [];
-    private $fastLookupArrayFlattened = [];
     private $allModulesByName = [];
 
 
@@ -50,6 +48,8 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
         if (empty($params['modules']['value'])) {
             $errors[] = __('No enrichmnent module selected');
             return false;
+        } else if (is_string($params['modules']['value'])) {
+            $params['modules']['value'] = [$params['modules']['value']];
         }
         $selectedModules = array_filter($params['modules']['value'], function($module) {
             return $module !== '';
@@ -125,21 +125,6 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
         return $queryResult;
     }
 
-    protected function _buildFastLookupForRoamingData($rData): void
-    {
-        foreach ($rData['Event']['Attribute'] as $i => $attribute) {
-            $this->fastLookupArrayMispFormat[$attribute['id']] = $i;
-        }
-        foreach ($rData['Event']['Object'] as $j => $object) {
-            foreach ($object['Attribute'] as $i => $attribute) {
-                $this->fastLookupArrayMispFormat[$attribute['id']] = [$j, $i];
-            }
-        }
-        foreach ($rData['Event']['_AttributeFlattened'] as $i => $attribute) {
-            $this->fastLookupArrayFlattened[$attribute['id']] = $i;
-        }
-    }
-
     protected function _attachEnrichmentData(array $attribute, array $queryResult, array $rData): array
     {
         $attributeID = $attribute['id'];
diff --git a/app/Model/WorkflowModules/action/Module_attach_warninglist.php b/app/Model/WorkflowModules/action/Module_attach_warninglist.php
index 9204907d2..11e335007 100644
--- a/app/Model/WorkflowModules/action/Module_attach_warninglist.php
+++ b/app/Model/WorkflowModules/action/Module_attach_warninglist.php
@@ -17,8 +17,6 @@ class Module_attach_warninglist extends WorkflowBaseActionModule
     /** @var Warninglist */
     private $Warninglist;
     private $warninglists;
-    private $fastLookupArrayMispFormat = [];
-    private $fastLookupArrayFlattened = [];
 
 
     public function __construct()
@@ -54,7 +52,6 @@ class Module_attach_warninglist extends WorkflowBaseActionModule
         if ($matchingItems === false) {
             return true;
         }
-        $this->_buildFastLookupForRoamingData($rData);
 
         $warninglists = [];
         if ($params['warninglists']['value'] == 'ALL') {
@@ -84,34 +81,4 @@ class Module_attach_warninglist extends WorkflowBaseActionModule
         $roamingData->setData($rData);
         return true;
     }
-
-    protected function _buildFastLookupForRoamingData($rData): void
-    {
-        foreach ($rData['Event']['Attribute'] as $i => $attribute) {
-            $this->fastLookupArrayMispFormat[$attribute['id']] = $i;
-        }
-        foreach ($rData['Event']['Object'] as $j => $object) {
-            foreach ($object['Attribute'] as $i => $attribute) {
-                $this->fastLookupArrayMispFormat[$attribute['id']] = [$j, $i];
-            }
-        }
-        foreach ($rData['Event']['_AttributeFlattened'] as $i => $attribute) {
-            $this->fastLookupArrayFlattened[$attribute['id']] = $i;
-        }
-    }
-
-    protected function _overrideAttribute(array $oldAttribute, array $newAttribute, array $rData): array
-    {
-        $attributeID = $oldAttribute['id'];
-        $rData['Event']['_AttributeFlattened'][$this->fastLookupArrayFlattened[$attributeID]] = $newAttribute;
-        if (is_array($this->fastLookupArrayMispFormat[$attributeID])) {
-            $objectID = $this->fastLookupArrayMispFormat[$attributeID][0];
-            $attributeID = $this->fastLookupArrayMispFormat[$attributeID][1];
-            $rData['Event']['Object'][$objectID]['Attribute'][$attributeID] = $newAttribute;
-        } else {
-            $attributeID = $this->fastLookupArrayMispFormat[$attributeID];
-            $rData['Event']['Attribute'][$attributeID] = $newAttribute;
-        }
-        return $rData;
-    }
 }
diff --git a/app/Model/WorkflowModules/action/Module_attribute_edition_operation.php b/app/Model/WorkflowModules/action/Module_attribute_edition_operation.php
index c06ded738..7389e0ac5 100644
--- a/app/Model/WorkflowModules/action/Module_attribute_edition_operation.php
+++ b/app/Model/WorkflowModules/action/Module_attribute_edition_operation.php
@@ -36,15 +36,21 @@ class Module_attribute_edition_operation extends WorkflowBaseActionModule
         return $attribute;
     }
 
-    protected function __saveAttribute(array $attributes, array $rData, array $params, array $user): bool
+    protected function __saveAttributes(array $attributes, array $rData, array $params, array $user): array
     {
         $success = false;
         foreach ($attributes as $attribute) {
-            $attribute = $this->_editAttribute($attribute, $rData, $params);
-            unset($attribute['timestamp']);
-            $saveSuccess = $this->Attribute->editAttribute($attribute, $rData, $user, $attribute['object_id']);
+            $newAttribute = $this->_editAttribute($attribute, $rData, $params);
+            unset($newAttribute['timestamp']);
+            $saveSuccess = $this->Attribute->editAttribute($newAttribute, $rData, $user, $newAttribute['object_id']);
+            if ($saveSuccess) {
+                $rData = $this->_overrideAttribute($attribute, $newAttribute, $rData);
+            }
             $success = $success || !empty($saveSuccess);
         }
-        return $success;
+        return [
+            'success' => $success,
+            'updated_rData' => $rData,
+        ];
     }
 }
diff --git a/app/Model/WorkflowModules/action/Module_attribute_ids_flag_operation.php b/app/Model/WorkflowModules/action/Module_attribute_ids_flag_operation.php
index b728f7932..640f927fa 100644
--- a/app/Model/WorkflowModules/action/Module_attribute_ids_flag_operation.php
+++ b/app/Model/WorkflowModules/action/Module_attribute_ids_flag_operation.php
@@ -45,8 +45,11 @@ class Module_attribute_ids_flag_operation extends Module_attribute_edition_opera
         if ($matchingItems === false) {
             return true;
         }
-        $result = $this->__saveAttribute($matchingItems, $rData, $params, $user);
-        return $result;
+        $result = $this->__saveAttributes($matchingItems, $rData, $params, $user);
+        $success = $result['success'];
+        $updatedRData = $result['updated_rData'];
+        $roamingData->setData($updatedRData);
+        return $success;
     }
 
     protected function _editAttribute(array $attribute, array $rData, array $params): array
diff --git a/app/Model/WorkflowModules/action/Module_tag_operation.php b/app/Model/WorkflowModules/action/Module_tag_operation.php
index 76add7559..76830ee2d 100644
--- a/app/Model/WorkflowModules/action/Module_tag_operation.php
+++ b/app/Model/WorkflowModules/action/Module_tag_operation.php
@@ -130,7 +130,7 @@ class Module_tag_operation extends WorkflowBaseActionModule
         return $result;
     }
 
-    private function __addTagsToAttributes(array $attributes, array $options, array $user): bool
+    protected function __addTagsToAttributes(array $attributes, array $options, array $user): bool
     {
         $success = false;
         foreach ($attributes as $attribute) {
@@ -140,7 +140,7 @@ class Module_tag_operation extends WorkflowBaseActionModule
         return $success;
     }
     
-    private function __removeTagsFromAttributes(array $attributes, array $options): bool
+    protected function __removeTagsFromAttributes(array $attributes, array $options): bool
     {
         $success = false;
         foreach ($attributes as $attribute) {
@@ -150,12 +150,12 @@ class Module_tag_operation extends WorkflowBaseActionModule
         return $success;
     }
 
-    private function __addTagsToEvent(array $event, array $options, array $user): bool
+    protected function __addTagsToEvent(array $event, array $options, array $user): bool
     {
         return !empty($this->Event->attachTagsToEventAndTouch($event['Event']['id'], $options, $user));
     }
 
-    private function __removeTagsFromEvent(array $event, array $options): bool
+    protected function __removeTagsFromEvent(array $event, array $options): bool
     {
         return !empty($this->Event->detachTagsFromEventAndTouch($event['Event']['id'], $options));
     }

From 082873550b85a14c14eda5d5ec569ebadb7e1c04 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 12 Jul 2023 09:49:03 -0400
Subject: [PATCH 24/96] chg: [workflow-modules:generic_filter] Added support of
 operator `any_value_in`

---
 app/Model/WorkflowModules/logic/Module_generic_filter_data.php | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/Model/WorkflowModules/logic/Module_generic_filter_data.php b/app/Model/WorkflowModules/logic/Module_generic_filter_data.php
index 18d504416..67bedc745 100644
--- a/app/Model/WorkflowModules/logic/Module_generic_filter_data.php
+++ b/app/Model/WorkflowModules/logic/Module_generic_filter_data.php
@@ -16,6 +16,7 @@ class Module_generic_filter_data extends WorkflowFilteringLogicModule
         'not_in' => 'Not in',
         'equals' => 'Equals',
         'not_equals' => 'Not equals',
+        'any_value_in' => 'Any value in',
     ];
 
     public function __construct()

From e5edbd6369fb6a4f4bb58446a31f6a81eb2ebe7e Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 12 Jul 2023 09:49:48 -0400
Subject: [PATCH 25/96] new: [workflow-modules:attribute_comment_operation]
 Added new module to set the comment of an Attribute

---
 .../Module_attribute_comment_operation.php    | 61 +++++++++++++++++++
 1 file changed, 61 insertions(+)
 create mode 100644 app/Model/WorkflowModules/action/Module_attribute_comment_operation.php

diff --git a/app/Model/WorkflowModules/action/Module_attribute_comment_operation.php b/app/Model/WorkflowModules/action/Module_attribute_comment_operation.php
new file mode 100644
index 000000000..2ec0f7ee9
--- /dev/null
+++ b/app/Model/WorkflowModules/action/Module_attribute_comment_operation.php
@@ -0,0 +1,61 @@
+<?php
+include_once APP . 'Model/WorkflowModules/action/Module_attribute_edition_operation.php';
+
+class Module_attribute_comment_operation extends Module_attribute_edition_operation
+{
+    public $version = '0.1';
+    public $blocking = false;
+    public $id = 'Module_attribute_comment_operation';
+    public $name = 'Attribute comment operation';
+    public $description = 'Set the Attribute\'s comment to the selected value';
+    public $icon = 'edit';
+    public $inputs = 1;
+    public $outputs = 1;
+    public $support_filters = true;
+    public $expect_misp_core_format = true;
+    public $params = [];
+
+
+    public function __construct()
+    {
+        parent::__construct();
+        $this->params = [
+            [
+                'id' => 'comment',
+                'label' => __('Comment'),
+                'type' => 'textarea',
+                'placeholder' => 'Comment to be set',
+            ],
+        ];
+    }
+
+    public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
+    {
+        parent::exec($node, $roamingData, $errors);
+        $params = $this->getParamsWithValues($node);
+
+        $rData = $roamingData->getData();
+        $user = $roamingData->getUser();
+
+        $matchingItems = $this->getMatchingItemsForAttributes($node, $rData);
+        if ($matchingItems === false) {
+            return true;
+        }
+        $result = $this->__saveAttributes($matchingItems, $rData, $params, $user);
+        $success = $result['success'];
+        $updatedRData = $result['updated_rData'];
+        $roamingData->setData($updatedRData);
+        return $success;
+    }
+
+    protected function _editAttribute(array $attribute, array $rData, array $params): array
+    {
+        $currentRData = $rData;
+        $currentRData['__currentAttribute'] = $attribute;
+        $renderedComment = $this->render_jinja_template($params['comment']['value'], $currentRData);
+        if ($attribute['comment'] !== $params['comment']['value']) {
+            $attribute['comment'] = $renderedComment;
+        }
+        return $attribute;
+    }
+}

From a702456c8a5709724b1c412cb8a687a27f79a2a2 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 12 Jul 2023 09:50:41 -0400
Subject: [PATCH 26/96] new: [workflow-modules:assign_country_from_enrichment]
 Added module that tags using the country galaxy based on the provided hash
 path

---
 .../Module_assign_country_from_enrichment.php | 143 ++++++++++++++++++
 1 file changed, 143 insertions(+)
 create mode 100644 app/Lib/WorkflowModules/action/Module_assign_country_from_enrichment.php

diff --git a/app/Lib/WorkflowModules/action/Module_assign_country_from_enrichment.php b/app/Lib/WorkflowModules/action/Module_assign_country_from_enrichment.php
new file mode 100644
index 000000000..30bea303e
--- /dev/null
+++ b/app/Lib/WorkflowModules/action/Module_assign_country_from_enrichment.php
@@ -0,0 +1,143 @@
+<?php
+include_once APP . 'Model/WorkflowModules/action/Module_tag_operation.php';
+
+class Module_assign_country_from_enrichment extends Module_tag_operation
+{
+    public $version = '0.1';
+    public $blocking = false;
+    public $id = 'assign_country';
+    public $name = 'Assign country';
+    public $description = 'Add or remove country Galaxy Cluster based on provided data';
+    public $icon = 'globe';
+    public $inputs = 1;
+    public $outputs = 1;
+    public $support_filters = true;
+    public $expect_misp_core_format = true;
+    public $params = [];
+
+    private $Galaxy;
+    private $countryClusters;
+
+
+    public function __construct()
+    {
+        parent::__construct();
+        $this->params = [
+            [
+                'id' => 'scope',
+                'label' => __('Scope'),
+                'type' => 'select',
+                'options' => [
+                    'event' => __('Event'),
+                    'attribute' => __('Attributes'),
+                ],
+                'default' => 'event',
+            ],
+            [
+                'id' => 'hash_path',
+                'label' => 'Country Hash path',
+                'type' => 'input',
+                'placeholder' => 'enrichment.{n}.{n}.values.0',
+                'default' => 'enrichment.{n}.{n}.values.0'
+            ],
+            [
+                'id' => 'locality',
+                'label' => __('Tag Locality'),
+                'type' => 'select',
+                'options' => [
+                    'local' => __('Local'),
+                    'global' => __('Global'),
+                ],
+                'default' => 'local',
+            ],
+            [
+                'id' => 'galaxy_name',
+                'label' => __('Galaxy Name'),
+                'type' => 'select',
+                'options' => [
+                    'country' => 'country',
+                ],
+                'placeholder' => __('Pick a galaxy name'),
+            ],
+            [
+                'id' => 'relationship_type',
+                'label' => __('Relationship Type'),
+                'type' => 'input',
+                'display_on' => [
+                    'action' => 'add',
+                ],
+            ],
+        ];
+
+        $this->Galaxy = ClassRegistry::init('Galaxy');
+        $this->countryClusters = $this->_fetchCountryGalaxyClusters();
+    }
+
+    public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
+    {
+        $params = $this->getParamsWithValues($node);
+
+        $rData = $roamingData->getData();
+        $user = $roamingData->getUser();
+
+        $countryExtractionPath = $params['hash_path']['value'];
+        if ($this->filtersEnabled($node)) {
+            $filters = $this->getFilters($node);
+            $extracted = $this->extractData($rData, $filters['selector']);
+            if ($extracted === false) {
+                return false;
+            }
+            $matchingItems = $this->getItemsMatchingCondition($extracted, $filters['value'], $filters['operator'], $filters['path']);
+        } else {
+            $matchingItems = $rData;
+            if ($params['scope']['value'] == 'attribute') {
+                $matchingItems = Hash::extract($matchingItems, 'Event._AttributeFlattened.{n}');
+                if (substr($countryExtractionPath, 0, 4) !== '{n}.') {
+                    $countryExtractionPath = '{n}.' . $countryExtractionPath;
+                }
+            }
+        }
+
+        $result = false;
+        $extractedCountries = Hash::extract($matchingItems, $countryExtractionPath);
+        $guessedCountryTags = $this->guessTagFromPath($extractedCountries);
+        $options = [
+            'tags' => $guessedCountryTags,
+            'local' => ($params['locality']['value'] == 'local' ? true : false),
+            'relationship_type' => $params['relationship_type']['value'],
+        ];
+        if ($params['scope']['value'] == 'event') {
+            $result = $this->__addTagsToEvent($matchingItems, $options, $user);
+        } else {
+            $result = $this->__addTagsToAttributes($matchingItems, $options, $user);
+        }
+        return $result;
+    }
+
+    protected function _fetchCountryGalaxyClusters(): array
+    {
+        $clusters = $this->Galaxy->find('first', [
+            'recursive' => -1,
+            'conditions' => [
+                'name' => 'Country',
+            ],
+            'contain' => [
+                'GalaxyCluster' => ['fields' => ['id', 'uuid', 'value', 'tag_name']],
+            ],
+        ]);
+        return $clusters['GalaxyCluster'];
+    }
+
+    protected function guessTagFromPath($countries)
+    {
+        $matchingTags = [];
+        foreach ($countries as $country) {
+            foreach ($this->countryClusters as $countryCluster) {
+                if (strtolower($countryCluster['value']) == strtolower($country)) {
+                    $matchingTags[] = $countryCluster['tag_name'];
+                }
+            }
+        }
+        return $matchingTags;
+    }
+}

From 3f7f6482fab5d775e6ca060edb32ae5fd18d3faa Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <sami.mokaddem@circl.lu>
Date: Wed, 12 Jul 2023 11:04:23 -0400
Subject: [PATCH 27/96] fix: [workflow-modules:generic_filter] Set a default
 filtering label for new dragged modules

---
 app/Model/WorkflowModules/logic/Module_generic_filter_data.php | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/Model/WorkflowModules/logic/Module_generic_filter_data.php b/app/Model/WorkflowModules/logic/Module_generic_filter_data.php
index 67bedc745..b2c7a97bc 100644
--- a/app/Model/WorkflowModules/logic/Module_generic_filter_data.php
+++ b/app/Model/WorkflowModules/logic/Module_generic_filter_data.php
@@ -28,6 +28,7 @@ class Module_generic_filter_data extends WorkflowFilteringLogicModule
                 'label' => __('Filtering Label'),
                 'type' => 'select',
                 'options' => $this->_genFilteringLabels(),
+                'default' => array_keys($this->_genFilteringLabels())[0],
             ],
             [
                 'id' => 'selector',

From 3b33ba8adb474222a07de2d200d90e6fe970f41b Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <sami.mokaddem@circl.lu>
Date: Wed, 12 Jul 2023 11:05:22 -0400
Subject: [PATCH 28/96] chg: [workflow:editor] Added support of new option
 `picker_create_new`

---
 .../js/workflows-editor/workflows-editor.js   | 32 +++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/app/webroot/js/workflows-editor/workflows-editor.js b/app/webroot/js/workflows-editor/workflows-editor.js
index f2131c702..124cebb01 100644
--- a/app/webroot/js/workflows-editor/workflows-editor.js
+++ b/app/webroot/js/workflows-editor/workflows-editor.js
@@ -1350,6 +1350,7 @@ function afterNodeDrawCallback() {
     var $nodes = $drawflow.find('.drawflow-node')
     $nodes.find('.start-chosen').chosen()
     toggleDisplayOnFields()
+    enablePickerCreateNewOptions()
 }
 
 function afterModalShowCallback() {
@@ -1402,6 +1403,30 @@ function toggleDisplayOnFields() {
     })
 }
 
+function enablePickerCreateNewOptions() {
+    var $nodes = $drawflow.find('.drawflow-node')
+    $nodes.find('.start-chosen[picker_create_new="1"]').each(function () {
+        var $select = $(this)
+        var $input = $select.parent().find('.chosen-search-input')
+
+        $input.on('keydown', function(evt) {
+            if (evt.which == 13) { // <ENTER>
+                var newVal = $input.val()
+                var optionExists = $select.find('option').filter(function () {
+                    return $(this).val() == newVal
+                }).length > 0
+                if (!optionExists) {
+                    var $newOption = $('<option>')
+                        .val(newVal)
+                        .text(newVal)
+                    $select.append($newOption);
+                    $select.trigger('chosen:updated');
+                }
+            }
+        })
+    })
+}
+
 function genParameterWarning(options) {
     var text = '', text_short = ''
     if (options.is_invalid) {
@@ -1445,6 +1470,13 @@ function genSelect(options, forNode = true) {
         $select.prop('multiple', true)
         $select.attr('size', 1)
     }
+    if (options.picker_create_new) {
+        $select.attr('picker_create_new', 1)
+        if (!options.options) {
+            options.options = []
+            $select.prop('multiple', true)
+        }
+    }
     var selectOptions = options.options
     if (!Array.isArray(selectOptions)) {
         selectOptions = Object.keys(options.options).map((k) => { return { name: options.options[k], value: k } })

From 774040bdf0b08a7b8966e72c76189e28b601b781 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <sami.mokaddem@circl.lu>
Date: Wed, 12 Jul 2023 11:13:50 -0400
Subject: [PATCH 29/96] chg: [workflow:editor] Added support of list of value
 for display_on parameter

---
 app/webroot/js/workflows-editor/workflows-editor.js | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/webroot/js/workflows-editor/workflows-editor.js b/app/webroot/js/workflows-editor/workflows-editor.js
index 124cebb01..51c37595b 100644
--- a/app/webroot/js/workflows-editor/workflows-editor.js
+++ b/app/webroot/js/workflows-editor/workflows-editor.js
@@ -1390,7 +1390,9 @@ function toggleDisplayOnFields() {
             Object.keys(node_param_config.display_on).forEach(function(target_param_id) {
                 var target_param_values = node_param_config.display_on[target_param_id]
                 var node_param_value = node.data.indexed_params[target_param_id]
-                if (target_param_values == node_param_value) {
+                if (Array.isArray(target_param_values) && target_param_values.includes(node_param_value)) {
+                    showContainer = true
+                } else if (target_param_values == node_param_value) {
                     showContainer = true
                 }
             });

From 44e031d6e6e9bc1ed8e54c291b5483fc3586574e Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <sami.mokaddem@circl.lu>
Date: Wed, 12 Jul 2023 14:03:14 -0400
Subject: [PATCH 30/96] chg: [workflow:matchingItems] Improved any_value and
 any_value_from for IF::Generic and Filter::Generic

---
 .../WorkflowModules/WorkflowBaseModule.php    | 16 +++++++++-----
 .../logic/Module_generic_filter_data.php      | 19 ++++++++++++++--
 .../logic/Module_generic_if.php               | 22 ++++++++++++++++++-
 .../js/workflows-editor/workflows-editor.js   | 10 ++++++++-
 4 files changed, 57 insertions(+), 10 deletions(-)

diff --git a/app/Model/WorkflowModules/WorkflowBaseModule.php b/app/Model/WorkflowModules/WorkflowBaseModule.php
index 771032ed5..e183d7971 100644
--- a/app/Model/WorkflowModules/WorkflowBaseModule.php
+++ b/app/Model/WorkflowModules/WorkflowBaseModule.php
@@ -217,7 +217,7 @@ class WorkflowBaseModule
             if (in_array($operator, ['equals', 'not_equals'])) {
                 $subItem = !empty($subItem) ? $subItem[0] : $subItem;
             }
-            if ($operator == 'any_value_in' && !empty($subItem)) {
+            if ($operator == 'any_value' && !empty($subItem)) {
                 continue;
             } else if (!$this->evaluateCondition($subItem, $operator, $value)) {
                 unset($items[$i]);
@@ -310,12 +310,16 @@ class WorkflowBaseActionModule extends WorkflowBaseModule
 
     protected function _buildFastLookupForRoamingData($rData): void
     {
-        foreach ($rData['Event']['Attribute'] as $i => $attribute) {
-            $this->fastLookupArrayMispFormat[$attribute['id']] = $i;
+        if (!empty($rData['Event']['Attribute'])) {
+            foreach ($rData['Event']['Attribute'] as $i => $attribute) {
+                $this->fastLookupArrayMispFormat[$attribute['id']] = $i;
+            }
         }
-        foreach ($rData['Event']['Object'] as $j => $object) {
-            foreach ($object['Attribute'] as $i => $attribute) {
-                $this->fastLookupArrayMispFormat[$attribute['id']] = [$j, $i];
+        if (!empty($rData['Event']['Object'])) {
+            foreach ($rData['Event']['Object'] as $j => $object) {
+                foreach ($object['Attribute'] as $i => $attribute) {
+                    $this->fastLookupArrayMispFormat[$attribute['id']] = [$j, $i];
+                }
             }
         }
         foreach ($rData['Event']['_AttributeFlattened'] as $i => $attribute) {
diff --git a/app/Model/WorkflowModules/logic/Module_generic_filter_data.php b/app/Model/WorkflowModules/logic/Module_generic_filter_data.php
index b2c7a97bc..6befd80e3 100644
--- a/app/Model/WorkflowModules/logic/Module_generic_filter_data.php
+++ b/app/Model/WorkflowModules/logic/Module_generic_filter_data.php
@@ -16,7 +16,8 @@ class Module_generic_filter_data extends WorkflowFilteringLogicModule
         'not_in' => 'Not in',
         'equals' => 'Equals',
         'not_equals' => 'Not equals',
-        'any_value_in' => 'Any value in',
+        'any_value' => 'Any value',
+        'in_or' => 'Any value from',
     ];
 
     public function __construct()
@@ -41,6 +42,18 @@ class Module_generic_filter_data extends WorkflowFilteringLogicModule
                 'label' => __('Value'),
                 'type' => 'input',
                 'placeholder' => 'tlp:red',
+                'display_on' => [
+                    'operator' => ['in', 'not_in', 'equals', 'not_equals',],
+                ],
+            ],
+            [
+                'id' => 'value_list',
+                'label' => __('Value list'),
+                'type' => 'input',
+                'placeholder' => '[\'ip-src\', \'ip-dst\']',
+                'display_on' => [
+                    'operator' => 'in_or',
+                ],
             ],
             [
                 'id' => 'operator',
@@ -66,6 +79,8 @@ class Module_generic_filter_data extends WorkflowFilteringLogicModule
         $path = $params['hash_path']['value'];
         $operator = $params['operator']['value'];
         $value = $params['value']['value'];
+        $value_list = $params['value_list']['value'];
+        $valueToEvaluate = $operator == 'in_or' ? $value_list : $value;
         $filteringLabel = $params['filtering-label']['value'];
         $rData = $roamingData->getData();
 
@@ -77,7 +92,7 @@ class Module_generic_filter_data extends WorkflowFilteringLogicModule
             'selector' => $selector,
             'path' => $path,
             'operator' => $operator,
-            'value' => $value,
+            'value' => $valueToEvaluate,
         ];
 
         $roamingData->setData($newRData);
diff --git a/app/Model/WorkflowModules/logic/Module_generic_if.php b/app/Model/WorkflowModules/logic/Module_generic_if.php
index 244efef53..728488844 100644
--- a/app/Model/WorkflowModules/logic/Module_generic_if.php
+++ b/app/Model/WorkflowModules/logic/Module_generic_if.php
@@ -17,6 +17,8 @@ class Module_generic_if extends WorkflowBaseLogicModule
         'not_in' => 'Not in',
         'equals' => 'Equals',
         'not_equals' => 'Not equals',
+        'any_value' => 'Any value',
+        'in_or' => 'Any value from',
     ];
 
     public function __construct()
@@ -28,6 +30,19 @@ class Module_generic_if extends WorkflowBaseLogicModule
                 'label' => 'Value',
                 'type' => 'input',
                 'placeholder' => 'tlp:red',
+                'display_on' => [
+                    'operator' => ['in', 'not_in', 'equals', 'not_equals',],
+                ],
+            ],
+            [
+                'id' => 'value_list',
+                'label' => __('Value list'),
+                'type' => 'picker',
+                'picker_create_new' => true,
+                'placeholder' => '[\'ip-src\', \'ip-dst\']',
+                'display_on' => [
+                    'operator' => 'in_or',
+                ],
             ],
             [
                 'id' => 'operator',
@@ -52,6 +67,8 @@ class Module_generic_if extends WorkflowBaseLogicModule
         $path = $params['hash_path']['value'];
         $operator = $params['operator']['value'];
         $value = $params['value']['value'];
+        $value_list = $params['value_list']['value'];
+        $valueToEvaluate = $operator == 'in_or' ? $value_list : $value;
         $data = $roamingData->getData();
         $extracted = [];
         if ($operator == 'equals' || $operator == 'not_equals') {
@@ -59,7 +76,10 @@ class Module_generic_if extends WorkflowBaseLogicModule
         } else {
             $extracted = Hash::extract($data, $path);
         }
-        $eval = $this->evaluateCondition($extracted, $operator, $value);
+        if ($operator == 'any_value' && !empty($extracted)) {
+            return true;
+        }
+        $eval = $this->evaluateCondition($extracted, $operator, $valueToEvaluate);
         return !empty($eval);
     }
 }
diff --git a/app/webroot/js/workflows-editor/workflows-editor.js b/app/webroot/js/workflows-editor/workflows-editor.js
index 51c37595b..646e1ce8b 100644
--- a/app/webroot/js/workflows-editor/workflows-editor.js
+++ b/app/webroot/js/workflows-editor/workflows-editor.js
@@ -1473,10 +1473,18 @@ function genSelect(options, forNode = true) {
         $select.attr('size', 1)
     }
     if (options.picker_create_new) {
+        options.multiple = true
         $select.attr('picker_create_new', 1)
+        $select.prop('multiple', true)
         if (!options.options) {
             options.options = []
-            $select.prop('multiple', true)
+        }
+        if (options.value) {
+            if (Array.isArray(options.value)) {
+                options.options = options.options.concat(options.value)
+            } else {
+                options.options.push(options.value)
+            }
         }
     }
     var selectOptions = options.options

From 003814ea2f52bf8c38a7fc81765c0c16cbf35852 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <sami.mokaddem@circl.lu>
Date: Wed, 12 Jul 2023 18:20:23 -0400
Subject: [PATCH 31/96] chg: [workflow-module:generic_filter] Added support of
 picker_create_new in value list

---
 app/Model/WorkflowModules/logic/Module_generic_filter_data.php | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/Model/WorkflowModules/logic/Module_generic_filter_data.php b/app/Model/WorkflowModules/logic/Module_generic_filter_data.php
index 6befd80e3..907ecafe7 100644
--- a/app/Model/WorkflowModules/logic/Module_generic_filter_data.php
+++ b/app/Model/WorkflowModules/logic/Module_generic_filter_data.php
@@ -49,7 +49,8 @@ class Module_generic_filter_data extends WorkflowFilteringLogicModule
             [
                 'id' => 'value_list',
                 'label' => __('Value list'),
-                'type' => 'input',
+                'type' => 'picker',
+                'picker_create_new' => true,
                 'placeholder' => '[\'ip-src\', \'ip-dst\']',
                 'display_on' => [
                     'operator' => 'in_or',

From 038fc9533febd52b903002f65f02303004778848 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 12 Jul 2023 18:21:41 -0400
Subject: [PATCH 32/96] chg: [workflow-modules:assign_country_from_enrichment]
 Moved from app/Lib to app/Model

---
 .../action/Module_assign_country_from_enrichment.php              | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename app/{Lib => Model}/WorkflowModules/action/Module_assign_country_from_enrichment.php (100%)

diff --git a/app/Lib/WorkflowModules/action/Module_assign_country_from_enrichment.php b/app/Model/WorkflowModules/action/Module_assign_country_from_enrichment.php
similarity index 100%
rename from app/Lib/WorkflowModules/action/Module_assign_country_from_enrichment.php
rename to app/Model/WorkflowModules/action/Module_assign_country_from_enrichment.php

From 81906bfded1aa93e9b8868c95e2f9150e8129c47 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <sami.mokaddem@circl.lu>
Date: Wed, 12 Jul 2023 18:22:45 -0400
Subject: [PATCH 33/96] new: [workflow-modules:tag_replacement] Added tag
 generic module and support for TLP and PAP

---
 .../action/Module_tag_replacement_generic.php | 162 ++++++++++++++++++
 .../action/Module_tag_replacement_pap.php     |  33 ++++
 .../action/Module_tag_replacement_tlp.php     |  33 ++++
 3 files changed, 228 insertions(+)
 create mode 100644 app/Model/WorkflowModules/action/Module_tag_replacement_generic.php
 create mode 100644 app/Model/WorkflowModules/action/Module_tag_replacement_pap.php
 create mode 100644 app/Model/WorkflowModules/action/Module_tag_replacement_tlp.php

diff --git a/app/Model/WorkflowModules/action/Module_tag_replacement_generic.php b/app/Model/WorkflowModules/action/Module_tag_replacement_generic.php
new file mode 100644
index 000000000..2d33c45a8
--- /dev/null
+++ b/app/Model/WorkflowModules/action/Module_tag_replacement_generic.php
@@ -0,0 +1,162 @@
+<?php
+include_once APP . 'Model/WorkflowModules/action/Module_tag_operation.php';
+
+class Module_tag_replacement_generic extends Module_tag_operation
+{
+    public $version = '0.1';
+    public $blocking = false;
+    public $id = 'tag_replacement_generic';
+    public $name = 'Tag Replacement Generic';
+    public $description = 'Toggle or remove the IDS flag on selected attributes.';
+    public $icon = 'tags';
+    public $inputs = 1;
+    public $outputs = 1;
+    public $support_filters = true;
+    public $expect_misp_core_format = true;
+    public $params = [];
+
+    public $searchRegex = '';
+    public $substitutionTemplate = '';  // Format the template using CakeText::insert where variables are surround by `{{ var }}`
+
+
+    public function __construct()
+    {
+        parent::__construct();
+        $this->params = [
+            [
+                'id' => 'scope',
+                'label' => __('Scope'),
+                'type' => 'select',
+                'options' => [
+                    'event' => __('Event'),
+                    'attribute' => __('Attributes'),
+                ],
+                'default' => 'event',
+            ],
+            [
+                'id' => 'remove_substituted',
+                'label' => 'Removed substituted tag',
+                'type' => 'select',
+                'default' => '1',
+                'options' => [
+                    'no' => __('No'),
+                    'yes' => __('Yes'),
+                ],
+            ],
+            [
+                'id' => 'locality',
+                'label' => __('Tag Locality'),
+                'type' => 'select',
+                'options' => [
+                    'local' => __('Local'),
+                    'global' => __('Global'),
+                ],
+                'default' => 'local',
+            ],
+            [
+                'id' => 'relationship_type',
+                'label' => __('Relationship Type'),
+                'type' => 'input',
+                'display_on' => [
+                    'action' => 'add',
+                ],
+            ],
+        ];
+    }
+
+    public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
+    {
+        $params = $this->getParamsWithValues($node);
+
+        $rData = $roamingData->getData();
+        $user = $roamingData->getUser();
+
+        if ($this->filtersEnabled($node)) {
+            $filters = $this->getFilters($node);
+            $extracted = $this->extractData($rData, $filters['selector']);
+            if ($extracted === false) {
+                return false;
+            }
+            $matchingItems = $this->getItemsMatchingCondition($extracted, $filters['value'], $filters['operator'], $filters['path']);
+        } else {
+            $matchingItems = $rData;
+            if ($params['scope']['value'] == 'attribute') {
+                $matchingItems = Hash::extract($matchingItems, 'Event._AttributeFlattened.{n}');
+            }
+        }
+
+        if (empty($matchingItems)) {
+            return true;
+        }
+
+        $result = false;
+        $optionsRemove = [];
+        $optionsAdd = [
+            'local' => $params['locality']['value'] == 'local' ? true : false,
+            'relationship_type' => $params['relationship_type']['value'],
+        ];
+        if ($params['scope']['value'] == 'event') {
+            $extractedTags = Hash::extract($matchingItems['Event']['Tag'], '{n}.name');
+            $options = $this->getReplacementOptions($extractedTags);
+            $optionsRemove['tags'] = $options['remove'];
+            $optionsAdd['tags'] = $options['add'];
+            if ($params['remove_substituted']['value'] == 'yes' && !empty($optionsRemove['tags'])) {
+                $result = $this->__removeTagsFromEvent($matchingItems, $optionsRemove);
+            }
+            if (!empty($optionsAdd['tags'])) {
+                $result = $this->__addTagsToEvent($matchingItems, $optionsAdd, $user);
+            }
+        } else {
+            foreach ($matchingItems as $attribute) {
+                $extractedTags = Hash::extract($attribute['Tag'], '{n}.name');
+                $options = $this->getReplacementOptions($extractedTags);
+                $optionsRemove['tags'] = $options['remove'];
+                $optionsAdd['tags'] = $options['add'];
+                if ($params['remove_substituted']['value'] == 'yes' && !empty($optionsRemove['tags'])) {
+                    $result = $this->__removeTagsFromAttributes([$attribute], $optionsRemove);
+                }
+                if (!empty($optionsAdd['tags'])) {
+                    $result = $this->__addTagsToAttributes([$attribute], $optionsAdd, $user);
+                }
+            }
+        }
+         return $result;
+    }
+
+    protected function isAMatch($matches): bool
+    {
+        return !empty($matches);
+    }
+
+    protected function searchAndReplaceTag(array $tags): array
+    {
+        $toReturn = [];
+        foreach ($tags as $tag) {
+            $matches = [];
+            preg_match($this->searchRegex, $tag, $matches);
+            if ($this->isAMatch($matches)) {
+                $toReturn[] = [
+                    'matched' => $tag,
+                    'substitution' => $this->formatSubstitution($matches),
+                ];
+            }
+        }
+        return $toReturn;
+    }
+
+    protected function getReplacementOptions(array $extractedTags)
+    {
+        $substitutionResult = $this->searchAndReplaceTag($extractedTags);
+        $tagsToRemove = Hash::extract($substitutionResult, '{n}.matched');
+        $tagsToAdd = Hash::extract($substitutionResult, '{n}.substitution');
+        return [
+            'remove' => $tagsToRemove,
+            'add' => $tagsToAdd,
+        ];
+    }
+
+    protected function formatSubstitution($matches)
+    {
+        return CakeText::insert($this->substitutionTemplate, $matches, ['before' => '{{', 'after' => '}}']);
+    }
+}
diff --git a/app/Model/WorkflowModules/action/Module_tag_replacement_pap.php b/app/Model/WorkflowModules/action/Module_tag_replacement_pap.php
new file mode 100644
index 000000000..c7531a888
--- /dev/null
+++ b/app/Model/WorkflowModules/action/Module_tag_replacement_pap.php
@@ -0,0 +1,33 @@
+<?php
+include_once APP . 'Model/WorkflowModules/action/Module_tag_replacement_generic.php';
+
+class Module_tag_replacement_pap extends Module_tag_replacement_generic
+{
+    public $version = '0.1';
+    public $blocking = false;
+    public $id = 'tag_replacement_pap';
+    public $name = 'Tag Replacement - PAP';
+    public $description = 'Attach a tag (or substitue) a tag by another for the PAP taxonomy';
+    public $icon = 'tags';
+    public $inputs = 1;
+    public $outputs = 1;
+    public $support_filters = true;
+    public $expect_misp_core_format = true;
+    public $params = [];
+
+    public $searchRegex = '/(\w*pap\w*|permissible\W+actions\W+protocol)([:\s\-=]{1,2})"?(?P<predicate>white|clear|green|amber|red)"?/i';
+
+
+    protected function isAMatch($matches): bool
+    {
+        $namespace = 'pap';
+        $predicates = ['white', 'clear', 'green', 'amber', 'red'];
+        return strtolower($matches[1]) == $namespace && in_array(strtolower($matches['predicate']), $predicates);
+    }
+
+    protected function formatSubstitution($matches)
+    {
+        return sprintf('PAP:%s', strtoupper($matches['predicate']));
+    }
+
+}
diff --git a/app/Model/WorkflowModules/action/Module_tag_replacement_tlp.php b/app/Model/WorkflowModules/action/Module_tag_replacement_tlp.php
new file mode 100644
index 000000000..1776f93ba
--- /dev/null
+++ b/app/Model/WorkflowModules/action/Module_tag_replacement_tlp.php
@@ -0,0 +1,33 @@
+<?php
+include_once APP . 'Model/WorkflowModules/action/Module_tag_replacement_generic.php';
+
+class Module_tag_replacement_tlp extends Module_tag_replacement_generic
+{
+    public $version = '0.1';
+    public $blocking = false;
+    public $id = 'tag_replacement_tlp';
+    public $name = 'Tag Replacement - TLP';
+    public $description = 'Attach a tag (or substitue) a tag by another for the TLP taxonomy';
+    public $icon = 'tags';
+    public $inputs = 1;
+    public $outputs = 1;
+    public $support_filters = true;
+    public $expect_misp_core_format = true;
+    public $params = [];
+
+    public $searchRegex = '/(\w*tlp\w*|traffic\W+light\W+protocol)([:\s\-=]{1,2})"?(?P<predicate>white|clear|green|amber|amber\+strict|red)"?/i';
+
+
+    protected function isAMatch($matches): bool
+    {
+        $namespace = 'tlp';
+        $predicates = ['white', 'clear', 'green', 'amber', 'amber+strict', 'red'];
+        return strtolower($matches[1]) == $namespace && in_array(strtolower($matches['predicate']), $predicates);
+    }
+
+    protected function formatSubstitution($matches)
+    {
+        return sprintf('tlp:%s', strtolower($matches['predicate']));
+    }
+
+}

From 6313cfaa5e752f0d7bd3929f636a8fe6758ae469 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 13 Jul 2023 10:46:58 -0400
Subject: [PATCH 34/96] fix: [workflow-modules:attach_enrichment] Make sure to
 include selected module config

---
 .../action/Module_attach_enrichment.php          | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/app/Model/WorkflowModules/action/Module_attach_enrichment.php b/app/Model/WorkflowModules/action/Module_attach_enrichment.php
index cc7347129..f65729a21 100644
--- a/app/Model/WorkflowModules/action/Module_attach_enrichment.php
+++ b/app/Model/WorkflowModules/action/Module_attach_enrichment.php
@@ -5,7 +5,7 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
 {
     public $id = 'attach-enrichment';
     public $name = 'Attach enrichment';
-    public $version = '0.2';
+    public $version = '0.3';
     public $description = 'Attach selected enrichment result to Attributes.';
     public $icon = 'asterisk';
     public $inputs = 1;
@@ -59,7 +59,6 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
         $options = [
             'user' => $roamingData->getUser(),
             'event_id' => $event_id,
-            'config' => ['_' => '_'], // avoid casting empty associative array in to empty list
         ];
 
         $matchingItems = $this->getMatchingItemsForAttributes($node, $rData);
@@ -72,6 +71,7 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
             foreach ($selectedModules as $selectedModule) {
                 $moduleConfig = $this->allModulesByName[$selectedModule];
                 $moduleData = $options;
+                $moduleData['config'] = $this->getModuleOptions($moduleConfig);
                 $moduleData['module'] = $selectedModule;
                 $moduleData['attribute'] = $attribute;
                 if (!$this->_checkIfInputSupported($attribute, $moduleConfig)) { // Queried module doesn't support the Attribute's type
@@ -139,4 +139,16 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
         }
         return $rData;
     }
+
+    protected function getModuleOptions(array $moduleConfig): array
+    {
+        $type = 'Enrichment';
+        $options = [];
+        if (isset($moduleConfig['meta']['config'])) {
+            foreach ($moduleConfig['meta']['config'] as $conf) {
+                $options[$conf] = Configure::read('Plugin.' . $type . '_' . $moduleConfig['name'] . '_' . $conf);
+            }
+        }
+        return !empty($options) ? $options : ['_' => '_']; // avoid casting empty associative array in to empty list
+    }
 }

From 9682609aed77b08cf6923dc759c3d0356bd7c790 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 13 Jul 2023 10:47:30 -0400
Subject: [PATCH 35/96] fix: [workflow-modules:tag_replacement_generic] Provide
 tag locality for deletion

---
 .../WorkflowModules/action/Module_tag_replacement_generic.php | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/Model/WorkflowModules/action/Module_tag_replacement_generic.php b/app/Model/WorkflowModules/action/Module_tag_replacement_generic.php
index 2d33c45a8..da514d542 100644
--- a/app/Model/WorkflowModules/action/Module_tag_replacement_generic.php
+++ b/app/Model/WorkflowModules/action/Module_tag_replacement_generic.php
@@ -90,7 +90,9 @@ class Module_tag_replacement_generic extends Module_tag_operation
         }
 
         $result = false;
-        $optionsRemove = [];
+        $optionsRemove = [
+            'local' => $params['locality']['value'] == 'local' ? true : false,
+        ];
         $optionsAdd = [
             'local' => $params['locality']['value'] == 'local' ? true : false,
             'relationship_type' => $params['relationship_type']['value'],

From f5e2b0f70659f0dec9ce08da43b513497d03dfe1 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 13 Jul 2023 14:30:53 -0400
Subject: [PATCH 36/96] chg: [workflow:auditLog] Removed auditlog behavior as
 it's blocking large workflows to be saved

---
 app/Model/Workflow.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Model/Workflow.php b/app/Model/Workflow.php
index 858c85996..ad809e140 100644
--- a/app/Model/Workflow.php
+++ b/app/Model/Workflow.php
@@ -13,7 +13,7 @@ class Workflow extends AppModel
     public $recursive = -1;
 
     public $actsAs = [
-        'AuditLog',
+        // 'AuditLog',
         'Containable',
         'SysLogLogable.SysLogLogable' => [
             'roleModel' => 'Role',

From 64ef573bfd38388f8e5f89fea9340f3fa8924391 Mon Sep 17 00:00:00 2001
From: Andras Iklody <andras.iklody@gmail.com>
Date: Thu, 13 Jul 2023 20:35:22 +0200
Subject: [PATCH 37/96] fix: [acl] sighting restsearch should be open to all,
 fixes #9116

---
 app/Controller/Component/ACLComponent.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php
index 5bc70675b..693703a66 100644
--- a/app/Controller/Component/ACLComponent.php
+++ b/app/Controller/Component/ACLComponent.php
@@ -613,7 +613,7 @@ class ACLComponent extends Component
             ),
             'sightings' => array(
                 'add' => array('perm_sighting'),
-                'restSearch' => array('perm_sighting'),
+                'restSearch' => array('*'),
                 'advanced' => array('perm_sighting'),
                 'delete' => ['AND' => ['perm_sighting', 'perm_modify_org']],
                 'index' => array('*'),

From 0a9a40c7d8eaff241f59ee4b64a32040afaf76a7 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <ahoj@jakubonderka.cz>
Date: Fri, 14 Jul 2023 10:35:01 +0200
Subject: [PATCH 38/96] NATO MISP only for cyber defense

According to NATO MISP terms of use, NATO MISP is open only for cyber defense related governmental entities, not to all governmental entities.
---
 app/files/community-metadata/defaults.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/community-metadata/defaults.json b/app/files/community-metadata/defaults.json
index 6e800b672..c664f602a 100644
--- a/app/files/community-metadata/defaults.json
+++ b/app/files/community-metadata/defaults.json
@@ -168,7 +168,7 @@
     "uuid": "53a830e6-8d07-4dd8-98d5-18cc4e66fc8c",
     "org_uuid": "",
     "org_name": "",
-    "description": "The ultimate goal of the project is to develop a NATO capability, available to all NATO nations, through which nations commit to sharing their information. This community is only open for official government entities, sponsored by their nation representative in the NATO Multinational MISP Steering Board.",
+    "description": "The ultimate goal of the project is to develop a NATO capability, available to all NATO nations, through which nations commit to sharing their information. This community is only open for official government cyber defense related entities, sponsored by their nation representative in the NATO Multinational MISP Steering Board.",
     "url": "https://misp.ncirc.nato.int",
     "sector": "Governmental",
     "nationality": "International",

From 8ca484a88245ac461c203b13c0143b3378344c4a Mon Sep 17 00:00:00 2001
From: Jeroen Pinoy <jeroen.pinoy@fortitude.ninja>
Date: Sat, 15 Jul 2023 18:05:09 +0200
Subject: [PATCH 39/96] fix: [UI] use acl to determine whether to show "audit
 logs" and "search logs" buttons (#9192)

fix #8949
---
 app/View/Elements/global_menu.ctp | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/View/Elements/global_menu.ctp b/app/View/Elements/global_menu.ctp
index ef2e49c4e..0771b6529 100755
--- a/app/View/Elements/global_menu.ctp
+++ b/app/View/Elements/global_menu.ctp
@@ -465,7 +465,7 @@
                     array(
                         'text' => __('Audit Logs'),
                         'url' => $baseurl . '/admin/audit_logs/index',
-                        'requirement' => Configure::read('MISP.log_new_audit') && $isAdmin,
+                        'requirement' => Configure::read('MISP.log_new_audit') && $this->Acl->canAccess('auditLogs', 'admin_index'),
                     ),
                     array(
                         'text' => __('Access Logs'),
@@ -475,7 +475,7 @@
                     array(
                         'text' => __('Search Logs'),
                         'url' => $baseurl . '/admin/logs/search',
-                        'requirement' => $isAdmin
+                        'requirement' => $this->Acl->canAccess('logs', 'admin_search')
                     )
                 )
             ),

From 71f074c0317759e8de3e75c651db71cd31aedaaf Mon Sep 17 00:00:00 2001
From: Jeroen Pinoy <jeroen.pinoy@fortitude.ninja>
Date: Sat, 15 Jul 2023 18:13:00 +0200
Subject: [PATCH 40/96] fix: [authkeys] allow admin read-only key to access
 audit logs (#9191)

fix #9190
---
 app/Model/AuthKey.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/Model/AuthKey.php b/app/Model/AuthKey.php
index b177180ff..fc2c2256e 100644
--- a/app/Model/AuthKey.php
+++ b/app/Model/AuthKey.php
@@ -219,9 +219,9 @@ class AuthKey extends AppModel
         $user['authkey_read_only'] = (bool)$authkey['AuthKey']['read_only'];
 
         if ($authkey['AuthKey']['read_only']) {
-            // Disable all permissions, keep just `perm_auth` unchanged
+            // Disable all permissions, keep just `perm_auth` and `perm_audit` unchanged
             foreach ($user['Role'] as $key => &$value) {
-                if (substr($key, 0, 5) === 'perm_' && $key !== 'perm_auth') {
+                if (substr($key, 0, 5) === 'perm_' && $key !== 'perm_auth' && $key !== 'perm_audit') {
                     $value = 0;
                 }
             }

From e8689044e35af86a4aefe74e4846943e0644d161 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 17 Jul 2023 00:50:04 +0200
Subject: [PATCH 41/96] fix: [restsearch] searching for ipv6s fails due to
 compression not being applied, fixes #9042

- compress ipv6 addresses in value searches to match the behaviour of automatic compression on saving attributes
---
 app/Model/Event.php | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/app/Model/Event.php b/app/Model/Event.php
index 08da95364..724b39ea2 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -2860,10 +2860,35 @@ class Event extends AppModel
         return $conditions;
     }
 
+    /**
+     * @param string $value
+     * @return string
+     */
+    private static function compressIpv6($value)
+    {
+        if (strpos($value, ':') && $converted = inet_pton($value)) {
+            return inet_ntop($converted);
+        }
+        return $value;
+    }
+
     public function set_filter_value(&$params, $conditions, $options)
     {
         if (!empty($params['value'])) {
             $params[$options['filter']] = $this->convert_filters($params['value']);
+            foreach (['OR', 'AND', 'NOT'] as $operand) {
+                if (!empty($params[$options['filter']][$operand])) {
+                    foreach ($params[$options['filter']][$operand] as $k => $v) {
+                        if ($operand === 'NOT') {
+                            $v = mb_substr($v, 1);
+                        }
+                        if (filter_var($v, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) {
+                            $v = $this->compressIpv6($v);
+                        }
+                        $params[$options['filter']][$operand][$k] = $operand === 'NOT' ? '!' . $v : $v;
+                    }
+                }
+            }
             $conditions = $this->generic_add_filter($conditions, $params['value'], ['Attribute.value1', 'Attribute.value2']);
         }
 

From 65bb3ef6eb0d0dc70cd2e1d6ae4af1e85352b21e Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 17 Jul 2023 01:15:34 +0200
Subject: [PATCH 42/96] fix: [security] otp reset otp_secret on logout

- changing users within the same session can otherwise lead to the creation of the same otp seed for multiple users
---
 app/Controller/UsersController.php | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php
index e63ce7923..dba396bfd 100644
--- a/app/Controller/UsersController.php
+++ b/app/Controller/UsersController.php
@@ -1383,6 +1383,7 @@ class UsersController extends AppController
         unset($user['User']['password']);
         $user['User']['action'] = 'logout';
         $this->User->save($user['User'], true, array('id'));
+        $this->Session->write('otp_secret', null);
         $this->redirect($this->Auth->logout());
     }
 

From 9e15f352d005b3078ac48c06ff7714e8661edb9a Mon Sep 17 00:00:00 2001
From: Mathieu Rollet <matletix@gmail.com>
Date: Mon, 17 Jul 2023 16:14:49 +0200
Subject: [PATCH 43/96] explicitly set loginAction with baseurl

---
 app/Controller/AppController.php | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php
index 1142b9015..225e9c0a2 100755
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@@ -111,6 +111,7 @@ class AppController extends Controller
 
         $this->_setupBaseurl();
         $this->Auth->loginRedirect = $this->baseurl . '/users/routeafterlogin';
+        $this->Auth->loginAction = $this->baseurl . '/users/login';
 
         $customLogout = Configure::read('Plugin.CustomAuth_custom_logout');
         $this->Auth->logoutRedirect = $customLogout ?: ($this->baseurl . '/users/login');

From ec07c3eb7c9d172fb733760bbc3cbf5cb51380e3 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 19 Jul 2023 12:26:51 +0200
Subject: [PATCH 44/96] fix: [attributes:validation] Allow telfhash to be
 either 70 or 72 chars long

---
 app/Lib/Tools/AttributeValidationTool.php | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/app/Lib/Tools/AttributeValidationTool.php b/app/Lib/Tools/AttributeValidationTool.php
index 20a734936..752fbe1e0 100644
--- a/app/Lib/Tools/AttributeValidationTool.php
+++ b/app/Lib/Tools/AttributeValidationTool.php
@@ -224,7 +224,6 @@ class AttributeValidationTool
         switch ($type) {
             case 'md5':
             case 'imphash':
-            case 'telfhash':
             case 'sha1':
             case 'sha224':
             case 'sha256':
@@ -255,6 +254,11 @@ class AttributeValidationTool
                     return true;
                 }
                 return __('Checksum has an invalid length or format (expected: at least 35 hexadecimal characters, optionally starting with t1 instead of hexadecimal characters). Please double check the value or select type "other".');
+            case 'telfhash':
+                if (self::isTelfhashValid($value)) {
+                    return true;
+                }
+                return __('Checksum has an invalid length or format (expected: %s or %s hexadecimal characters). Please double check the value or select type "other".', 70, 72);
             case 'pehash':
                 if (self::isHashValid('pehash', $value)) {
                     return true;
@@ -635,6 +639,15 @@ class AttributeValidationTool
         return strlen($value) > 35 && ctype_xdigit($value);
     }
 
+    /**
+     * @param string $value
+     * @return bool
+     */
+    private static function isTelfhashValid($value)
+    {
+        return strlen($value) == 70 || strlen($value) == 72;
+    }
+
 
     /**
      * @param string $type

From 38701d055cf853a64a461770762ce9e5fcf74b16 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 20 Jul 2023 15:49:30 +0200
Subject: [PATCH 45/96] fix: [workflow:editor] Prevent Run workflow popover
 after closing

---
 .../js/workflows-editor/workflows-editor.js   | 66 ++++++++++---------
 1 file changed, 34 insertions(+), 32 deletions(-)

diff --git a/app/webroot/js/workflows-editor/workflows-editor.js b/app/webroot/js/workflows-editor/workflows-editor.js
index 646e1ce8b..9846a6450 100644
--- a/app/webroot/js/workflows-editor/workflows-editor.js
+++ b/app/webroot/js/workflows-editor/workflows-editor.js
@@ -1161,41 +1161,43 @@ function runWorkflow() {
         container: 'body',
         template: '<div class="popover" role="tooltip"><div class="arrow"></div><h3 class="popover-title"></h3><div class="popover-content"><div class="data-content"></div></div></div>'
     }
-    $runWorkflowButton
-        .popover(popoverOptions)
-        .on('shown.bs.popover', function () {
-            var $popover = $runWorkflowButton.data('popover').tip()
-            $popover.find('button').click(function() {
-                var url = baseurl + "/workflows/executeWorkflow/" + workflow.Workflow.id
-                fetchFormDataAjax(url, function (formHTML) {
-                    $('body').append($('<div id="temp" style="display: none"/>').html(formHTML))
-                    var $tmpForm = $('#temp form')
-                    var formUrl = $tmpForm.attr('action')
-                    data = $popover.find('textarea').val()
-                    $tmpForm.find('[name="data[Workflow][data]"]').val(data)
-                    
-                    $.ajax({
-                        data: $tmpForm.serialize(),
-                        beforeSend: function() {
-                            $popover.find('pre').empty()
-                            $popover.find('button i').removeClass('hidden')
-                        },
-                        success: function (data) {
-                            $popover.find('pre').text(data)
-                        },
-                        error: xhrFailCallback,
-                        complete: function () {
-                            $('#temp').remove();
-                            $popover.find('button i').addClass('hidden')
-                        },
-                        type: 'post',
-                        cache: false,
-                        url: formUrl,
+    if ($runWorkflowButton.data().popover === undefined) {
+        $runWorkflowButton
+            .popover(popoverOptions)
+            .on('shown.bs.popover', function () {
+                var $popover = $runWorkflowButton.data('popover').tip()
+                $popover.find('button').click(function() {
+                    var url = baseurl + "/workflows/executeWorkflow/" + workflow.Workflow.id
+                    fetchFormDataAjax(url, function (formHTML) {
+                        $('body').append($('<div id="temp" style="display: none"/>').html(formHTML))
+                        var $tmpForm = $('#temp form')
+                        var formUrl = $tmpForm.attr('action')
+                        data = $popover.find('textarea').val()
+                        $tmpForm.find('[name="data[Workflow][data]"]').val(data)
+
+                        $.ajax({
+                            data: $tmpForm.serialize(),
+                            beforeSend: function() {
+                                $popover.find('pre').empty()
+                                $popover.find('button i').removeClass('hidden')
+                            },
+                            success: function (data) {
+                                $popover.find('pre').text(data)
+                            },
+                            error: xhrFailCallback,
+                            complete: function () {
+                                $('#temp').remove();
+                                $popover.find('button i').addClass('hidden')
+                            },
+                            type: 'post',
+                            cache: false,
+                            url: formUrl,
+                        })
                     })
                 })
             })
-        })
-    $runWorkflowButton.popover('show')
+            .popover('show')
+    }
 }
 
 function getSelectedNodeID() {

From b2fcbba5f77ee4b2f1349b715e41a10ac539c52c Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 20 Jul 2023 15:50:12 +0200
Subject: [PATCH 46/96] new: [workflow-module] Added debug function to send
 custom request to debug endpoint

---
 app/Model/WorkflowModules/WorkflowBaseModule.php | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/app/Model/WorkflowModules/WorkflowBaseModule.php b/app/Model/WorkflowModules/WorkflowBaseModule.php
index e183d7971..23da2a3ee 100644
--- a/app/Model/WorkflowModules/WorkflowBaseModule.php
+++ b/app/Model/WorkflowModules/WorkflowBaseModule.php
@@ -23,7 +23,7 @@ class WorkflowBaseModule
     ];
     public $params = [];
 
-    private $Event;
+    private $Workflow;
 
     /** @var PubSubTool */
     private static $loadedPubSubTool;
@@ -32,6 +32,16 @@ class WorkflowBaseModule
     {
     }
 
+    public function debug(array $node, WorkflowRoamingData $roamingData, array $data=[]): void
+    {
+        if (!isset($this->Workflow)) {
+            $this->Workflow = ClassRegistry::init('Workflow');
+        }
+        $workflow = $roamingData->getWorkflow();
+        $path = sprintf('/debug/%s', $node['data']['id'] ?? '');
+        $this->Workflow->sendRequestToDebugEndpoint($workflow, $node, $path, $data);
+    }
+
     protected function mergeNodeConfigIntoParameters($node): array
     {
         $fullIndexedParams = [];

From 779fed3e6f886012a9fe40a2d739ae2039ff38c4 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 20 Jul 2023 15:51:07 +0200
Subject: [PATCH 47/96] fix: [event:attachTagsdToEventAndTouch] Make sure to
 continue adding tag in case of success

---
 app/Model/Event.php | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/app/Model/Event.php b/app/Model/Event.php
index 08da95364..4116c0ae1 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -540,7 +540,8 @@ class Event extends AppModel
                 'local' => $local,
                 'relationship_type' => $relationship,
             ];
-            $success = $success || $this->EventTag->attachTagToEvent($event_id, $tag, $nothingToChange);
+            $attachSuccess = $this->EventTag->attachTagToEvent($event_id, $tag, $nothingToChange);
+            $success = $success || $attachSuccess;
             $touchEvent = $touchEvent || !$nothingToChange;
         }
         if ($touchEvent) {
@@ -562,7 +563,8 @@ class Event extends AppModel
                 $success = $success || true;
                 continue;
             }
-            $success = $success || $this->EventTag->detachTagFromEvent($event_id, $tag_id, $local, $nothingToChange);
+            $detachSuccess = $this->EventTag->detachTagFromEvent($event_id, $tag_id, $local, $nothingToChange);
+            $success = $success || $detachSuccess;
             $touchEvent = $touchEvent || !$nothingToChange;
         }
         if ($touchEvent) {

From 64532581f70b227005008d5e5123e7de5c59029f Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 20 Jul 2023 15:51:47 +0200
Subject: [PATCH 48/96] fix: [workflow-modules] Prevent exception if no match

---
 .../WorkflowModules/action/Module_tag_replacement_pap.php      | 3 +++
 .../WorkflowModules/action/Module_tag_replacement_tlp.php      | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/app/Model/WorkflowModules/action/Module_tag_replacement_pap.php b/app/Model/WorkflowModules/action/Module_tag_replacement_pap.php
index c7531a888..f67b00b5e 100644
--- a/app/Model/WorkflowModules/action/Module_tag_replacement_pap.php
+++ b/app/Model/WorkflowModules/action/Module_tag_replacement_pap.php
@@ -22,6 +22,9 @@ class Module_tag_replacement_pap extends Module_tag_replacement_generic
     {
         $namespace = 'pap';
         $predicates = ['white', 'clear', 'green', 'amber', 'red'];
+        if (empty($matches)) {
+            return false;
+        }
         return strtolower($matches[1]) == $namespace && in_array(strtolower($matches['predicate']), $predicates);
     }
 
diff --git a/app/Model/WorkflowModules/action/Module_tag_replacement_tlp.php b/app/Model/WorkflowModules/action/Module_tag_replacement_tlp.php
index 1776f93ba..ad3d19183 100644
--- a/app/Model/WorkflowModules/action/Module_tag_replacement_tlp.php
+++ b/app/Model/WorkflowModules/action/Module_tag_replacement_tlp.php
@@ -22,6 +22,9 @@ class Module_tag_replacement_tlp extends Module_tag_replacement_generic
     {
         $namespace = 'tlp';
         $predicates = ['white', 'clear', 'green', 'amber', 'amber+strict', 'red'];
+        if (empty($matches)) {
+            return false;
+        }
         return strtolower($matches[1]) == $namespace && in_array(strtolower($matches['predicate']), $predicates);
     }
 

From d8d8b2d67208e040de958562feeb81bb2020bc5c Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 20 Jul 2023 15:52:43 +0200
Subject: [PATCH 49/96] chg: [workflow-modules:tag_replacement] Improved
 behavior and added `all` scope

---
 .../action/Module_tag_replacement_generic.php | 64 ++++++++++++-------
 1 file changed, 42 insertions(+), 22 deletions(-)

diff --git a/app/Model/WorkflowModules/action/Module_tag_replacement_generic.php b/app/Model/WorkflowModules/action/Module_tag_replacement_generic.php
index da514d542..b4508097a 100644
--- a/app/Model/WorkflowModules/action/Module_tag_replacement_generic.php
+++ b/app/Model/WorkflowModules/action/Module_tag_replacement_generic.php
@@ -30,6 +30,7 @@ class Module_tag_replacement_generic extends Module_tag_operation
                 'options' => [
                     'event' => __('Event'),
                     'attribute' => __('Attributes'),
+                    'all' => __('All'),
                 ],
                 'default' => 'event',
             ],
@@ -80,9 +81,12 @@ class Module_tag_replacement_generic extends Module_tag_operation
             $matchingItems = $this->getItemsMatchingCondition($extracted, $filters['value'], $filters['operator'], $filters['path']);
         } else {
             $matchingItems = $rData;
-            if ($params['scope']['value'] == 'attribute') {
-                $matchingItems = Hash::extract($matchingItems, 'Event._AttributeFlattened.{n}');
-            }
+        }
+
+        $matchingEvent = $matchingItems;
+        $matchingAttributes = [];
+        if ($params['scope']['value'] == 'attribute' || $params['scope']['value'] == 'all') {
+            $matchingAttributes = Hash::extract($matchingItems, 'Event._AttributeFlattened.{n}');
         }
 
         if (empty($matchingItems)) {
@@ -91,38 +95,54 @@ class Module_tag_replacement_generic extends Module_tag_operation
 
         $result = false;
         $optionsRemove = [
-            'local' => $params['locality']['value'] == 'local' ? true : false,
+            'local' => [0, 1],
         ];
         $optionsAdd = [
             'local' => $params['locality']['value'] == 'local' ? true : false,
             'relationship_type' => $params['relationship_type']['value'],
         ];
-        if ($params['scope']['value'] == 'event') {
-            $extractedTags = Hash::extract($matchingItems['Event']['Tag'], '{n}.name');
+        if ($params['scope']['value'] == 'event' || $params['scope']['value'] == 'all') {
+            $result = $this->replaceOnEvent($matchingEvent, $params, $user, $optionsRemove, $optionsAdd);
+        }
+        if ($params['scope']['value'] == 'attribute' || $params['scope']['value'] == 'all') {
+            $result = $this->replaceOnAttribute($matchingAttributes, $params, $user, $optionsRemove, $optionsAdd);
+        }
+         return $result;
+    }
+
+
+    protected function replaceOnEvent(array $matchingItems, array $params, array $user, array $optionsRemove, array $optionsAdd): bool
+    {
+        $result = true;
+        $extractedTags = Hash::extract($matchingItems['Event']['Tag'], '{n}.name');
+        $options = $this->getReplacementOptions($extractedTags);
+        $optionsRemove['tags'] = $options['remove'];
+        $optionsAdd['tags'] = $options['add'];
+        if ($params['remove_substituted']['value'] == 'yes' && !empty($optionsRemove['tags'])) {
+            $result = $this->__removeTagsFromEvent($matchingItems, $optionsRemove);
+        }
+        if (!empty($optionsAdd['tags'])) {
+            $result = $this->__addTagsToEvent($matchingItems, $optionsAdd, $user);
+        }
+        return $result;
+    }
+
+    protected function replaceOnAttribute(array $matchingItems, array $params, array $user, array $optionsRemove, array $optionsAdd): bool
+    {
+        $result = true;
+        foreach ($matchingItems as $attribute) {
+            $extractedTags = Hash::extract($attribute['Tag'], '{n}.name');
             $options = $this->getReplacementOptions($extractedTags);
             $optionsRemove['tags'] = $options['remove'];
             $optionsAdd['tags'] = $options['add'];
             if ($params['remove_substituted']['value'] == 'yes' && !empty($optionsRemove['tags'])) {
-                $result = $this->__removeTagsFromEvent($matchingItems, $optionsRemove);
+                $result = $this->__removeTagsFromAttributes([$attribute], $optionsRemove);
             }
             if (!empty($optionsAdd['tags'])) {
-                $result = $this->__addTagsToEvent($matchingItems, $optionsAdd, $user);
-            }
-        } else {
-            foreach ($matchingItems as $attribute) {
-                $extractedTags = Hash::extract($attribute['Tag'], '{n}.name');
-                $options = $this->getReplacementOptions($extractedTags);
-                $optionsRemove['tags'] = $options['remove'];
-                $optionsAdd['tags'] = $options['add'];
-                if ($params['remove_substituted']['value'] == 'yes' && !empty($optionsRemove['tags'])) {
-                    $result = $this->__removeTagsFromAttributes([$attribute], $optionsRemove);
-                }
-                if (!empty($optionsAdd['tags'])) {
-                    $result = $this->__addTagsToAttributes([$attribute], $optionsAdd, $user);
-                }
+                $result = $this->__addTagsToAttributes([$attribute], $optionsAdd, $user);
             }
         }
-         return $result;
+        return $result;
     }
 
     protected function isAMatch($matches): bool

From cc88c319aca4795269bfdf04290472bb0d251e53 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 20 Jul 2023 15:54:14 +0200
Subject: [PATCH 50/96] chg: [workflow-modules:assign_country] Improved
 behavior when dealing with scopes

---
 .../Module_assign_country_from_enrichment.php | 35 ++++++++++++-------
 1 file changed, 22 insertions(+), 13 deletions(-)

diff --git a/app/Model/WorkflowModules/action/Module_assign_country_from_enrichment.php b/app/Model/WorkflowModules/action/Module_assign_country_from_enrichment.php
index 30bea303e..888dc20f7 100644
--- a/app/Model/WorkflowModules/action/Module_assign_country_from_enrichment.php
+++ b/app/Model/WorkflowModules/action/Module_assign_country_from_enrichment.php
@@ -90,26 +90,35 @@ class Module_assign_country_from_enrichment extends Module_tag_operation
             $matchingItems = $this->getItemsMatchingCondition($extracted, $filters['value'], $filters['operator'], $filters['path']);
         } else {
             $matchingItems = $rData;
-            if ($params['scope']['value'] == 'attribute') {
-                $matchingItems = Hash::extract($matchingItems, 'Event._AttributeFlattened.{n}');
-                if (substr($countryExtractionPath, 0, 4) !== '{n}.') {
-                    $countryExtractionPath = '{n}.' . $countryExtractionPath;
-                }
+        }
+        $matchingAttributes = Hash::extract($matchingItems, 'Event._AttributeFlattened.{n}');
+        if ($params['scope']['value'] == 'event') {
+            if (substr($countryExtractionPath, 0, 4) !== '{n}.') {
+                $countryExtractionPath = '{n}.' . $countryExtractionPath;
             }
         }
 
         $result = false;
-        $extractedCountries = Hash::extract($matchingItems, $countryExtractionPath);
-        $guessedCountryTags = $this->guessTagFromPath($extractedCountries);
-        $options = [
-            'tags' => $guessedCountryTags,
-            'local' => ($params['locality']['value'] == 'local' ? true : false),
-            'relationship_type' => $params['relationship_type']['value'],
-        ];
         if ($params['scope']['value'] == 'event') {
+            $extractedCountries = array_unique(Hash::extract($matchingAttributes, $countryExtractionPath));
+            $guessedCountryTags = $this->guessTagFromPath($extractedCountries);
+            $options = [
+                'tags' => $guessedCountryTags,
+                'local' => ($params['locality']['value'] == 'local' ? true : false),
+                'relationship_type' => $params['relationship_type']['value'],
+            ];
             $result = $this->__addTagsToEvent($matchingItems, $options, $user);
         } else {
-            $result = $this->__addTagsToAttributes($matchingItems, $options, $user);
+            foreach ($matchingAttributes as $attribute) {
+                $extractedCountries = Hash::extract($attribute, $countryExtractionPath);
+                $guessedCountryTags = $this->guessTagFromPath($extractedCountries);
+                $options = [
+                    'tags' => $guessedCountryTags,
+                    'local' => ($params['locality']['value'] == 'local' ? true : false),
+                    'relationship_type' => $params['relationship_type']['value'],
+                ];
+                $result = $this->__addTagsToAttributes([$attribute], $options, $user);
+            }
         }
         return $result;
     }

From 79f961de9d033cc05b174d4aa51c630e58cb087e Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 20 Jul 2023 16:44:57 +0200
Subject: [PATCH 51/96] fix: [workflow:editor] Avoid duplicating labels when
 path merges on one node

---
 app/Model/Workflow.php | 21 +++++++++++++++++----
 1 file changed, 17 insertions(+), 4 deletions(-)

diff --git a/app/Model/Workflow.php b/app/Model/Workflow.php
index ad809e140..0c6837122 100644
--- a/app/Model/Workflow.php
+++ b/app/Model/Workflow.php
@@ -1321,19 +1321,32 @@ class Workflow extends AppModel
             $node = $graphNode['node'];
             $nodeID = $node['id'];
             $parsedPathList = GraphWalker::parsePathList($graphNode['path_list']);
+            if (!empty($parsedPathList)) {
+                $lastNodeInPath = $parsedPathList[count($parsedPathList)-1];
+                $previousNodeId = $lastNodeInPath['source_id'];
+                $connections[$nodeID][$previousNodeId] = [];
+            }
             foreach ($parsedPathList as $pathEntry) {
                 if (!empty($filterNodeIDToLabel[$pathEntry['source_id']])) {
-                    $connections[$nodeID][] = $filterNodeIDToLabel[$pathEntry['source_id']];
+                    $connections[$nodeID][$previousNodeId][] = $filterNodeIDToLabel[$pathEntry['source_id']];
                 }
                 if (!empty($resetFilterNodeIDToLabel[$pathEntry['source_id']])) {
                     if ($resetFilterNodeIDToLabel[$pathEntry['source_id']] == 'all') {
-                        $connections[$nodeID] = [];
+                        $connections[$nodeID][$previousNodeId] = [];
                     } else {
-                        $connections[$nodeID] = array_values(array_diff($connections[$nodeID], [$resetFilterNodeIDToLabel[$pathEntry['source_id']]]));
+                        $connections[$nodeID][$previousNodeId] = array_values(array_diff($connections[$nodeID][$previousNodeId], [$resetFilterNodeIDToLabel[$pathEntry['source_id']]]));
                     }
                 }
             }
         }
+        $connections = array_filter($connections, function($connection) {
+            foreach ($connection as $labels) {
+                if (!empty($labels)) {
+                    return true;
+                }
+            }
+            return false;
+        });
         return $connections;
     }
 
@@ -1355,7 +1368,7 @@ class Workflow extends AppModel
                                 'name' => $label,
                                 'variant' => 'info',
                             ];
-                        }, $labelsByNodes[$node['id']]);
+                        }, $labelsByNodes[$node['id']][$connection['node']]);
                     }
                 }
             }

From be2a6bb57cc6b7fe8961067bb6fd0e784f48089c Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 20 Jul 2023 17:01:09 +0200
Subject: [PATCH 52/96] chg: [workflow:editor] Add class if node expect MISP
 core format

---
 app/Model/Workflow.php                              | 1 +
 app/webroot/css/workflows-editor.css                | 4 ++++
 app/webroot/js/workflows-editor/workflows-editor.js | 6 ++++++
 3 files changed, 11 insertions(+)

diff --git a/app/Model/Workflow.php b/app/Model/Workflow.php
index 0c6837122..4e1c4804e 100644
--- a/app/Model/Workflow.php
+++ b/app/Model/Workflow.php
@@ -1421,6 +1421,7 @@ class Workflow extends AppModel
                 'indexed_params' => $indexed_params,
                 'saved_filters' => $module_config['saved_filters'],
                 'module_data' => $module_config,
+                'expect_misp_core_format' => $module_config['expect_misp_core_format'],
             ],
             'inputs' => [],
             'outputs' => [],
diff --git a/app/webroot/css/workflows-editor.css b/app/webroot/css/workflows-editor.css
index f285b42e1..b9314afff 100644
--- a/app/webroot/css/workflows-editor.css
+++ b/app/webroot/css/workflows-editor.css
@@ -346,6 +346,10 @@
 }
 
 /* Custom input/output CSS */
+/* .drawflow .drawflow-node.expect-misp-core-format>.inputs>.input {
+    background-color: #2fa1db;
+} */
+
 .drawflow .drawflow-node > .inputs > .input::before {
     line-height: var(--dfInputHeight);
     vertical-align: top;
diff --git a/app/webroot/js/workflows-editor/workflows-editor.js b/app/webroot/js/workflows-editor/workflows-editor.js
index 9846a6450..01d2fbaa7 100644
--- a/app/webroot/js/workflows-editor/workflows-editor.js
+++ b/app/webroot/js/workflows-editor/workflows-editor.js
@@ -652,6 +652,9 @@ function addNode(block, position, additionalData={}) {
     if (newNode.data.module_data.module_type == 'logic') {
         blockClass.push('block-type-logic')
     }
+    if (newNode.data.module_data.expect_misp_core_format) {
+        blockClass.push('expect-misp-core-format')
+    }
     editor.addNode(
         newNode.name,
         module.inputs === undefined ? 1 : module.inputs,
@@ -754,6 +757,9 @@ function loadWorkflow(workflow) {
         if (newNode.data.module_data.module_type == 'logic') {
             nodeClass.push('block-type-logic')
         }
+        if (newNode.data.module_data.expect_misp_core_format) {
+            nodeClass.push('expect-misp-core-format')
+        }
         if (newNode.data.module_data.disabled) {
             nodeClass.push('disabled')
         }

From 15139fe8652abbf46bc3f47edc7c4765ba1e4d5c Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Fri, 21 Jul 2023 09:13:02 +0200
Subject: [PATCH 53/96] chg: [workflow:editor] Added support of chosen options
 and disabled inputs

---
 .../action/Module_attach_enrichment.php       | 14 ++++++++--
 .../js/workflows-editor/workflows-editor.js   | 26 +++++++++++++++++--
 2 files changed, 36 insertions(+), 4 deletions(-)

diff --git a/app/Model/WorkflowModules/action/Module_attach_enrichment.php b/app/Model/WorkflowModules/action/Module_attach_enrichment.php
index f65729a21..cdf05e78d 100644
--- a/app/Model/WorkflowModules/action/Module_attach_enrichment.php
+++ b/app/Model/WorkflowModules/action/Module_attach_enrichment.php
@@ -23,11 +23,19 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
         parent::__construct();
         $this->Module = ClassRegistry::init('Module');
         $modules = $this->Module->getModules('Enrichment');
-        $this->allModulesByName = Hash::combine($modules, '{n}.name', '{n}');
         if (is_array($modules)) {
+            $this->allModulesByName = Hash::combine($modules, '{n}.name', '{n}');
+        }
+        $moduleOptions = [];
+        $pickerOptions = [];
+        $enrichmentAvailable = false;
+        if (!empty($modules) && is_array($modules)) {
+            $enrichmentAvailable = true;
             $moduleOptions = array_merge([''], Hash::combine($modules, '{n}.name', '{n}.name'));
         } else {
-            $moduleOptions[] = $modules;
+            $pickerOptions = [
+                'placeholder_text_multiple' => __('No enrichment module available'),
+            ];
         }
         sort($moduleOptions);
         $this->params = [
@@ -36,7 +44,9 @@ class Module_attach_enrichment extends WorkflowBaseActionModule
                 'label' => 'Modules',
                 'type' => 'picker',
                 'multiple' => true,
+                'disabled' => !$enrichmentAvailable,
                 'options' => $moduleOptions,
+                'picker_options' => $pickerOptions,
             ],
         ];
     }
diff --git a/app/webroot/js/workflows-editor/workflows-editor.js b/app/webroot/js/workflows-editor/workflows-editor.js
index 01d2fbaa7..ed4c1f0e5 100644
--- a/app/webroot/js/workflows-editor/workflows-editor.js
+++ b/app/webroot/js/workflows-editor/workflows-editor.js
@@ -1356,13 +1356,19 @@ function genNodeParamHtml(node, forNode = true) {
 
 function afterNodeDrawCallback() {
     var $nodes = $drawflow.find('.drawflow-node')
-    $nodes.find('.start-chosen').chosen()
+    $nodes.find('.start-chosen').each(function() {
+        var chosenOptions = $(this).data('chosen_options')
+        $(this).chosen(chosenOptions)
+    })
     toggleDisplayOnFields()
     enablePickerCreateNewOptions()
 }
 
 function afterModalShowCallback() {
-    $blockModal.find('.start-chosen').chosen()
+    $blockModal.find('.start-chosen').each(function() {
+        var chosenOptions = $(this).data('chosen_options')
+        $(this).chosen(chosenOptions)
+    })
     var cmOptions = {
         theme: 'default',
         lineNumbers: true,
@@ -1495,6 +1501,9 @@ function genSelect(options, forNode = true) {
             }
         }
     }
+    if (options.disabled !== undefined) {
+        $select.prop('disabled', options.disabled == true)
+    }
     var selectOptions = options.options
     if (!Array.isArray(selectOptions)) {
         selectOptions = Object.keys(options.options).map((k) => { return { name: options.options[k], value: k } })
@@ -1543,6 +1552,10 @@ function genPicker(options, forNode = true) {
     var $container = genSelect(options)
     var $select = $container.find('select')
     $select.addClass('start-chosen')
+    if (options.picker_options) {
+        // $select.data('chosen_options', options.picker_options)
+        $select.attr('data-chosen_options', JSON.stringify(options.picker_options))
+    }
     return $container
 }
 
@@ -1587,6 +1600,9 @@ function genInput(options, isTextArea, forNode = true) {
     if (options.placeholder !== undefined) {
         $input.attr('placeholder', options.placeholder)
     }
+    if (options.disabled !== undefined) {
+        $input.prop('disabled', options.disabled == true)
+    }
     $label.append($input)
     $container.append($label)
     return $container
@@ -1614,6 +1630,9 @@ function genCheckbox(options, forNode = true) {
     } else if (options.default) {
         $input.attr('checked', '')
     }
+    if (options.disabled !== undefined) {
+        $input.prop('disabled', options.disabled == true)
+    }
     $label.append($input)
     var $container = $('<div>')
         .addClass('node-param-container')
@@ -1670,6 +1689,9 @@ function genRadio(options, forNode = true) {
         } else if (options.default) {
             $input.attr('checked', '')
         }
+        if (options.disabled !== undefined) {
+            $input.prop('disabled', options.disabled == true)
+        }
         var $label = $('<label>')
             .addClass('radio')
             .css({

From af92f52a30231fa847fd2cc265cc4edc06365fed Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Fri, 21 Jul 2023 10:33:45 +0200
Subject: [PATCH 54/96] fix: [workflow:triggers] Fixed typo in column
 description

---
 app/View/Workflows/triggers.ctp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/View/Workflows/triggers.ctp b/app/View/Workflows/triggers.ctp
index 7ec199539..cc79b3f33 100644
--- a/app/View/Workflows/triggers.ctp
+++ b/app/View/Workflows/triggers.ctp
@@ -113,7 +113,7 @@
             'element' => 'checkbox_action',
             'onclick' => "enableWorkflowDebugMode(%s, %s)",
             'onclick_params_data_path' => ['Workflow.id', 'Workflow.debug_enabled'],
-            'title' => __('Put the workflow in debug mode. Each nodes will send data to the provided debug URL')
+            'title' => __('Set the workflow in debug mode. Each nodes will send data to the provided debug URL')
         ],
         [
             'name' => __('Enabled'),

From 6320295ac677b5e2c32a698ac1e83eb780be4db9 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Fri, 21 Jul 2023 10:34:13 +0200
Subject: [PATCH 55/96] chg: [workflow:logging] Changed logging behavior to be
 less verbose when debug is not enabled

- When debug is enabled, it will log eveything as it used to be but also include successfull node execution
- When debug is disabled, it will only log execution errors
---
 app/Model/Workflow.php | 44 +++++++++++++++++++++++++++++-------------
 1 file changed, 31 insertions(+), 13 deletions(-)

diff --git a/app/Model/Workflow.php b/app/Model/Workflow.php
index 4e1c4804e..4263ada24 100644
--- a/app/Model/Workflow.php
+++ b/app/Model/Workflow.php
@@ -518,15 +518,14 @@ class Workflow extends AppModel
     {
         $this->Log = ClassRegistry::init('Log');
         $message =  __('Started executing workflow for trigger `%s` (%s)', $triggerModule->id, $workflow['Workflow']['id']);
-        $this->Log->createLogEntry('SYSTEM', 'execute_workflow', 'Workflow', $workflow['Workflow']['id'], $message);
-        $this->__logToFile($workflow, $message);
+        $this->logExecutionIfDebug($workflow, $message);
         $workflow = $this->__incrementWorkflowExecutionCount($workflow);
         $walkResult = [];
         $debugData = ['original' => $data];
         $data = $this->__normalizeDataForTrigger($triggerModule, $data);
         $debugData['normalized'] = $data;
         $for_path = !empty($triggerModule->blocking) ? GraphWalker::PATH_TYPE_BLOCKING : GraphWalker::PATH_TYPE_NON_BLOCKING;
-        $this->sendRequestToDebugEndpoint($workflow, [], '/init?type=' . $for_path, $debugData);
+        $this->sendRequestToDebugEndpointIfDebug($workflow, [], '/init?type=' . $for_path, $debugData);
 
         $blockingPathExecutionSuccess = $this->walkGraph($workflow, $startNodeID, $for_path, $data, $blockingErrors, $walkResult);
         $executionStoppedByStopModule = in_array('stop-execution', Hash::extract($walkResult, 'blocking_nodes.{n}.data.id'));
@@ -542,9 +541,8 @@ class Workflow extends AppModel
         }
         $message =  __('Finished executing workflow for trigger `%s` (%s). Outcome: %s', $triggerModule->id, $workflow['Workflow']['id'], $outcomeText);
 
-        $this->Log->createLogEntry('SYSTEM', 'execute_workflow', 'Workflow', $workflow['Workflow']['id'], $message);
-        $this->__logToFile($workflow, $message);
-        $this->sendRequestToDebugEndpoint($workflow, [], '/end?outcome=' . $outcomeText, $walkResult);
+        $this->logExecutionIfDebug($workflow, $message);
+        $this->sendRequestToDebugEndpointIfDebug($workflow, [], '/end?outcome=' . $outcomeText, $walkResult);
         return [
             'outcomeText' => $outcomeText,
             'walkResult' => $walkResult,
@@ -661,7 +659,7 @@ class Workflow extends AppModel
             $message = __('Could not execute disabled module `%s`.', $node['data']['id']);
             $this->logExecutionError($roamingData->getWorkflow(), $message);
             $errors[] = $message;
-            $this->sendRequestToDebugEndpoint($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s', $moduleClass->id, 'disabled_module'), $roamingData->getData());
+            $this->sendRequestToDebugEndpointIfDebug($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s', $moduleClass->id, 'disabled_module'), $roamingData->getData());
             return false;
         }
         if (!is_null($moduleClass)) {
@@ -671,17 +669,25 @@ class Workflow extends AppModel
                 $message = __('Error while executing module %s. Error: %s', $node['data']['id'], $e->getMessage());
                 $this->logExecutionError($roamingData->getWorkflow(), $message);
                 $errors[] = $message;
-                $this->sendRequestToDebugEndpoint($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s&message=%s', $moduleClass->id, 'error', $e->getMessage()), $roamingData->getData());
+                $this->sendRequestToDebugEndpointIfDebug($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s&message=%s', $moduleClass->id, 'error', $e->getMessage()), $roamingData->getData());
                 return false;
             }
         } else {
             $message = sprintf(__('Could not load class for module: %s'), $node['data']['id']);
             $this->logExecutionError($roamingData->getWorkflow(), $message);
             $errors[] = $message;
-            $this->sendRequestToDebugEndpoint($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s', $node['data']['id'], 'loading_error'), $roamingData->getData());
+            $this->sendRequestToDebugEndpointIfDebug($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s', $node['data']['id'], 'loading_error'), $roamingData->getData());
             return false;
         }
-        $this->sendRequestToDebugEndpoint($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s', $moduleClass->id, 'success'), $roamingData->getData());
+        $message = __('Executed node `%s`' .  PHP_EOL . 'Node `%s` (%s) from Workflow `%s` (%s) executed successfully',
+            $node['data']['id'],
+            $node['data']['id'],
+            $node['id'],
+            $roamingData->getWorkflow()['Workflow']['name'],
+            $roamingData->getWorkflow()['Workflow']['id'],
+        );
+        $this->logExecutionIfDebug($roamingData->getWorkflow(), $message);
+        $this->sendRequestToDebugEndpointIfDebug($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s', $moduleClass->id, 'success'), $roamingData->getData());
         return $success;
     }
 
@@ -993,6 +999,14 @@ class Workflow extends AppModel
         $this->__logToFile($workflow, $message);
     }
 
+    public function logExecutionIfDebug(array $workflow, $message): void
+    {
+        if ($workflow['Workflow']['debug_enabled']) {
+            $this->Log->createLogEntry('SYSTEM', 'execute_workflow', 'Workflow', $workflow['Workflow']['id'], $message);
+            $this->__logToFile($workflow, $message);
+        }
+    }
+
     /**
      * __logToFile Log to file
      *
@@ -1478,12 +1492,16 @@ class Workflow extends AppModel
         return $saveSuccess;
     }
 
+    public function sendRequestToDebugEndpointIfDebug(array $workflow, array $node, $path='/', array $data=[])
+    {
+        if ($workflow['Workflow']['debug_enabled']) {
+            $this->sendRequestToDebugEndpoint($workflow, $node, $path, $data);
+        }
+    }
+
     public function sendRequestToDebugEndpoint(array $workflow, array $node, $path='/', array $data=[])
     {
         $debug_url = Configure::read('Plugin.Workflow_debug_url');
-        if (empty($workflow['Workflow']['debug_enabled'])) {
-            return;
-        }
         App::uses('HttpSocket', 'Network/Http');
         $socket = new HttpSocket([
             'timeout' => 5

From 00f09cefbe87a655dc41704c2a99b948e3446b1c Mon Sep 17 00:00:00 2001
From: Christian Studer <christian.studer@circl.lu>
Date: Fri, 21 Jul 2023 16:15:51 +0200
Subject: [PATCH 56/96] fix: [taxii_push] Passing standard MISP JSON format to
 the `taxii_push` script and by extension to misp-stix

---
 app/Model/TaxiiServer.php | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/app/Model/TaxiiServer.php b/app/Model/TaxiiServer.php
index 86ad6bfe4..4ae9c327f 100644
--- a/app/Model/TaxiiServer.php
+++ b/app/Model/TaxiiServer.php
@@ -4,6 +4,8 @@ App::uses('EncryptedValue', 'Tools');
 App::uses('Folder', 'Utility');
 App::uses('File', 'Utility');
 App::uses('RandomTool', 'Tools');
+App::uses('JSONConverterTool', 'Tools');
+App::uses('JsonTool', 'Tools');
 
 class TaxiiServer extends AppModel
 {
@@ -101,7 +103,11 @@ class TaxiiServer extends AppModel
         $this->Job->id = $jobId;
         foreach ($result as $event) {
             $temporaryFile = $this->temporaryFile($temporaryFolderPath);
-            $temporaryFile->write(json_encode($event));
+            $temporaryFile->write(
+                JsonTool::encode(
+                    JSONConverterTool::convert($event, false, true)
+                )
+            );
             $temporaryFile->close();
             if ($jobId && $i % 10 == 0) {
                 $this->Job->saveField('progress', intval((100 * $i) / $eventCount));

From f35456baa0e9f6a845fbf0b005e4d94be1c74977 Mon Sep 17 00:00:00 2001
From: Christian Studer <christian.studer@circl.lu>
Date: Fri, 21 Jul 2023 16:15:51 +0200
Subject: [PATCH 57/96] fix: [taxii_push] Passing standard MISP JSON format to
 the `taxii_push` script and by extension to misp-stix

---
 app/Model/TaxiiServer.php | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/app/Model/TaxiiServer.php b/app/Model/TaxiiServer.php
index 106ad7cf3..343619965 100644
--- a/app/Model/TaxiiServer.php
+++ b/app/Model/TaxiiServer.php
@@ -4,6 +4,8 @@ App::uses('EncryptedValue', 'Tools');
 App::uses('Folder', 'Utility');
 App::uses('File', 'Utility');
 App::uses('RandomTool', 'Tools');
+App::uses('JSONConverterTool', 'Tools');
+App::uses('JsonTool', 'Tools');
 
 class TaxiiServer extends AppModel
 {
@@ -102,7 +104,11 @@ class TaxiiServer extends AppModel
         $this->Job->id = $jobId;
         foreach ($result as $event) {
             $temporaryFile = $this->temporaryFile($temporaryFolderPath);
-            $temporaryFile->write(json_encode($event));
+            $temporaryFile->write(
+                JsonTool::encode(
+                    JSONConverterTool::convert($event, false, true)
+                )
+            );
             $temporaryFile->close();
             if ($jobId && $i % 10 == 0) {
                 $this->Job->saveField('progress', intval((100 * $i) / $eventCount));

From b3180624bbf3e38843120b4f16361a8ee879e2e0 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 24 Jul 2023 00:08:53 +0200
Subject: [PATCH 58/96] fix: [sightings] only pushed via full push to avoid
 congestion

- the old behaviour can be re-enabled via Sightings.enable_realtime_publish
- massive performance gain on heavily interconnected instances
---
 app/Controller/SightingsController.php | 3 ++-
 app/Model/Server.php                   | 7 +++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/app/Controller/SightingsController.php b/app/Controller/SightingsController.php
index bb5259072..7ba059918 100644
--- a/app/Controller/SightingsController.php
+++ b/app/Controller/SightingsController.php
@@ -66,7 +66,8 @@ class SightingsController extends AppController
                 $filters = !empty($this->request->data['filters']) ? $this->request->data['filters'] : false;
             }
             if (!$error) {
-                $result = $this->Sighting->saveSightings($id, $values, $timestamp, $this->Auth->user(), $type, $source, false, true, false, $filters);
+                $publish_sighting = !empty(Configure::read('Sightings_enable_realtime_publish'));
+                $result = $this->Sighting->saveSightings($id, $values, $timestamp, $this->Auth->user(), $type, $source, false, $publish_sighting, false, $filters);
             }
             if (!is_numeric($result)) {
                 $error = $result;
diff --git a/app/Model/Server.php b/app/Model/Server.php
index b0d69eea5..e4157a451 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -7201,6 +7201,13 @@ class Server extends AppModel
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
+                'Sightings_enable_realtime_publish' => array(
+                    'level' => 1,
+                    'description' => __('By default, sightings will not be immediately pushed to connected instances, as this can have a heavy impact on the performance of sighting attributes. Enable realtime publishing to trigger the publishing of sightings immediately as they are added.'),
+                    'value' => false,
+                    'test' => 'testBool',
+                    'type' => 'boolean'
+                ),
                 'CustomAuth_enable' => array(
                     'level' => 2,
                     'description' => __('Enable this functionality if you would like to handle the authentication via an external tool and authenticate with MISP using a custom header.'),

From a0f6c4e45a2c8468df624e80be003da06282f6b2 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 24 Jul 2023 00:19:24 +0200
Subject: [PATCH 59/96] fix: [proposal accept] fixed for deletions

- soft delete rather than hard delete or the propagation will fail
---
 app/Controller/ShadowAttributesController.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Controller/ShadowAttributesController.php b/app/Controller/ShadowAttributesController.php
index d8df99e5a..aa8e87781 100644
--- a/app/Controller/ShadowAttributesController.php
+++ b/app/Controller/ShadowAttributesController.php
@@ -78,7 +78,7 @@ class ShadowAttributesController extends AppController
             }
 
             if (isset($shadow['proposal_to_delete']) && $shadow['proposal_to_delete']) {
-                $this->Attribute->delete($activeAttribute['Attribute']['id']);
+                $this->Attribute->deleteAttribute($activeAttribute['Attribute']['id'], $this->Auth->user(), false);
             } else {
                 // Update the live attribute with the shadow data
                 $fieldsToUpdate = array('value1', 'value2', 'value', 'type', 'category', 'comment', 'to_ids', 'first_seen', 'last_seen');

From d14728abbb932e3f1ec608612ab1bf77b4053a79 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 24 Jul 2023 00:32:19 +0200
Subject: [PATCH 60/96] fix: [background workers] speculative fix for issues
 with publishing

- job object not found or not retrieved correctly
---
 app/Console/Command/EventShell.php | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/app/Console/Command/EventShell.php b/app/Console/Command/EventShell.php
index 069df17c4..9fb01fd45 100644
--- a/app/Console/Command/EventShell.php
+++ b/app/Console/Command/EventShell.php
@@ -405,7 +405,17 @@ class EventShell extends AppShell
         $jobId = $this->args[2];
         $userId = $this->args[3];
         $user = $this->getUser($userId);
-        $job = $this->Job->read(null, $jobId);
+        $job = $this->Job->find('first', [
+            'recursive' => -1,
+            'conditions' => [
+                'Job.id' => $jobId
+            ]
+        ]);
+        if (empty($job)) {
+            $log = ClassRegistry::init('Log');
+            $log->createLogEntry($user, 'publish', 'Event', $id, 'Event (' . $id . '): could not be published - valid job not found.', '');
+            return true;
+        }
         $this->Event->Behaviors->unload('SysLogLogable.SysLogLogable');
         $result = $this->Event->publish($id, $passAlong);
         $job['Job']['progress'] = 100;

From 38208f97c38e7384699198bce06bb48126a73379 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 24 Jul 2023 00:47:23 +0200
Subject: [PATCH 61/96] fix: [proposal] sync fixes

- include disable correlation / proposal to delete fields in the proposal index
- this is used on pulls, causing these fields to not be included
  - especially the proposal to delete field's absence is nasty, as it changes the meaning of the proposal
---
 app/Model/ShadowAttribute.php | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/Model/ShadowAttribute.php b/app/Model/ShadowAttribute.php
index a6f4478ec..cf74fb127 100644
--- a/app/Model/ShadowAttribute.php
+++ b/app/Model/ShadowAttribute.php
@@ -705,6 +705,8 @@ class ShadowAttribute extends AppModel
                 $this->logException("Could not fetch page $i of proposals from remote server {$serverSync->server()['Server']['id']}", $e);
                 return $fetchedCount;
             }
+            debug($data);
+            throw new Exception();
             $returnSize = count($data);
             if ($returnSize === 0) {
                 return $fetchedCount;

From b9136ab058869e4adec7933a1115dfffdca83291 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 24 Jul 2023 00:49:54 +0200
Subject: [PATCH 62/96] fix: [debug reverted] reverted erroneously committed
 debug / exception

---
 app/Model/ShadowAttribute.php | 2 --
 1 file changed, 2 deletions(-)

diff --git a/app/Model/ShadowAttribute.php b/app/Model/ShadowAttribute.php
index cf74fb127..a6f4478ec 100644
--- a/app/Model/ShadowAttribute.php
+++ b/app/Model/ShadowAttribute.php
@@ -705,8 +705,6 @@ class ShadowAttribute extends AppModel
                 $this->logException("Could not fetch page $i of proposals from remote server {$serverSync->server()['Server']['id']}", $e);
                 return $fetchedCount;
             }
-            debug($data);
-            throw new Exception();
             $returnSize = count($data);
             if ($returnSize === 0) {
                 return $fetchedCount;

From 786f46edb41b8f86d423eae95759c96e129fbc39 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 24 Jul 2023 00:50:29 +0200
Subject: [PATCH 63/96] fix: [proposal] proposal index fix as described 2
 commits ago

---
 app/Controller/ShadowAttributesController.php | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/app/Controller/ShadowAttributesController.php b/app/Controller/ShadowAttributesController.php
index aa8e87781..7580a0680 100644
--- a/app/Controller/ShadowAttributesController.php
+++ b/app/Controller/ShadowAttributesController.php
@@ -906,7 +906,23 @@ class ShadowAttributesController extends AppController
         }
         $params = array(
             'conditions' => $conditions,
-            'fields' => array('ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.timestamp', 'ShadowAttribute.first_seen', 'ShadowAttribute.last_seen'),
+            'fields' => array(
+                'ShadowAttribute.id',
+                'ShadowAttribute.old_id',
+                'ShadowAttribute.event_id',
+                'ShadowAttribute.type',
+                'ShadowAttribute.category',
+                'ShadowAttribute.uuid',
+                'ShadowAttribute.to_ids',
+                'ShadowAttribute.value',
+                'ShadowAttribute.comment',
+                'ShadowAttribute.org_id',
+                'ShadowAttribute.timestamp',
+                'ShadowAttribute.first_seen',
+                'ShadowAttribute.last_seen',
+                'ShadowAttribute.proposal_to_delete',
+                'ShadowAttribute.disable_correlation'
+            ),
             'contain' => array(
                     'Event' => array(
                             'fields' => array('id', 'org_id', 'info', 'orgc_id', 'uuid'),

From 47ba11e246ed0ba23263fcec90ee17ac3bbcd796 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 24 Jul 2023 00:52:44 +0200
Subject: [PATCH 64/96] fix: [proposal] index should also include the "deleted"
 field

---
 app/Controller/ShadowAttributesController.php | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/Controller/ShadowAttributesController.php b/app/Controller/ShadowAttributesController.php
index 7580a0680..bfaf2be34 100644
--- a/app/Controller/ShadowAttributesController.php
+++ b/app/Controller/ShadowAttributesController.php
@@ -920,6 +920,7 @@ class ShadowAttributesController extends AppController
                 'ShadowAttribute.timestamp',
                 'ShadowAttribute.first_seen',
                 'ShadowAttribute.last_seen',
+                'ShadowAttribute.deleted',
                 'ShadowAttribute.proposal_to_delete',
                 'ShadowAttribute.disable_correlation'
             ),

From 18fd906e52065e8a46b06c420c562d957459d639 Mon Sep 17 00:00:00 2001
From: Christian Studer <christian.studer@circl.lu>
Date: Mon, 24 Jul 2023 23:47:14 +0200
Subject: [PATCH 65/96] fix: [stix export] Avoiding issues in the case of empty
 input

- With no input, the python script called to
  convert the MISP input used to barf because
  there is no input.
- Should fix MISP/misp-stix#44
---
 app/Lib/Export/StixExport.php | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/app/Lib/Export/StixExport.php b/app/Lib/Export/StixExport.php
index 6abc20b75..4c759e991 100644
--- a/app/Lib/Export/StixExport.php
+++ b/app/Lib/Export/StixExport.php
@@ -83,6 +83,12 @@ abstract class StixExport
         if ($this->__empty_file) {
             $this->__tmp_file->close();
             $this->__tmp_file->delete();
+            if (empty($this->__filenames)) {
+                $framing = $this->getFraming();
+                $tmpFile = new TmpFileTool();
+                $tmpFile->write($framing['header'] . $framing['footer']);
+                return $tmpFile;
+            }
         } else {
             if (!empty($this->__event_galaxies)) {
                 $this->__write_event_galaxies();

From 83634a651868b0d2b33707b12557dd5dd5f0258e Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Sat, 22 Jul 2023 11:50:50 +0200
Subject: [PATCH 66/96] new: [workflow:editor] Added support of quick insert on
 link

---
 app/webroot/css/workflows-editor.css          |   5 +
 .../js/workflows-editor/workflows-editor.js   | 112 +++++++++++++++++-
 2 files changed, 114 insertions(+), 3 deletions(-)

diff --git a/app/webroot/css/workflows-editor.css b/app/webroot/css/workflows-editor.css
index b9314afff..07de95a2f 100644
--- a/app/webroot/css/workflows-editor.css
+++ b/app/webroot/css/workflows-editor.css
@@ -345,6 +345,11 @@
     border: 3px solid #e35651;
 }
 
+.drawflow svg.connection > path.link-hover-for-insertion {
+    stroke: #7210cd;
+    filter: drop-shadow(0px 0px 6px #7210cddd);
+}
+
 /* Custom input/output CSS */
 /* .drawflow .drawflow-node.expect-misp-core-format>.inputs>.input {
     background-color: #2fa1db;
diff --git a/app/webroot/js/workflows-editor/workflows-editor.js b/app/webroot/js/workflows-editor/workflows-editor.js
index ed4c1f0e5..48fbac536 100644
--- a/app/webroot/js/workflows-editor/workflows-editor.js
+++ b/app/webroot/js/workflows-editor/workflows-editor.js
@@ -304,9 +304,33 @@ function initDrawflow() {
             if (ui.draggable.data('blueprint')) {
                 addWorkflowBlueprint(ui.draggable.data('blueprint').WorkflowBlueprint.id, ui.position)
             } else {
-                addNode(ui.draggable.data('module'), ui.position)
+                var node = addNode(ui.draggable.data('module'), ui.position)
+                var fakeUi = {
+                    draggable: ui.helper,
+                    position: { left: ui.helper[0].getBoundingClientRect().left, top: ui.helper[0].getBoundingClientRect().top }
+                }
+                var link = getLinkUnderElement(fakeUi)
+                if (link !== undefined) {
+                    insertNodeOnLink(node, link)
+                }
             }
         },
+        activate: function (event, ui) {
+            var $pathsWithClass = $()
+            ui.helper.mousemove(function (event) {
+                var fakeUi = {
+                    draggable: $(this),
+                    position: {left: this.getBoundingClientRect().left, top: this.getBoundingClientRect().top}
+                }
+                var links = getLinkUnderElement(fakeUi)
+                $pathsWithClass.removeClass('link-hover-for-insertion')
+                if (links) {
+                    $paths = $(links).find('path')
+                    $paths.addClass('link-hover-for-insertion')
+                    $pathsWithClass = $paths
+                }
+            })
+        },
     });
 
     graphPooler = new TaskScheduler(checkGraphProperties, {
@@ -666,6 +690,7 @@ function addNode(block, position, additionalData={}) {
         html
     )
     afterNodeDrawCallback()
+    return editor.getNodeFromId(editor.nodeId-1)
 }
 
 function getEditorData(cleanNodes) {
@@ -850,7 +875,8 @@ function duplicateNodesFromHtml(currentSelection) {
                         oldNewIDMapping[node_id],
                         oldNewIDMapping[connection.node],
                         outputName,
-                        connection.output
+                        connection.output,
+                        []
                     )
                 }
             });
@@ -916,7 +942,8 @@ function addNodesFromBlueprint(workflowBlueprint, cursorPosition) {
                             oldNewIDMapping[node.id],
                             oldNewIDMapping[connection.node],
                             outputName,
-                            connection.output
+                            connection.output,
+                            []
                         )
                     }
                 });
@@ -1266,6 +1293,85 @@ function addWorkflowBlueprint(blueprintId, cursorPosition) {
     }
 }
 
+function getLinkUnderElement(ui) {
+    var orig_pos_x = ui.position.left
+    var orig_pos_y = ui.position.top
+    var elementHeight = ui.draggable[0].clientHeight
+    var elementCenterY = ui.position.top + ui.draggable[0].clientHeight / 2
+
+     // Credit: Drawflow example page
+    var transpositionFactorX = (editor.precanvas.clientWidth / (editor.precanvas.clientWidth * editor.zoom))
+    var transpositionOffsetX = -(editor.precanvas.getBoundingClientRect().x * (editor.precanvas.clientWidth / (editor.precanvas.clientWidth * editor.zoom)))
+    var transpositionFactorY = (editor.precanvas.clientHeight / (editor.precanvas.clientHeight * editor.zoom))
+    var transpositionOffsetY = -(editor.precanvas.getBoundingClientRect().y * (editor.precanvas.clientHeight / (editor.precanvas.clientHeight * editor.zoom)))
+
+    var pos_x = orig_pos_x * transpositionFactorX + transpositionOffsetX
+    var pos_y = orig_pos_y * transpositionFactorY + transpositionOffsetY
+    var transposedHeight = (orig_pos_y + elementHeight) * transpositionFactorY + transpositionOffsetY
+    var transposedCenterY = elementCenterY * transpositionFactorY + transpositionOffsetY
+
+    var $allConnectionMiddlePoint = $drawflow.find('svg.connection > foreignObject')
+    var allValidLinks = []
+    $allConnectionMiddlePoint.each(function() {
+        var middlePointY = this.y.baseVal.value
+        if (pos_y <= middlePointY && middlePointY <= transposedHeight) {
+            var $svg = this.parentElement.childNodes[0]
+            var svgBR = $svg.getBoundingClientRect()
+            var linkTransposedLeftPosition = svgBR.x * transpositionFactorX + transpositionOffsetX
+            var linkTransposedRightPosition = (svgBR.x + svgBR.width) * transpositionFactorX + transpositionOffsetX
+            if (linkTransposedLeftPosition <= pos_x && pos_x <= linkTransposedRightPosition) {
+                allValidLinks.push(this.parentElement)
+            }
+        }
+    })
+
+    if (allValidLinks.length == 1) {
+        return allValidLinks[0]
+    } else if (allValidLinks.length > 1) {
+        // sort based on distance between link middle point and element input
+        function calcDistance(pt1, pt2) {
+            return Math.sqrt((pt2.x - pt1.x)**2 + (pt2.y - pt1.y)**2)
+        }
+        var elementInputPosition = { x: pos_x, y: transposedCenterY }
+        var allValidLinksSorted = allValidLinks.sort(function (link1, link2) {
+            var middlePoint1 = {
+                x: $(link1).find('foreignObject')[0].x.baseVal.value,
+                y: $(link1).find('foreignObject')[0].y.baseVal.value,
+            }
+            var middlePoint2 = {
+                x: $(link2).find('foreignObject')[0].x.baseVal.value,
+                y: $(link2).find('foreignObject')[0].y.baseVal.value,
+            }
+            var distance1 = calcDistance(elementInputPosition, middlePoint1)
+            var distance2 = calcDistance(elementInputPosition, middlePoint2)
+            return distance1 <= distance2 ? -1 : 1
+        });
+        return allValidLinksSorted[0]
+    }
+    return
+}
+
+function insertNodeOnLink(newNode, link) {
+    var defaultConnectionName = 'output_1'
+    var ids = getIDsFromSvgLink(link)
+    var nodeIn = ids.nodeIn
+    var inConnectionName = ids.inConnectionName
+    var nodeOut = ids.nodeOut
+    var outConnectionName = ids.outConnectionName
+    editor.addConnection(nodeOut, newNode.id, outConnectionName, inConnectionName, [])
+    editor.addConnection(newNode.id, nodeIn, defaultConnectionName, inConnectionName, [])
+    editor.removeSingleConnection(nodeOut, nodeIn, outConnectionName, inConnectionName)
+}
+
+function getIDsFromSvgLink(link) {
+    return {
+        'nodeIn': parseInt(Array.from(link.classList).filter(c => c.startsWith('node_in_node-'))[0].replace('node_in_node-', '')),
+        'inConnectionName': Array.from(link.classList).filter(c => c.startsWith('input_'))[0],
+        'nodeOut': parseInt(Array.from(link.classList).filter(c => c.startsWith('node_out_node-'))[0].replace('node_out_node-', '')),
+        'outConnectionName': Array.from(link.classList).filter(c => c.startsWith('output_'))[0],
+    }
+}
+
 /* UI Utils */
 function toggleSaveButton(enabled) {
     $saveWorkflowButton

From 145435ef516b8cc53a0f65dc1ec11105df33daf8 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Tue, 25 Jul 2023 15:39:45 +0200
Subject: [PATCH 67/96] new: [workflow:editor] Added hash-path picker helper
 functionality

---
 app/View/Elements/Workflows/infoModal.ctp     |   2 +-
 app/webroot/css/workflows-editor.css          |  23 ++
 app/webroot/js/misp.js                        |   4 +-
 .../js/workflows-editor/workflows-editor.js   | 297 ++++++++++++++++++
 4 files changed, 323 insertions(+), 3 deletions(-)

diff --git a/app/View/Elements/Workflows/infoModal.ctp b/app/View/Elements/Workflows/infoModal.ctp
index 1f2fcc3bb..b5ad6260f 100644
--- a/app/View/Elements/Workflows/infoModal.ctp
+++ b/app/View/Elements/Workflows/infoModal.ctp
@@ -133,7 +133,7 @@ $data_passed_to_if_module = [
                         </ul>
                 </ul>
                 <p><strong><?= __('Sample:') ?></strong></p>
-                <pre>
+                <pre id="misp-core-format-sample">
 {
     "Event": {
         "id": "64",
diff --git a/app/webroot/css/workflows-editor.css b/app/webroot/css/workflows-editor.css
index 07de95a2f..f0bed546a 100644
--- a/app/webroot/css/workflows-editor.css
+++ b/app/webroot/css/workflows-editor.css
@@ -350,6 +350,29 @@
     filter: drop-shadow(0px 0px 6px #7210cddd);
 }
 
+#core-format-picker {
+    padding: 5px;
+    border-radius: 5px;
+    background-color: #f9f9f9;
+    border: 1px solid #e1e1e1;
+}
+#core-format-picker .selectable-key {
+    padding: 2px 3px;
+}
+#core-format-picker .selectable-key:hover {
+    box-shadow: 0 0 5px #00000077;
+    cursor: pointer;
+}
+
+#core-format-picker .selectable-value {
+    padding: 2px 3px;
+}
+#core-format-picker .selectable-value:hover {
+    box-shadow: 0 0 5px #00000077;
+    font-weight: bold;
+    cursor: pointer;
+}
+
 /* Custom input/output CSS */
 /* .drawflow .drawflow-node.expect-misp-core-format>.inputs>.input {
     background-color: #2fa1db;
diff --git a/app/webroot/js/misp.js b/app/webroot/js/misp.js
index 5c505f42f..a3f37fc70 100644
--- a/app/webroot/js/misp.js
+++ b/app/webroot/js/misp.js
@@ -1998,9 +1998,9 @@ function choicePopup(legend, list) {
     openPopup("#popover_form");
 }
 
-function openModal(heading, body, footer, modal_option, css_container, css_body) {
+function openModal(heading, body, footer, modal_option, css_container, css_body, class_container) {
     var modal_id = 'dynamic_modal_' + new Date().getTime();
-    var modal_html = '<div id="' + modal_id + '" class="modal hide fade" style="' + (css_container !== undefined ? css_container : '') + '" tabindex="-1" role="dialog" aria-hidden="true">';
+    var modal_html = '<div id="' + modal_id + '" class="modal hide fade ' + (class_container !== undefined ? class_container : '') + '" style="' + (css_container !== undefined ? css_container : '') + '" tabindex="-1" role="dialog" aria-hidden="true">';
     if (heading !== undefined && heading !== '') {
         modal_html += '<div class="modal-header">'
                         + '<button type="button" class="close" data-dismiss="modal" aria-hidden="true">×</button>'
diff --git a/app/webroot/js/workflows-editor/workflows-editor.js b/app/webroot/js/workflows-editor/workflows-editor.js
index 48fbac536..67d25aa9d 100644
--- a/app/webroot/js/workflows-editor/workflows-editor.js
+++ b/app/webroot/js/workflows-editor/workflows-editor.js
@@ -143,6 +143,11 @@ var iconBySeverity = {
     'error': 'fa-exclamation-circle',
 }
 var severities = ['info', 'warning', 'error']
+var haspathQuickPickMenu = [
+    { 'name': 'All Attributes', 'path': 'Event._AttributeFlattened.{n}' },
+    { 'name': 'All tags attached to Attributes', 'path': 'Event._AttributeFlattened.{n}.Tag.{n}.name' },
+    { 'name': 'All tags attached to the Event', 'path': 'Event.Tag.{n}.name' },
+]
 
 var workflow_id = 0
 var contentChanged = false
@@ -1437,6 +1442,9 @@ function genNodeParamHtml(node, forNode = true) {
             case 'input':
                 paramHtml = genInput(param, false, forNode)[0].outerHTML
                 break;
+            case 'hashpath':
+                paramHtml = genHashpathInput(param, false, forNode)[0].outerHTML
+                break;
             case 'textarea':
                 paramHtml = genInput(param, true, forNode)[0].outerHTML
                 break;
@@ -1468,6 +1476,7 @@ function afterNodeDrawCallback() {
     })
     toggleDisplayOnFields()
     enablePickerCreateNewOptions()
+    enableHashpathPicker()
 }
 
 function afterModalShowCallback() {
@@ -1549,6 +1558,65 @@ function enablePickerCreateNewOptions() {
     })
 }
 
+function enableHashpathPicker() {
+    var $nodes = $drawflow.find('.drawflow-node')
+    $nodes.find('.hashpath-picker-container').each(function () {
+        $(this).find('.hashpath-quick-picker').click(function () {
+            $(this).closest('.input-append').find('input')
+                .val($(this).data('hashpath'))
+                .trigger('input')
+            
+        })
+        $(this).find('.hashpath-format-picker').click(function() {
+            toggleCoreFormatPicker(this)
+        })
+    })
+}
+
+function toggleCoreFormatPicker(btn) {
+    var associatedParamId = $(btn).closest('.input-append').find('input').data('paramid')
+    var sample = JSON.parse($('#misp-core-format-sample').text())
+    var UIPicker = generateCoreFormatUI(sample, associatedParamId)
+    var $selectedPath = $('<input>')
+        .attr({
+            id: 'selected-hashpath-input',
+            type: 'text',
+            placeholder: 'Click on a elemet on the JSON to show the path',
+            onchange: 'setValueOnAssociatedInput(this.value, "' + associatedParamId + '")',
+        })
+        .css({ margin: '0 0.75em 0 0', 'flex-grow': 2 })
+    var $selectedPathOperators = $('<select>')
+        .attr({
+            id: 'selected-hashpath-operator',
+            onchange: 'setHashpathOnInput(this, "' + associatedParamId + '")',
+        })
+        .css({margin: '0', 'max-width': '15%'})
+    var pathOperators = [
+        { name: 'Has key []', stringToFormat: '[{$key}]' },
+        { name: 'Match [=]', stringToFormat: '[{$key}={$value}]', default: true },
+        { name: 'Match [!=]', stringToFormat: '[{$key}!={$value}]' },
+        { name: 'Match [>]', stringToFormat: '[{$key}>{$value}]' },
+        { name: 'Match [>=]', stringToFormat: '[{$key}>={$value}]' },
+        { name: 'Match [<]', stringToFormat: '[{$key}<{$value}]' },
+        { name: 'Match [<=]', stringToFormat: '[{$key}<={$value}]' },
+        { name: 'Regex match [/.../]', stringToFormat: '[{$key}=/\S*\{$value}\S*/]' },
+    ]
+    pathOperators.forEach((opt) => {
+        var $option = $('<option>')
+            .val(opt.stringToFormat)
+            .text(opt.name)
+        if (opt.default === true) {
+            $option.attr('selected', 'selected')
+        }
+        $selectedPathOperators.append($option)
+    })
+    $pathGroup = $('<span>').css({'display': 'flex', 'width': '100%'})
+        .append($selectedPathOperators, $selectedPath)
+    var $closeButton = $('<div>').append($('<a href="#" class="btn" data-dismiss="modal">Close</a>'))
+    var $footer = $('<div>').css({display: 'flex'}).append($pathGroup, $closeButton)
+    openModal('Pick Hash path', UIPicker[0].outerHTML, $footer[0].outerHTML, undefined, undefined, 'max-height: 70vh;', 'modal-lg')
+}
+
 function genParameterWarning(options) {
     var text = '', text_short = ''
     if (options.is_invalid) {
@@ -1714,6 +1782,83 @@ function genInput(options, isTextArea, forNode = true) {
     return $container
 }
 
+function genHashpathInput(options, forNode = true) {
+    function hashPathGenDropdownMenu() {
+        var $divider = $('<li>').addClass('divider')
+        var $dropdownMenu = $('<ul>').addClass('dropdown-menu pull-right')
+        var $liPicker = $('<li>').append(
+            $('<a>')
+                .attr({
+                    'tabindex': '-1',
+                    'href': '#',
+                })
+                .addClass('hashpath-format-picker')
+                .text('Show picker')
+        )
+        $dropdownMenu.append($liPicker)
+        $dropdownMenu.append($divider)
+        haspathQuickPickMenu.forEach((entry) => {
+            var $li = $('<li>').append(
+                $('<a>')
+                    .attr({
+                        'tabindex': '-1',
+                        'href': '#',
+                        'data-hashpath': entry.path
+                    })
+                    .addClass('hashpath-quick-picker')
+                    .text(entry.name)
+            )
+            $dropdownMenu.append($li)
+        })
+        return $dropdownMenu
+    }
+
+    var $container = $('<div>')
+        .addClass('node-param-container')
+        .attr('param-id', options.id)
+    if (options.display_on) {
+        $container.addClass('display-on')
+    }
+    var $addonContainer = $('<div>')
+        .addClass('input-append')
+        .css({'display': 'flex'})
+    var $label = $('<label>')
+        .css({
+            marginLeft: '0.25em',
+            marginBbottom: 0,
+        })
+        .append(
+            $('<span>').text(options.label),
+            genParameterWarning(options)
+        )
+    var $input = $('<input>').attr('type', 'text').css({ height: '30px' })
+    $input.css({
+        'flex-grow': '1',
+        'box-sizing': 'border-box',
+    })
+    $input
+        .attr('oninput', 'handleInputChange(this)')
+        .attr('data-paramid', options.param_id)
+    $input.attr('value', options.value !== undefined ? options.value : options.default)
+    if (options.placeholder !== undefined) {
+        $input.attr('placeholder', options.placeholder)
+    }
+    if (options.disabled !== undefined) {
+        $input.prop('disabled', options.disabled == true)
+    }
+    var $dropdownContainer = $('<div>').addClass(['btn-group', 'hashpath-picker-container'])
+    var $dropdownButton = $('<button>')
+        .addClass(['btn', 'dropdown-toggle'])
+        .attr('data-toggle', 'dropdown')
+        .text('Pick ')
+        .append($('<span>').addClass('caret'))
+    var $dropdownMenu = hashPathGenDropdownMenu()
+    $dropdownContainer.append($dropdownButton, $dropdownMenu)
+    $addonContainer.append($input, $dropdownContainer)
+    $container.append($label, $addonContainer)
+    return $container
+}
+
 function genCheckbox(options, forNode = true) {
     var $label = $('<label>')
         .css({
@@ -2125,4 +2270,156 @@ function highlightGraphIssues(graphProperties) {
     highlightAcyclic(graphProperties.is_acyclic)
     highlightMultipleOutputConnection(graphProperties.multiple_output_connection)
     highlightPathWarning(graphProperties.path_warnings)
+}
+
+function setHashpathOnInput(clicked, associatedParamId, isValue) {
+    var path = $(clicked).data('hashpath')
+    if (isValue === true) {
+        var key = $(clicked).data('hashpath-key')
+        var value = $(clicked).data('hashpath-value')
+        path += getPathFiltering(key, value)
+    }
+    $('#selected-hashpath-input').val(path)
+    setValueOnAssociatedInput(associatedParamId, path)
+}
+
+function setValueOnAssociatedInput(associatedParamId, value) {
+    var $associatedInput = $drawflow.find('input[data-paramid="' + associatedParamId + '"]')
+    $associatedInput.val(value)
+    $associatedInput.trigger('input')
+}
+
+function getPathFiltering(key, value) {
+    var stringToFormat = $('#selected-hashpath-operator').val()
+    var filter = stringToFormat
+        .replace('{$key}', key)
+        .replace('{$value}', value)
+    return filter
+}
+
+function generateCoreFormatUI(event, associatedParamId) {
+    var indent_size = 'calc(1em)'
+    var color_key = '#005cd5'
+    var color_index = '#727272'
+    var color_string = '#cf5900'
+    var color_null = '#747474'
+    var color_bool = '#004bad'
+    var color_brace = '#727272'
+    var color_column = '#727272'
+
+    function generate(item, depth, path) {
+        if (Array.isArray(item)) {
+            var $container = $('<span>').append(
+                depth == 1 ? '' : braceOpen(true),
+                genArray(item, depth, path),
+                depth == 1 ? '' : braceClose(true),
+            )
+        } else if (typeof item === 'object' && item !== null) {
+            var $container = $('<span>').append(
+                depth == 1 ? '' : braceOpen(),
+                genObject(item, depth, path),
+                depth == 1 ? '' : braceClose(),
+            )
+        } else {
+            var $container = genValue(item, path)
+        }
+        return $container
+    }
+
+    function genArray(arr, depth, path) {
+        var $container = $('<div>')
+        arr.forEach(function (v, i) {
+            var nextPath = path + '.{n}'
+            var $index = genIndex(i, nextPath)
+            var $value = generate(v, depth + 1, nextPath)
+            var $div = $('<div>')
+            $div.append($index, column(), $value)
+            $container.append($div)
+        })
+        setDepth($container, depth, path)
+        return $container
+    }
+
+    function genObject(obj, depth, path) {
+        var $container = $('<div>')
+        Object.keys(obj).forEach(function (k) {
+            var nextPath = path + '.' + k
+            var v = obj[k]
+            var $key = genKey(k, nextPath)
+            var $value = generate(v, depth+1, nextPath)
+            var $div = $('<div>')
+            $div.append($key, column(), $value)
+            $container.append($div)
+        })
+        setDepth($container, depth)
+        return $container
+    }
+
+    function genValue(val, path) {
+        var exploded_path = path.split('.')
+        var path_without_last_key = exploded_path.slice(0, -1).join('.')
+        var key = exploded_path.pop()
+        var path_filtered_by_value = path_without_last_key// + addFiltering(key, val)
+        var $value
+        if (val === null) {
+            $value = $('<span>').text('null').css({'color': color_null })
+        } else if (typeof val === 'boolean') {
+            $value = $('<span>').text(val).css({'color': color_bool })
+        } else {
+            $value = $('<span>').text(val).css({'color': color_string })
+        }
+        $value
+            .addClass('selectable-value')
+            .attr('data-hashpath-key', key)
+            .attr('data-hashpath-value', val)
+            .attr('data-hashpath', path_filtered_by_value.slice(1))
+            .attr('onclick', 'setHashpathOnInput(this, "' + associatedParamId + '", true)')
+        return $value
+    }
+
+    function genKey(key, path) {
+        return $('<span>')
+            .text(key)
+            .css({
+                'color': color_key,
+                'font-weight': 'bold',
+            })
+            .addClass('selectable-key')
+            .attr('data-hashpath', path.slice(1))
+            .attr('onclick', 'setHashpathOnInput(this, "' + associatedParamId + '")')
+    }
+
+    function genIndex(i, path) {
+        return $('<span>')
+            .text(i)
+            .addClass('selectable-key')
+            .css({ 'color': color_index })
+            .attr('data-hashpath', path.slice(1))
+            .attr('onclick', 'setHashpathOnInput(this, "' + associatedParamId + '")')
+    }
+
+    function header() {
+        return $('<div>').append(braceOpen())
+    }
+
+    function footer() {
+        return $('<div>').append(braceClose())
+    }
+
+    function braceOpen(isArray) {
+        return $('<span>').text(isArray ? '[' : '{').css({ 'color': color_brace, margin: '0 0.25em' })
+    }
+    function braceClose(isArray) {
+        return $('<span>').text(isArray ? ']' : '}').css({ 'color': color_brace, margin: '0 0.25em' })
+    }
+    function column() {
+        return $('<span>').text(':').css({ 'color': color_column, margin: '0 0.25em' })
+    }
+    function setDepth($obj) {
+        $obj.css('margin-left', 'calc( ' + indent_size + ' )')
+    }
+
+    var $mainContainer = $('<div id="core-format-picker">')
+    $mainContainer.append(header(), generate(event, 1, ''), footer())
+    return $mainContainer
 }
\ No newline at end of file

From fad61eb3f550d2e5eb53954f8c57ee335fd0d12b Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Tue, 25 Jul 2023 15:46:45 +0200
Subject: [PATCH 68/96] chg: [workflow-modules] Replace param type for hashpath
 input to `hashpath`

---
 .../action/Module_assign_country_from_enrichment.php          | 4 ++--
 app/Model/WorkflowModules/action/Module_ms_teams_webhook.php  | 4 ++--
 app/Model/WorkflowModules/action/Module_push_zmq.php          | 3 ++-
 app/Model/WorkflowModules/action/Module_webhook.php           | 4 ++--
 .../WorkflowModules/logic/Module_generic_filter_data.php      | 3 ++-
 app/Model/WorkflowModules/logic/Module_generic_if.php         | 3 ++-
 6 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/app/Model/WorkflowModules/action/Module_assign_country_from_enrichment.php b/app/Model/WorkflowModules/action/Module_assign_country_from_enrichment.php
index 888dc20f7..5711f70d6 100644
--- a/app/Model/WorkflowModules/action/Module_assign_country_from_enrichment.php
+++ b/app/Model/WorkflowModules/action/Module_assign_country_from_enrichment.php
@@ -3,7 +3,7 @@ include_once APP . 'Model/WorkflowModules/action/Module_tag_operation.php';
 
 class Module_assign_country_from_enrichment extends Module_tag_operation
 {
-    public $version = '0.1';
+    public $version = '0.2';
     public $blocking = false;
     public $id = 'assign_country';
     public $name = 'Assign country';
@@ -36,7 +36,7 @@ class Module_assign_country_from_enrichment extends Module_tag_operation
             [
                 'id' => 'hash_path',
                 'label' => 'Country Hash path',
-                'type' => 'input',
+                'type' => 'hashpath',
                 'placeholder' => 'enrichment.{n}.{n}.values.0',
                 'default' => 'enrichment.{n}.{n}.values.0'
             ],
diff --git a/app/Model/WorkflowModules/action/Module_ms_teams_webhook.php b/app/Model/WorkflowModules/action/Module_ms_teams_webhook.php
index 667c2a483..4960f094a 100644
--- a/app/Model/WorkflowModules/action/Module_ms_teams_webhook.php
+++ b/app/Model/WorkflowModules/action/Module_ms_teams_webhook.php
@@ -5,7 +5,7 @@ class Module_ms_teams_webhook extends Module_webhook
 {
     public $id = 'ms-teams-webhook';
     public $name = 'MS Teams Webhook';
-    public $version = '0.3';
+    public $version = '0.4';
     public $description = 'Perform callbacks to the MS Teams webhook provided by the "Incoming Webhook" connector';
     public $icon_path = 'MS_Teams.png';
 
@@ -31,7 +31,7 @@ class Module_ms_teams_webhook extends Module_webhook
             [
                 'id' => 'data_extraction_path',
                 'label' => 'Data extraction path',
-                'type' => 'input',
+                'type' => 'hashpath',
                 'default' => '',
                 'placeholder' => 'Attribute.{n}.AttributeTag.{n}.Tag.name',
             ],
diff --git a/app/Model/WorkflowModules/action/Module_push_zmq.php b/app/Model/WorkflowModules/action/Module_push_zmq.php
index a4c852906..ddab32965 100644
--- a/app/Model/WorkflowModules/action/Module_push_zmq.php
+++ b/app/Model/WorkflowModules/action/Module_push_zmq.php
@@ -6,6 +6,7 @@ class Module_push_zmq extends WorkflowBaseActionModule
     public $blocking = false;
     public $id = 'push-zmq';
     public $name = 'Push to ZMQ';
+    public $version = '0.2';
     public $description = 'Push to the ZMQ channel';
     public $icon_path = 'zeromq.png';
     public $inputs = 1;
@@ -19,7 +20,7 @@ class Module_push_zmq extends WorkflowBaseActionModule
             [
                 'id' => 'data_extraction_path',
                 'label' => 'Data extraction path',
-                'type' => 'input',
+                'type' => 'hashpath',
                 'default' => '',
                 'placeholder' => 'Attribute.{n}.AttributeTag.{n}.Tag.name',
             ],
diff --git a/app/Model/WorkflowModules/action/Module_webhook.php b/app/Model/WorkflowModules/action/Module_webhook.php
index ef4e59d45..9baf9500e 100644
--- a/app/Model/WorkflowModules/action/Module_webhook.php
+++ b/app/Model/WorkflowModules/action/Module_webhook.php
@@ -8,7 +8,7 @@ class Module_webhook extends WorkflowBaseActionModule
 {
     public $id = 'webhook';
     public $name = 'Webhook';
-    public $version = '0.3';
+    public $version = '0.4';
     public $description = 'Allow to perform custom callbacks to the provided URL';
     public $icon_path = 'webhook.png';
     public $inputs = 1;
@@ -42,7 +42,7 @@ class Module_webhook extends WorkflowBaseActionModule
             [
                 'id' => 'data_extraction_path',
                 'label' => __('Data extraction path'),
-                'type' => 'input',
+                'type' => 'hashpath',
                 'default' => '',
                 'placeholder' => 'Attribute.{n}.AttributeTag.{n}.Tag.name',
             ],
diff --git a/app/Model/WorkflowModules/logic/Module_generic_filter_data.php b/app/Model/WorkflowModules/logic/Module_generic_filter_data.php
index 907ecafe7..581a029b9 100644
--- a/app/Model/WorkflowModules/logic/Module_generic_filter_data.php
+++ b/app/Model/WorkflowModules/logic/Module_generic_filter_data.php
@@ -5,6 +5,7 @@ class Module_generic_filter_data extends WorkflowFilteringLogicModule
 {
     public $id = 'generic-filter-data';
     public $name = 'Filter :: Generic';
+    public $version = '0.2';
     public $description = 'Generic data filtering block. The module filters incoming data and forward the matching data to its output.';
     public $icon = 'filter';
     public $inputs = 1;
@@ -66,7 +67,7 @@ class Module_generic_filter_data extends WorkflowFilteringLogicModule
             [
                 'id' => 'hash_path',
                 'label' => __('Hash path'),
-                'type' => 'input',
+                'type' => 'hashpath',
                 'placeholder' => 'Tag.name',
             ],
         ];
diff --git a/app/Model/WorkflowModules/logic/Module_generic_if.php b/app/Model/WorkflowModules/logic/Module_generic_if.php
index 728488844..4aafb778c 100644
--- a/app/Model/WorkflowModules/logic/Module_generic_if.php
+++ b/app/Model/WorkflowModules/logic/Module_generic_if.php
@@ -5,6 +5,7 @@ class Module_generic_if extends WorkflowBaseLogicModule
 {
     public $id = 'generic-if';
     public $name = 'IF :: Generic';
+    public $version = '0.2';
     public $description = 'Generic IF / ELSE condition block. The `then` output will be used if the encoded conditions is satisfied, otherwise the `else` output will be used.';
     public $icon = 'code-branch';
     public $inputs = 1;
@@ -54,7 +55,7 @@ class Module_generic_if extends WorkflowBaseLogicModule
             [
                 'id' => 'hash_path',
                 'label' => 'Hash path',
-                'type' => 'input',
+                'type' => 'hashpath',
                 'placeholder' => 'Attribute.{n}.Tag',
             ],
         ];

From a3b07ff49ffc131d4077bbe4ee46a0f786644bd3 Mon Sep 17 00:00:00 2001
From: Luciano Righetti <luciano.righetti@gmail.com>
Date: Wed, 26 Jul 2023 09:38:58 +0200
Subject: [PATCH 69/96] fix: light pagination bug in /attributes/search/results
 see #9157

---
 app/Controller/AttributesController.php | 1 +
 app/View/Attributes/index.ctp           | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php
index ac76e5e7b..76eca938e 100644
--- a/app/Controller/AttributesController.php
+++ b/app/Controller/AttributesController.php
@@ -1539,6 +1539,7 @@ class AttributesController extends AppController
         $user = $this->Auth->user();
         $exception = null;
         $filters = $this->__getSearchFilters($exception);
+        $this->set('passedArgsArray', ['results' => $continue]);
         if ($this->request->is('post') || !empty($this->request->params['named']['tags'])) {
             if ($filters === false) {
                 return $exception;
diff --git a/app/View/Attributes/index.ctp b/app/View/Attributes/index.ctp
index d40766ccd..acd33d09b 100755
--- a/app/View/Attributes/index.ctp
+++ b/app/View/Attributes/index.ctp
@@ -250,7 +250,8 @@ echo $this->element('/genericElements/IndexTable/index_table', [
                     return $this->Acl->canModifyEvent($object) && empty($object['Event']['publish_timestamp']);
                 },
             ]
-        ]
+        ],
+        'persistUrlParams' => ['results']
     ]
 ]);
 

From fdbb88505b2f52cd556cc456437527f4d662a4d1 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 26 Jul 2023 11:55:03 +0200
Subject: [PATCH 70/96] chg: [workflow:editor] Added support of collapse in
 hashpath picker

---
 app/webroot/css/workflows-editor.css          | 17 +++++
 .../js/workflows-editor/workflows-editor.js   | 62 ++++++++++++++++---
 2 files changed, 70 insertions(+), 9 deletions(-)

diff --git a/app/webroot/css/workflows-editor.css b/app/webroot/css/workflows-editor.css
index f0bed546a..0a354616b 100644
--- a/app/webroot/css/workflows-editor.css
+++ b/app/webroot/css/workflows-editor.css
@@ -357,6 +357,7 @@
     border: 1px solid #e1e1e1;
 }
 #core-format-picker .selectable-key {
+    font-weight: bold;
     padding: 2px 3px;
 }
 #core-format-picker .selectable-key:hover {
@@ -373,6 +374,22 @@
     cursor: pointer;
 }
 
+#core-format-picker .children-counter {
+    background-color: #6b6b6b;
+    color: #f2f2f2;
+    padding: 2px 5px;
+    margin: 0px 0.25rem;
+    font-size: smaller;
+    border-radius: 3px;
+}
+#core-format-picker .collaspe-button {
+    cursor: pointer;
+}
+#core-format-picker .collaspe-button:hover {
+    color: #292929;
+    cursor: pointer;
+}
+
 /* Custom input/output CSS */
 /* .drawflow .drawflow-node.expect-misp-core-format>.inputs>.input {
     background-color: #2fa1db;
diff --git a/app/webroot/js/workflows-editor/workflows-editor.js b/app/webroot/js/workflows-editor/workflows-editor.js
index 67d25aa9d..0c553ec5b 100644
--- a/app/webroot/js/workflows-editor/workflows-editor.js
+++ b/app/webroot/js/workflows-editor/workflows-editor.js
@@ -2298,7 +2298,7 @@ function getPathFiltering(key, value) {
 }
 
 function generateCoreFormatUI(event, associatedParamId) {
-    var indent_size = 'calc(1em)'
+    var indent_size = 'calc(1.25em)'
     var color_key = '#005cd5'
     var color_index = '#727272'
     var color_string = '#cf5900'
@@ -2306,20 +2306,30 @@ function generateCoreFormatUI(event, associatedParamId) {
     var color_bool = '#004bad'
     var color_brace = '#727272'
     var color_column = '#727272'
+    var color_collapse = '#727272'
+    var defaultCollapseList = ['Org', 'Orgc']
 
-    function generate(item, depth, path) {
+    function generate(item, depth, path, forceObjectCollaspe) {
         if (Array.isArray(item)) {
             var $container = $('<span>').append(
                 depth == 1 ? '' : braceOpen(true),
+                depth == 1 ? '' : childrenCount(item),
                 genArray(item, depth, path),
                 depth == 1 ? '' : braceClose(true),
             )
+            if (depth > 2) {
+                $container.children("div").toggleClass("hidden")
+            }
         } else if (typeof item === 'object' && item !== null) {
             var $container = $('<span>').append(
                 depth == 1 ? '' : braceOpen(),
+                depth == 1 ? '' : childrenCount(item),
                 genObject(item, depth, path),
                 depth == 1 ? '' : braceClose(),
             )
+            if (forceObjectCollaspe === true) {
+                $container.children("div").toggleClass("hidden")
+            }
         } else {
             var $container = genValue(item, path)
         }
@@ -2345,10 +2355,18 @@ function generateCoreFormatUI(event, associatedParamId) {
         Object.keys(obj).forEach(function (k) {
             var nextPath = path + '.' + k
             var v = obj[k]
+            var forceCollaspe = defaultCollapseList.includes(k)
             var $key = genKey(k, nextPath)
-            var $value = generate(v, depth+1, nextPath)
+            var $value = generate(v, depth+1, nextPath, forceCollaspe)
             var $div = $('<div>')
-            $div.append($key, column(), $value)
+            var $collase = ''
+            if (isIterable(v)) {
+                $collase = collapseIcon()
+                if (depth > 1 && (Array.isArray(v) || forceCollaspe)) {
+                    $collase.addClass('fa-rotate-270')
+                }
+            }
+            $div.append($collase, $key, column(), $value)
             $container.append($div)
         })
         setDepth($container, depth)
@@ -2359,7 +2377,7 @@ function generateCoreFormatUI(event, associatedParamId) {
         var exploded_path = path.split('.')
         var path_without_last_key = exploded_path.slice(0, -1).join('.')
         var key = exploded_path.pop()
-        var path_filtered_by_value = path_without_last_key// + addFiltering(key, val)
+        var path_filtered_by_value = path_without_last_key
         var $value
         if (val === null) {
             $value = $('<span>').text('null').css({'color': color_null })
@@ -2380,10 +2398,7 @@ function generateCoreFormatUI(event, associatedParamId) {
     function genKey(key, path) {
         return $('<span>')
             .text(key)
-            .css({
-                'color': color_key,
-                'font-weight': 'bold',
-            })
+            .css({ 'color': color_key })
             .addClass('selectable-key')
             .attr('data-hashpath', path.slice(1))
             .attr('onclick', 'setHashpathOnInput(this, "' + associatedParamId + '")')
@@ -2406,6 +2421,21 @@ function generateCoreFormatUI(event, associatedParamId) {
         return $('<div>').append(braceClose())
     }
 
+    function collapseIcon() {
+        return $('<i>')
+            .addClass(['fas fa-caret-down', 'collaspe-button'])
+            .css({ 'color': color_collapse, 'margin-right': '0.25rem', 'font-size': '1.25em' })
+            .attr('onclick', '$(this).toggleClass("fa-rotate-270").parent().children().last().children("div").toggleClass("hidden")')
+    }
+    function childrenCount(iterable) {
+        var count = getChildrenCount(iterable)
+        var $span = $('<span>').text(count).addClass('children-counter')
+        if (count === 0) {
+            $span.css('background-color', '#a3a3a3')
+        }
+        return $span
+    }
+
     function braceOpen(isArray) {
         return $('<span>').text(isArray ? '[' : '{').css({ 'color': color_brace, margin: '0 0.25em' })
     }
@@ -2415,10 +2445,24 @@ function generateCoreFormatUI(event, associatedParamId) {
     function column() {
         return $('<span>').text(':').css({ 'color': color_column, margin: '0 0.25em' })
     }
+
     function setDepth($obj) {
         $obj.css('margin-left', 'calc( ' + indent_size + ' )')
     }
 
+    function isIterable(obj) {
+        return typeof obj === 'object' && obj !== null
+    }
+    function getChildrenCount(iterable) {
+        var count = 0
+        if (Array.isArray(iterable)) {
+            count = iterable.length
+        } else if (typeof iterable === 'object') {
+            count = Object.keys(iterable).length
+        }
+        return count
+    }
+
     var $mainContainer = $('<div id="core-format-picker">')
     $mainContainer.append(header(), generate(event, 1, ''), footer())
     return $mainContainer

From 79a1a88b53d4412cc3a4a8db1935d1bbeb569b09 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 26 Jul 2023 14:16:40 +0200
Subject: [PATCH 71/96] new: [workflow:editor] Added min/max-imize support for
 module sidebar

---
 app/View/Workflows/editor.ctp        | 12 +++++++++++
 app/webroot/css/workflows-editor.css | 30 ++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+)

diff --git a/app/View/Workflows/editor.ctp b/app/View/Workflows/editor.ctp
index d575f3263..05fe31c06 100644
--- a/app/View/Workflows/editor.ctp
+++ b/app/View/Workflows/editor.ctp
@@ -71,6 +71,12 @@ $debugEnabled = !empty($selectedWorkflow['Workflow']['debug_enabled']);
     <div class="main-container">
         <div class="sidebar">
             <div class="side-panel">
+                <span class="sidebar-minimize-button">
+                    <i class="<?= $this->FontAwesome->getClass('angle-double-left') ?>"></i>
+                </span>
+                <span class="sidebar-maximize-button">
+                    <i class="<?= $this->FontAwesome->getClass('angle-double-right') ?>"></i>
+                </span>
                 <ul class="nav nav-tabs" id="block-tabs">
                     <li class="active">
                         <a href="#container-actions">
@@ -335,5 +341,11 @@ echo $this->element('genericElements/assetLoader', [
 
     $(document).ready(function() {
         initDrawflow()
+        $('.sidebar-minimize-button').click(function() {
+            $(this).closest('.sidebar').addClass('minimized')
+        })
+        $('.sidebar-maximize-button').click(function() {
+            $(this).closest('.sidebar').removeClass('minimized')
+        })
     })
 </script>
\ No newline at end of file
diff --git a/app/webroot/css/workflows-editor.css b/app/webroot/css/workflows-editor.css
index 0a354616b..eba7b35ab 100644
--- a/app/webroot/css/workflows-editor.css
+++ b/app/webroot/css/workflows-editor.css
@@ -41,6 +41,11 @@
     flex-direction: column;
     position: absolute;
     top: var(--topbar-height)px;
+    left: 0;
+    transition: left 0.15s ease-out;
+}
+.sidebar.minimized {
+    left: calc(-1 * var(--editor-sidepanel-width));
 }
 
 .side-panel {
@@ -63,6 +68,31 @@
     top: 0;
 }
 
+.sidebar-minimize-button {
+    position: absolute;
+    top: 3px;
+    right: 5px;
+    cursor: pointer;
+}
+.sidebar-maximize-button {
+    position: absolute;
+    font-size: 1.25rem;
+    top: 3px;
+    right: -32px;
+    cursor: pointer;
+    background-color: #fff;
+    padding: 5px;
+    border-radius: 3px;
+    border: 1px #e3e3e3 solid;
+    display: none;
+    opacity: 0;
+    transition: opacity 0.15s ease-out;
+}
+.sidebar.minimized .sidebar-maximize-button {
+    display: inline-block;
+    opacity: 1;
+}
+
 .side-panel #block-tabs {
     margin-bottom: 10px;
 }

From ff5707cd652c387a65d4b8065991b85aaecc1192 Mon Sep 17 00:00:00 2001
From: Luciano Righetti <luciano.righetti@gmail.com>
Date: Thu, 27 Jul 2023 08:28:55 +0200
Subject: [PATCH 72/96] chg: do not show last button when using light paginator

---
 app/View/Helper/LightPaginatorHelper.php | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/app/View/Helper/LightPaginatorHelper.php b/app/View/Helper/LightPaginatorHelper.php
index 9857a486d..87cba5d11 100644
--- a/app/View/Helper/LightPaginatorHelper.php
+++ b/app/View/Helper/LightPaginatorHelper.php
@@ -13,6 +13,11 @@ class LightPaginatorHelper extends PaginatorHelper
         return '';
     }
 
+    public function last($last = 'last >>', $options = array())
+    {
+        return '';
+    }
+
     public function hasNext($model = null)
     {
         $model = $this->defaultModel();

From 4bec6bfca4fc23a70ccb8385d660bf7cc26da6a1 Mon Sep 17 00:00:00 2001
From: ref <56499429+referefref@users.noreply.github.com>
Date: Thu, 27 Jul 2023 15:17:15 +0800
Subject: [PATCH 73/96] Added James Brine Bruteforce IPs to feed-metadata
 defaults json

Added freetext feed endpoint for Bruteforce IPV4 addresses
---
 app/files/feed-metadata/defaults.json | 26 ++++++++++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/app/files/feed-metadata/defaults.json b/app/files/feed-metadata/defaults.json
index cac087c92..680718aa7 100644
--- a/app/files/feed-metadata/defaults.json
+++ b/app/files/feed-metadata/defaults.json
@@ -1663,5 +1663,31 @@
         "exportable": true,
         "hide_tag": false
     }
+  },
+  {
+    "Feed": {
+        "name": "James Brine Bruteforce IPs",
+        "provider": "jamesbrine.com.au",
+        "url": "https://jamesbrine.com.au/csv",
+        "rules": "{\"tags\":{\"OR\":[],\"NOT\":[]},\"orgs\":{\"OR\":[],\"NOT\":[]},\"type_attributes\":{\"NOT\":[]},\"type_objects\":{\"NOT\":[]},\"url_params\":\"\"}",
+        "enabled": true,
+        "distribution": "3",
+        "default": false,
+        "source_format": "freetext",
+        "fixed_event": true,
+        "delta_merge": false,
+        "publish": false,
+        "override_ids": false,
+        "settings": "{\"disable_correlation\":\"0\",\"csv\":{\"value\":\"\",\"delimiter\":\"\"},\"common\":{\"excluderegex\":\"\\\"\\/\\\\W*,.*:\\/\\\"\"}}",
+        "input_source": "network",
+        "delete_local_file": false,
+        "lookup_visible": false
+    },
+    "Tag": {
+        "name": "osint:source-type=\"block-or-filter-list\"",
+        "colour": "#004f89",
+        "exportable": true,
+        "hide_tag": false
+    }
   }
 ]

From 68ca75359345aefe21342485226e7cbb114ddf56 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 27 Jul 2023 10:24:54 +0200
Subject: [PATCH 74/96] chg: [workflow:editor] Improved description in hashpath
 picker for quick link

---
 app/webroot/js/workflows-editor/workflows-editor.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/webroot/js/workflows-editor/workflows-editor.js b/app/webroot/js/workflows-editor/workflows-editor.js
index 0c553ec5b..73cfc4ec1 100644
--- a/app/webroot/js/workflows-editor/workflows-editor.js
+++ b/app/webroot/js/workflows-editor/workflows-editor.js
@@ -145,7 +145,7 @@ var iconBySeverity = {
 var severities = ['info', 'warning', 'error']
 var haspathQuickPickMenu = [
     { 'name': 'All Attributes', 'path': 'Event._AttributeFlattened.{n}' },
-    { 'name': 'All tags attached to Attributes', 'path': 'Event._AttributeFlattened.{n}.Tag.{n}.name' },
+    { 'name': 'All tags attached to all Attributes', 'path': 'Event._AttributeFlattened.{n}.Tag.{n}.name' },
     { 'name': 'All tags attached to the Event', 'path': 'Event.Tag.{n}.name' },
 ]
 

From 64580168622aeea59997cea5739cf0b8dbcf8bda Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 27 Jul 2023 13:22:07 +0200
Subject: [PATCH 75/96] chg: [event:publishSightingsRouter] Change from prio
 worker to default

There is no need to keep this task in prio as sightings are not sync inline anymore.
---
 app/Model/Event.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Model/Event.php b/app/Model/Event.php
index 724b39ea2..22703a13c 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -4501,7 +4501,7 @@ class Event extends AppModel
             /** @var Job $job */
             $job = ClassRegistry::init('Job');
             $message = empty($sightingUuids) ? __('Publishing sightings.') : __('Publishing %s sightings.', count($sightingUuids));
-            $jobId = $job->createJob($user, Job::WORKER_PRIO, 'publish_event', "Event ID: $id", $message);
+            $jobId = $job->createJob($user, Job::WORKER_DEFAULT, 'publish_event', "Event ID: $id", $message);
 
             $args = ['publish_sightings', $id, $passAlong, $jobId, $user['id']];
             if (!empty($sightingUuids)) {

From 262e4e2af9269e2b217994a4058b0c0cd4f0d1c1 Mon Sep 17 00:00:00 2001
From: ref <56499429+referefref@users.noreply.github.com>
Date: Thu, 27 Jul 2023 22:59:39 +0800
Subject: [PATCH 76/96] Changed feed type to csv and added field identifier
 value and delimiter

---
 app/files/feed-metadata/defaults.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/files/feed-metadata/defaults.json b/app/files/feed-metadata/defaults.json
index 680718aa7..1b63cbe39 100644
--- a/app/files/feed-metadata/defaults.json
+++ b/app/files/feed-metadata/defaults.json
@@ -1673,12 +1673,12 @@
         "enabled": true,
         "distribution": "3",
         "default": false,
-        "source_format": "freetext",
+        "source_format": "csv",
         "fixed_event": true,
         "delta_merge": false,
         "publish": false,
         "override_ids": false,
-        "settings": "{\"disable_correlation\":\"0\",\"csv\":{\"value\":\"\",\"delimiter\":\"\"},\"common\":{\"excluderegex\":\"\\\"\\/\\\\W*,.*:\\/\\\"\"}}",
+        "settings": "{\"disable_correlation\":\"0\",\"csv\":{\"value\":\"1\",\"delimiter\":\"\"},\"common\":{\"excluderegex\":\"\"}}",
         "input_source": "network",
         "delete_local_file": false,
         "lookup_visible": false

From 9ddc335c07e073412d8744017ce284700e61306e Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Fri, 28 Jul 2023 08:39:10 +0200
Subject: [PATCH 77/96] new: [workflow:editor] Added support of frame nodes in
 editor and drawflow lib

---
 app/Lib/Tools/WorkflowGraphTool.php           |  34 ++-
 app/Model/Workflow.php                        |   3 +
 app/View/Workflows/editor.ctp                 |   4 +
 app/webroot/css/workflows-editor.css          |   7 +
 app/webroot/js/drawflow.js                    | 251 +++++++++++++++++-
 .../js/workflows-editor/workflows-editor.js   |  62 ++++-
 6 files changed, 348 insertions(+), 13 deletions(-)

diff --git a/app/Lib/Tools/WorkflowGraphTool.php b/app/Lib/Tools/WorkflowGraphTool.php
index 915df1e89..60c7b8382 100644
--- a/app/Lib/Tools/WorkflowGraphTool.php
+++ b/app/Lib/Tools/WorkflowGraphTool.php
@@ -6,16 +6,18 @@ class GraphUtil
 {
     public function __construct($graphData)
     {
-        $this->graph = $graphData;
+        $this->graph = array_filter($graphData, function($i) {
+            return $i != '_frames';
+        }, ARRAY_FILTER_USE_KEY);
         $this->numberNodes = count($this->graph);
-        $this->edgeList = $this->_buildEdgeList($graphData);
+        $this->edgeList = $this->_buildEdgeList($this->graph);
         $this->properties = [];
     }
 
     private function _buildEdgeList($graphData): array
     {
         $list = [];
-        foreach ($graphData as $node) {
+        foreach ($graphData as $i => $node) {
             $list[(int)$node['id']] = [];
             foreach (($node['outputs'] ?? []) as $output_id => $outputs) {
                 foreach ($outputs as $connections) {
@@ -356,6 +358,20 @@ class WorkflowRoamingData
 
 class WorkflowGraphTool
 {
+
+    /**
+     * cleanGraphData Remove frame nodes from the graph data
+     *
+     * @param  array $graphData
+     * @return array
+     */
+    public static function cleanGraphData(array $graphData): array
+    {
+        return array_filter($graphData, function($i) {
+            return $i != '_frames';
+        }, ARRAY_FILTER_USE_KEY);
+    }
+
     /**
      * extractTriggerFromWorkflow Return the trigger id (or full module) that are specified in the workflow
      *
@@ -382,8 +398,9 @@ class WorkflowGraphTool
      */
     public static function extractTriggersFromWorkflow(array $graphData, bool $fullNode = false): array
     {
+        $graphData = self::cleanGraphData($graphData);
         $triggers = [];
-        foreach ($graphData as $node) {
+        foreach ($graphData as $i => $node) {
             if ($node['data']['module_type'] == 'trigger') {
                 if (!empty($fullNode)) {
                     $triggers[] = $node;
@@ -404,8 +421,9 @@ class WorkflowGraphTool
      */
     public static function extractConcurrentTasksFromWorkflow(array $graphData, bool $fullNode = false): array
     {
+        $graphData = self::cleanGraphData($graphData);
         $nodes = [];
-        foreach ($graphData as $node) {
+        foreach ($graphData as $i => $node) {
             if ($node['data']['module_type'] == 'logic' && $node['data']['id'] == 'concurrent-task') {
                 if (!empty($fullNode)) {
                     $nodes[] = $node;
@@ -426,8 +444,9 @@ class WorkflowGraphTool
      */
     public static function extractFilterNodesFromWorkflow(array $graphData, bool $fullNode = false): array
     {
+        $graphData = self::cleanGraphData($graphData);
         $nodes = [];
-        foreach ($graphData as $node) {
+        foreach ($graphData as $i => $node) {
             if ($node['data']['module_type'] == 'logic' && $node['data']['id'] == 'generic-filter-data') {
                 if (!empty($fullNode)) {
                     $nodes[] = $node;
@@ -448,8 +467,9 @@ class WorkflowGraphTool
      */
     public static function extractResetFilterFromWorkflow(array $graphData, bool $fullNode = false): array
     {
+        $graphData = self::cleanGraphData($graphData);
         $nodes = [];
-        foreach ($graphData as $node) {
+        foreach ($graphData as $i => $node) {
             if ($node['data']['module_type'] == 'logic' && $node['data']['id'] == 'generic-filter-reset') {
                 if (!empty($fullNode)) {
                     $nodes[] = $node;
diff --git a/app/Model/Workflow.php b/app/Model/Workflow.php
index 4263ada24..27e22b865 100644
--- a/app/Model/Workflow.php
+++ b/app/Model/Workflow.php
@@ -1373,6 +1373,9 @@ class Workflow extends AppModel
         }
         $labelsByNodes = $this->getLabelsForConnections($workflow, $trigger_id);
         foreach ($graphData as $i => $node) {
+            if ($i == '_frames') {
+                continue;
+            }
             if (!empty($labelsByNodes[$node['id']])) {
                 foreach ($node['inputs'] as $inputName => $inputs) {
                     foreach ($inputs['connections'] as $j => $connection) {
diff --git a/app/View/Workflows/editor.ctp b/app/View/Workflows/editor.ctp
index 05fe31c06..a52b6c8b6 100644
--- a/app/View/Workflows/editor.ctp
+++ b/app/View/Workflows/editor.ctp
@@ -179,6 +179,9 @@ $debugEnabled = !empty($selectedWorkflow['Workflow']['debug_enabled']);
                     <button id="control-duplicate" class="btn btn-small btn-primary disabled" type="button" title="<?= __('Duplicate') ?>">
                         <i class="fa-fw <?= $this->FontAwesome->getClass('clone') ?>"></i> <?= __('Duplicate') ?>
                     </button>
+                    <button id="control-frame-node" class="btn btn-small btn-primary disabled" type="button" title="<?= __('Create frame node') ?>">
+                        <i class="fa-fw <?= $this->FontAwesome->getClass('object-group') ?>"></i> <?= __('Frame') ?>
+                    </button>
                     <button id="control-delete" class="btn btn-small btn-danger disabled" type="button" title="<?= __('Delete') ?>">
                         <i class="fa-fw <?= $this->FontAwesome->getClass('trash') ?>"></i> <?= __('Delete') ?>
                     </button>
@@ -309,6 +312,7 @@ echo $this->element('genericElements/assetLoader', [
     var $blockNotificationModal = $('#block-notifications-modal')
     var $blockFilteringModal = $('#block-filtering-modal')
     var $controlDuplicateButton = $('.control-buttons #control-duplicate')
+    var $controlFrameNodeButton = $('.control-buttons #control-frame-node')
     var $controlDeleteButton = $('.control-buttons #control-delete')
     var $controlExportBlocksLi = $('.control-buttons #control-export-blocks')
     var $controlSaveBlocksLi = $('.control-buttons #control-save-blocks')
diff --git a/app/webroot/css/workflows-editor.css b/app/webroot/css/workflows-editor.css
index eba7b35ab..e3a1a16ab 100644
--- a/app/webroot/css/workflows-editor.css
+++ b/app/webroot/css/workflows-editor.css
@@ -425,6 +425,13 @@
     background-color: #2fa1db;
 } */
 
+.drawflow .drawflow-framenode.selected {
+    box-shadow: 0 0 5px 2px #33333366;
+}
+.drawflow .drawflow-framenode .drawflow-framenode-text {
+    cursor: text;
+}
+
 .drawflow .drawflow-node > .inputs > .input::before {
     line-height: var(--dfInputHeight);
     vertical-align: top;
diff --git a/app/webroot/js/drawflow.js b/app/webroot/js/drawflow.js
index a55fac6d6..66c94794d 100644
--- a/app/webroot/js/drawflow.js
+++ b/app/webroot/js/drawflow.js
@@ -6,6 +6,7 @@ class Drawflow {
     this.nodeId = 1;
     this.ele_selected = null;
     this.node_selected = null;
+    this.framenode_selected = null;
     this.drag = false;
     this.reroute = false;
     this.reroute_fix_curvature = false;
@@ -32,10 +33,12 @@ class Drawflow {
     this.draggable_inputs = true;
     this.useuuid = false;
     this.parent = parent;
+    this.frame_to_nodes_registry = {};
+    this.nodes_to_frames_registry = {};
 
     this.noderegister = {};
     this.render = render;
-    this.drawflow = { "drawflow": { "Home": { "data": {} } } };
+    this.drawflow = { "drawflow": { "Home": { "data": { _frames: {}} } } };
     // Configurable options
     this.module = 'Home';
     this.editor_mode = 'edit';
@@ -139,7 +142,9 @@ class Drawflow {
   /* End Mobile Zoom */
   load() {
     for (var key in this.drawflow.drawflow[this.module].data) {
-      this.addNodeImport(this.drawflow.drawflow[this.module].data[key], this.precanvas);
+      if (!key.startsWith('_')) {
+        this.addNodeImport(this.drawflow.drawflow[this.module].data[key], this.precanvas);
+      }
     }
 
     if (this.reroute) {
@@ -212,6 +217,10 @@ class Drawflow {
           this.removeReouteConnectionSelected();
           this.connection_selected = null;
         }
+        if (this.framenode_selected != null) {
+          this.framenode_selected.classList.remove("selected");
+          this.framenode_selected = null
+        }
         if (this.node_selected != this.ele_selected) {
           this.dispatch('nodeSelected', this.ele_selected.id.slice(5));
         }
@@ -239,8 +248,30 @@ class Drawflow {
           this.removeReouteConnectionSelected();
           this.connection_selected = null;
         }
+        if (this.framenode_selected != null) {
+          this.framenode_selected.classList.remove("selected");
+          this.framenode_selected = null
+        }
         this.drawConnection(e.target);
         break;
+      case 'drawflow-framenode':
+        if (this.node_selected != null) {
+          this.node_selected.classList.remove("selected");
+          this.node_selected = null
+          this.dispatch('nodeUnselected', true);
+        }
+        if (this.connection_selected != null) {
+          this.connection_selected.classList.remove("selected");
+          this.removeReouteConnectionSelected();
+          this.connection_selected = null;
+        }
+        if (this.framenode_selected != null) {
+          this.framenode_selected.classList.remove("selected");
+          this.framenode_selected = null
+        }
+        this.framenode_selected = this.ele_selected;
+        this.framenode_selected.classList.add("selected");
+        break;
       case 'parent-drawflow':
         if (this.node_selected != null) {
           this.node_selected.classList.remove("selected");
@@ -252,6 +283,10 @@ class Drawflow {
           this.removeReouteConnectionSelected();
           this.connection_selected = null;
         }
+        if (this.framenode_selected != null) {
+          this.framenode_selected.classList.remove("selected");
+          this.framenode_selected = null
+        }
         this.editor_selected = true;
         break;
       case 'drawflow':
@@ -265,6 +300,10 @@ class Drawflow {
           this.removeReouteConnectionSelected();
           this.connection_selected = null;
         }
+        if (this.framenode_selected != null) {
+          this.framenode_selected.classList.remove("selected");
+          this.framenode_selected = null
+        }
         this.editor_selected = true;
         break;
       case 'main-path':
@@ -273,6 +312,10 @@ class Drawflow {
           this.node_selected = null;
           this.dispatch('nodeUnselected', true);
         }
+        if (this.framenode_selected != null) {
+          this.framenode_selected.classList.remove("selected");
+          this.framenode_selected = null
+        }
         if (this.connection_selected != null) {
           this.connection_selected.classList.remove("selected");
           this.removeReouteConnectionSelected();
@@ -303,6 +346,10 @@ class Drawflow {
           this.removeConnection();
         }
 
+        if (this.framenode_selected) {
+          this.removeFrameNodeId(this.framenode_selected.id);
+        }
+
         if (this.node_selected != null) {
           this.node_selected.classList.remove("selected");
           this.node_selected = null;
@@ -313,6 +360,10 @@ class Drawflow {
           this.removeReouteConnectionSelected();
           this.connection_selected = null;
         }
+        if (this.framenode_selected != null) {
+          this.framenode_selected.classList.remove("selected");
+          this.framenode_selected = null
+        }
 
         break;
       default:
@@ -543,6 +594,21 @@ class Drawflow {
 
       }
 
+    } else {
+      if (e.target.classList.contains('drawflow-framenode') || e.target.classList.contains('drawflow-framenode-text')) {
+        const frameNode = e.target.classList.contains('drawflow-framenode') ? e.target : e.target.parentElement
+        const deletebox = document.createElement('div');
+        deletebox.classList.add("drawflow-delete");
+        deletebox.innerHTML = "x";
+
+        const frameNodeBCR = frameNode.getBoundingClientRect()
+        const pos_y = frameNodeBCR.top - 30 // roughly include margin and node size
+        const pos_x = frameNodeBCR.left + frameNodeBCR.width
+        deletebox.style.top = pos_y * (this.precanvas.clientHeight / (this.precanvas.clientHeight * this.zoom)) - (this.precanvas.getBoundingClientRect().y * (this.precanvas.clientHeight / (this.precanvas.clientHeight * this.zoom))) + "px";
+        deletebox.style.left = pos_x * (this.precanvas.clientWidth / (this.precanvas.clientWidth * this.zoom)) - (this.precanvas.getBoundingClientRect().x * (this.precanvas.clientWidth / (this.precanvas.clientWidth * this.zoom))) + "px";
+
+        this.precanvas.appendChild(deletebox);
+      }
     }
 
   }
@@ -1092,6 +1158,16 @@ class Drawflow {
 
       }
     })
+
+    var nodeIdInt = parseInt(id.slice(5));
+    if (this.nodes_to_frames_registry[nodeIdInt] !== undefined && this.nodes_to_frames_registry[nodeIdInt].length > 0) {
+      var _this = this;
+      this.nodes_to_frames_registry[nodeIdInt].forEach((frameNodeUuid) => {
+        const containedNodeIDs = _this.frame_to_nodes_registry[frameNodeUuid]
+        const frameNode = _this.container.querySelector('#framenode-' + frameNodeUuid)
+        _this.updateFrameNodePosition(frameNode, containedNodeIDs);
+      })
+    }
   }
 
   dblclick(e) {
@@ -1516,6 +1592,133 @@ class Drawflow {
     });
   }
 
+  addFrameNode(frameNodeConfig) {
+    frameNodeConfig.nodes = [...new Set(frameNodeConfig.nodes)]
+    if (frameNodeConfig.nodes.length == 0) {
+      return
+    }
+
+    var newNodeId = this.getUuid();
+    this.frame_to_nodes_registry[newNodeId] = frameNodeConfig.nodes
+    frameNodeConfig.nodes.forEach(nodeId => {
+      if (this.nodes_to_frames_registry[nodeId] === undefined) {
+        this.nodes_to_frames_registry[nodeId] = []
+      }
+      this.nodes_to_frames_registry[nodeId].push(newNodeId)
+    });
+    const parent = document.createElement('div');
+    parent.classList.add(...["parent-framenode"]);
+
+    const frameNode = document.createElement('div');
+    frameNode.innerHTML = '';
+    frameNode.setAttribute('id', 'framenode-' + newNodeId);
+    frameNode.classList.add(...['drawflow-framenode']);
+    if (frameNodeConfig.class !== undefined && frameNodeConfig.class != '') {
+      frameNode.classList.add(...frameNodeConfig.class.split(' '));
+    }
+
+    frameNode.style.backgroundColor = '#b5b5b570';
+    frameNode.style.border = '1px solid #333333aa';
+    frameNode.style.position = 'absolute'
+    frameNode.style['text-align'] = 'center'
+    frameNode.style['z-index'] = -1
+    frameNode.style['border-radius'] = '5px'
+    this.updateFrameNodePosition(frameNode, frameNodeConfig.nodes)
+
+    const frameTextNode = document.createElement('span');
+    frameTextNode.innerText = frameNodeConfig.text;
+    frameTextNode.style['line-height'] = '1.5em'
+    frameTextNode.style.color = '#505050'
+    frameTextNode.style['font-size'] = '1.5em'
+    frameTextNode.style['font-weight'] = 'bold'
+    frameTextNode.classList.add('drawflow-framenode-text')
+    frameNode.appendChild(frameTextNode);
+
+    parent.appendChild(frameNode);
+    this.precanvas.appendChild(parent);
+
+    var json = {
+      id: newNodeId,
+      text: frameNodeConfig.text,
+      nodes: frameNodeConfig.nodes,
+      class: frameNodeConfig.class,
+    }
+    this.drawflow.drawflow[this.module].data._frames[newNodeId] = json;
+    this.dispatch('frameNodeCreated', newNodeId);
+  }
+
+  updateFrameNodePosition(frameNode, nodeIDs) {
+    if (!frameNode) {
+      return
+    }
+
+    var offsetPadding = {
+      top: 100,
+      left: 30,
+      right: 30,
+      bottom: 20,
+    }
+
+    var _this = this
+    var nodesHtml = nodeIDs.map(function (id) {
+      return _this.container.querySelector('#node-' + id)
+    })
+
+    function getFrameCoordinatesFromNodes(nodesHtml) {
+      var topLeft = { x: Number.MAX_SAFE_INTEGER, y: Number.MAX_SAFE_INTEGER }
+      var bottomRight = { x: Number.MIN_SAFE_INTEGER, y: Number.MIN_SAFE_INTEGER }
+      nodesHtml.forEach(function (nodeHtml) {
+        var bcr = nodeHtml.getBoundingClientRect()
+        topLeft.x = Math.min(topLeft.x, bcr.left)
+        topLeft.y = Math.min(topLeft.y, bcr.top)
+        bottomRight.x = Math.max(bottomRight.x, bcr.left + bcr.width)
+        bottomRight.y = Math.max(bottomRight.y, bcr.top + bcr.height)
+      })
+      return { topLeft: topLeft, bottomRight: bottomRight }
+    }
+
+    var frameCoordinates = getFrameCoordinatesFromNodes(nodesHtml)
+
+    var pos_x = frameCoordinates.topLeft.x - offsetPadding.left / 2
+    var pos_y = frameCoordinates.topLeft.y - offsetPadding.top / 2
+    var frameNodeHeight = (frameCoordinates.bottomRight.y - frameCoordinates.topLeft.y) + (offsetPadding.top / 2 + offsetPadding.bottom)
+    var frameNodeWidth = (frameCoordinates.bottomRight.x - frameCoordinates.topLeft.x) + (offsetPadding.left / 2 + offsetPadding.right)
+
+    var transpositionFactorX = (this.precanvas.clientWidth / (this.precanvas.clientWidth * this.zoom))
+    var transpositionOffsetX = -(this.precanvas.getBoundingClientRect().x * (this.precanvas.clientWidth / (this.precanvas.clientWidth * this.zoom)))
+    var transpositionFactorY = (this.precanvas.clientHeight / (this.precanvas.clientHeight * this.zoom))
+    var transpositionOffsetY = -(this.precanvas.getBoundingClientRect().y * (this.precanvas.clientHeight / (this.precanvas.clientHeight * this.zoom)))
+
+    pos_x = pos_x * transpositionFactorX + transpositionOffsetX
+    pos_y = pos_y * transpositionFactorY + transpositionOffsetY
+    frameNodeWidth = frameNodeWidth * transpositionFactorX
+    frameNodeHeight = frameNodeHeight * transpositionFactorY
+
+    frameNode.style.top = pos_y + 'px';
+    frameNode.style.left = pos_x + 'px';
+    frameNode.style.height = frameNodeHeight + 'px';
+    frameNode.style.width = frameNodeWidth + 'px';
+  }
+
+  updateFramenode(frameNodeUuid, frameNodeConfig) {
+    const frameNode = document.getElementById(frameNodeUuid);
+    frameNode.classList.remove(...frameNode.classList)
+    frameNode.classList.add(...['drawflow-framenode']);
+    if (frameNodeConfig.class !== undefined && frameNodeConfig.class != '') {
+      frameNode.classList.add(...frameNodeConfig.class.split(' '));
+    }
+
+    const frameTextNode = frameNode.querySelector('.drawflow-framenode-text')
+    frameTextNode.innerText = frameNodeConfig.text
+
+    var oldJson = this.drawflow.drawflow[this.module].data._frames[frameNodeUuid.slice(10)]
+    oldJson.text = frameNodeConfig.text
+    oldJson.class = frameNodeConfig.class
+    this.drawflow.drawflow[this.module].data._frames[frameNodeUuid.slice(10)] = oldJson;
+
+    this.dispatch('frameNodeUpdated', frameNodeUuid.slice(10));
+  }
+
   updateNodeValue(event) {
     var attr = event.target.attributes
     for (var i = 0; i < attr.length; i++) {
@@ -1769,6 +1972,25 @@ class Drawflow {
     if (this.module === moduleName) {
       this.container.querySelector(`#${id}`).remove();
     }
+    
+    var _this = this
+    var new_frame_to_nodes_registry = []
+    if (this.nodes_to_frames_registry[id.slice(5)] !== undefined) {
+      this.nodes_to_frames_registry[id.slice(5)].forEach((frameUuid) => {
+        new_frame_to_nodes_registry = _this.frame_to_nodes_registry[frameUuid].filter((nodeId) => {
+          return id.slice(5) != nodeId;
+        })
+        if (new_frame_to_nodes_registry.length == 0) {
+          _this.removeFrameNodeId('framenode-' + frameUuid)
+        } else {
+          const frameNode = _this.container.querySelector('#framenode-' + frameUuid)
+          _this.frame_to_nodes_registry[frameUuid] = new_frame_to_nodes_registry
+          _this.updateFrameNodePosition(frameNode, _this.frame_to_nodes_registry[frameUuid]);
+        }
+      })
+      delete this.nodes_to_frames_registry[id.slice(5)]
+    }
+
     delete this.drawflow.drawflow[moduleName].data[id.slice(5)];
     this.dispatch('nodeRemoved', id.slice(5));
   }
@@ -1874,6 +2096,21 @@ class Drawflow {
     }
   }
 
+  removeFrameNodeId(fullID) {
+    const frameUuid = fullID.slice(10);
+    const moduleName = this.getModuleFromNodeId(this.frame_to_nodes_registry[frameUuid][0])
+    this.container.querySelector(`#${fullID}`).remove();
+    var _this = this;
+    this.frame_to_nodes_registry[frameUuid].forEach((nodeID) => {
+      _this.nodes_to_frames_registry[nodeID] = _this.nodes_to_frames_registry[nodeID].filter((frameNodeUuid) => {
+        return frameNodeUuid != frameUuid;
+      })
+    })
+    delete this.frame_to_nodes_registry[frameUuid];
+    delete this.drawflow.drawflow[moduleName].data._frames[frameUuid];
+    this.dispatch('frameNodeRemoved', frameUuid);
+  }
+
   getModuleFromNodeId(id) {
     var nameModule;
     const editor = this.drawflow.drawflow
@@ -1888,7 +2125,7 @@ class Drawflow {
   }
 
   addModule(name) {
-    this.drawflow.drawflow[name] = { "data": {} };
+    this.drawflow.drawflow[name] = { "data": { _frames: {} } };
     this.dispatch('moduleCreated', name);
   }
   changeModule(name) {
@@ -1917,12 +2154,16 @@ class Drawflow {
 
   clearModuleSelected() {
     this.precanvas.innerHTML = "";
-    this.drawflow.drawflow[this.module] = { "data": {} };
+    this.drawflow.drawflow[this.module] = { "data": { _frames: {}} };
+    this.frame_to_nodes_registry = {};
+    this.nodes_to_frames_registry = {};
   }
 
   clear() {
     this.precanvas.innerHTML = "";
-    this.drawflow = { "drawflow": { "Home": { "data": {} } } };
+    this.drawflow = { "drawflow": { "Home": { "data": { _frames: {} } } } };
+    this.frame_to_nodes_registry = {};
+    this.nodes_to_frames_registry = {};
   }
   export() {
     const dataExport = JSON.parse(JSON.stringify(this.drawflow));
diff --git a/app/webroot/js/workflows-editor/workflows-editor.js b/app/webroot/js/workflows-editor/workflows-editor.js
index 73cfc4ec1..05ec4120c 100644
--- a/app/webroot/js/workflows-editor/workflows-editor.js
+++ b/app/webroot/js/workflows-editor/workflows-editor.js
@@ -175,6 +175,21 @@ function initDrawflow() {
     editor.on('nodeRemoved', function () {
         invalidateContentCache()
     })
+    editor.on('frameNodeCreated', function (framenodeUuid) {
+        invalidateContentCache()
+        var frameNodeFullUuid = 'framenode-' + framenodeUuid
+        $('#' + frameNodeFullUuid).find('.drawflow-framenode-text').dblclick(function() {
+            var existingText = $(this).text()
+            var newText = prompt('Edit frame node text', existingText)
+            editor.updateFramenode(frameNodeFullUuid, {text: newText})
+        })
+    })
+    editor.on('frameNodeUpdated', function () {
+        invalidateContentCache()
+    })
+    editor.on('frameNodeRemoved', function () {
+        invalidateContentCache()
+    })
     editor.on('nodeDataChanged', invalidateContentCache)
     editor.on('nodeMoved', invalidateContentCache)
     editor.on('connectionCreated', function() {
@@ -204,6 +219,9 @@ function initDrawflow() {
             duplicateSelection()
             evt.preventDefault()
         }
+        if (evt.keyCode == 70 && $drawflow.is(evt.target)) {
+            createFrameNodeForSelected()
+        }
     })
     editor.translate_to = function (x, y) {
         this.canvas_x = x;
@@ -415,6 +433,7 @@ function initDrawflow() {
     })
     editor.on('nodeSelected', function(node_id) {
         $controlDuplicateButton.removeClass('disabled')
+        $controlFrameNodeButton.removeClass('disabled')
         $controlDeleteButton.removeClass('disabled')
         $controlSaveBlocksLi.removeClass('disabled')
         $controlEditBlocksLiContainer.removeClass('disabled').find('.dropdown-menu')
@@ -426,6 +445,7 @@ function initDrawflow() {
         })
         selection.clearSelection()
         $controlDuplicateButton.addClass('disabled')
+        $controlFrameNodeButton.addClass('disabled')
         $controlDeleteButton.addClass('disabled')
         $controlSaveBlocksLi.addClass('disabled')
         $controlEditBlocksLiContainer.addClass('disabled').find('.dropdown-menu')
@@ -473,6 +493,9 @@ function initDrawflow() {
     $controlDuplicateButton.click(function() {
         duplicateSelection()
     })
+    $controlFrameNodeButton.click(function() {
+        createFrameNodeForSelected()
+    })
     $controlDeleteButton.click(function() {
         deleteSelectedNodes(false)
     })
@@ -574,6 +597,26 @@ function duplicateSelection() {
     selection.select(newNodes)
 }
 
+function createFrameNodeForSelected() {
+    var text = prompt('Enter text for the frame node')
+    var selectedNodesHtml = selection.getSelection()
+    var selectedIDs = selectedNodesHtml.map(function(nodeHtml) {
+        return parseInt(nodeHtml.id.slice(5))
+    })
+    createFrameForNodes(selectedIDs, text)
+    selection.clearSelection()
+    invalidateContentCache()
+}
+
+function createFrameForNodes(nodesIDs, text) {
+    const frameNode = {
+        nodes: nodesIDs,
+        text: text,
+        class: "",
+    }
+    editor.addFrameNode(frameNode)
+}
+
 function buildModalForBlock(node_id, node) {
     var html = genNodeParamHtml(node, false)
     $blockModal
@@ -701,6 +744,8 @@ function addNode(block, position, additionalData={}) {
 function getEditorData(cleanNodes) {
     var data = {} // Make sure nodes are index by their internal IDs
     var editorExport = editor.export().drawflow.Home.data
+    var frameNodes = editorExport._frames !== undefined ? editorExport._frames : {}
+    delete editorExport._frames
     editorExport = Array.isArray(editorExport) ? editorExport : Object.values(editorExport)
     editorExport.forEach(function(node) {
         if (node !== null) { // for some reason, the editor create null nodes
@@ -725,6 +770,7 @@ function getEditorData(cleanNodes) {
             data[node.id] = node
         }
     })
+    data._frames = frameNodes
     return data
 }
 
@@ -756,7 +802,12 @@ function loadWorkflow(workflow) {
     }
     // We cannot rely on the editor's import function as it recreates the nodes with the saved HTML instead of rebuilding them
     // We have to manually add the nodes and their connections
-    Object.values(workflow.data).forEach(function (node) {
+    Object.entries(workflow.data).forEach(function (entry) {
+        var i = entry[0]
+        var node = entry[1]
+        if (i == '_frames') {
+            return
+        }
         var module = all_modules_by_id[node.data.id] || all_triggers_by_id[node.data.id]
         if (!module) {
             console.error('Tried to add node for unknown module ' + node.data.id + ' (' + node.id + ')')
@@ -818,6 +869,10 @@ function loadWorkflow(workflow) {
             })
         }
     })
+    var frameNodes = workflow.data._frames || {}
+    Object.values(frameNodes).forEach(function (frameNode) {
+        editor.addFrameNode(frameNode)
+    })
 }
 
 function filterModules(clicked) {
@@ -1294,6 +1349,11 @@ function addWorkflowBlueprint(blueprintId, cursorPosition) {
     if (newNodes.length > 0) {
         selection.clearSelection()
         selection.select(newNodes)
+        var newNodeIDs = newNodes.map(function(node) {
+            return node.id.slice(5)
+        })
+        createFrameForNodes(newNodeIDs, workflowBlueprint.WorkflowBlueprint.name)
+
         editor.dispatch('nodeSelected', newNodes[0].id);
     }
 }

From 4117ccc00a1b2c062989169d2b27e6f167bf1589 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Fri, 28 Jul 2023 08:58:32 +0200
Subject: [PATCH 78/96] chg: [workflow:editor] Usage of proxy function to
 delete nodes

---
 app/webroot/js/workflows-editor/workflows-editor.js | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/app/webroot/js/workflows-editor/workflows-editor.js b/app/webroot/js/workflows-editor/workflows-editor.js
index 05ec4120c..3134917d1 100644
--- a/app/webroot/js/workflows-editor/workflows-editor.js
+++ b/app/webroot/js/workflows-editor/workflows-editor.js
@@ -1312,7 +1312,16 @@ function getSelectedNode() {
 }
 
 function deleteSelectedNode() {
-    editor.removeNodeId(getSelectedNodeID())
+    deleteNodeByID(getSelectedNodeID())
+}
+
+function deleteNodeByID(nodeId) {
+    nodeIdInt = nodeId.slice(5)
+    editorData = getEditorData()
+    if (all_triggers_by_id[editorData[nodeIdInt].data.id]) {
+        return
+    }
+    editor.removeNodeId(nodeId)
 }
 
 function getNodeFromContainedHtml(htmlNode) {
@@ -1327,7 +1336,7 @@ function deleteSelectedNodes(fromDelKey) {
         if (fromDelKey && getSelectedNodeID() !== null && getSelectedNodeID() == node.id) {
             return // This node will be removed by drawflow delete callback
         }
-        editor.removeNodeId(node.id)
+        deleteNodeByID(node.id)
     })
     editor.dispatch('nodeUnselected')
 }

From 784cb3f66d4fbf301ec81571b90de20f5a0c1f5e Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Fri, 28 Jul 2023 09:04:57 +0200
Subject: [PATCH 79/96] chg: [workflow:editor] Moved styling in a class rather
 than in drawflow lib

---
 app/webroot/css/workflows-editor.css | 15 ++++++++++++++-
 app/webroot/js/drawflow.js           | 10 ----------
 2 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/app/webroot/css/workflows-editor.css b/app/webroot/css/workflows-editor.css
index e3a1a16ab..6bf600770 100644
--- a/app/webroot/css/workflows-editor.css
+++ b/app/webroot/css/workflows-editor.css
@@ -425,10 +425,23 @@
     background-color: #2fa1db;
 } */
 
+.drawflow .drawflow-framenode {
+    position: absolute;
+    z-index: -1;
+    text-align: center;
+    background-color: #b5b5b570;
+    border: 1px solid #333333aa;
+    border-radius: 5px;
+}
+
 .drawflow .drawflow-framenode.selected {
     box-shadow: 0 0 5px 2px #33333366;
 }
-.drawflow .drawflow-framenode .drawflow-framenode-text {
+.drawflow .drawflow-framenode > .drawflow-framenode-text {
+    line-height: 1.5em;
+    color: 505050;
+    font-size: 1.5em;
+    font-weight: bold;
     cursor: text;
 }
 
diff --git a/app/webroot/js/drawflow.js b/app/webroot/js/drawflow.js
index 66c94794d..37a1ee627 100644
--- a/app/webroot/js/drawflow.js
+++ b/app/webroot/js/drawflow.js
@@ -1617,20 +1617,10 @@ class Drawflow {
       frameNode.classList.add(...frameNodeConfig.class.split(' '));
     }
 
-    frameNode.style.backgroundColor = '#b5b5b570';
-    frameNode.style.border = '1px solid #333333aa';
-    frameNode.style.position = 'absolute'
-    frameNode.style['text-align'] = 'center'
-    frameNode.style['z-index'] = -1
-    frameNode.style['border-radius'] = '5px'
     this.updateFrameNodePosition(frameNode, frameNodeConfig.nodes)
 
     const frameTextNode = document.createElement('span');
     frameTextNode.innerText = frameNodeConfig.text;
-    frameTextNode.style['line-height'] = '1.5em'
-    frameTextNode.style.color = '#505050'
-    frameTextNode.style['font-size'] = '1.5em'
-    frameTextNode.style['font-weight'] = 'bold'
     frameTextNode.classList.add('drawflow-framenode-text')
     frameNode.appendChild(frameTextNode);
 

From 976b6f91f0dc0b46fc2f0b5c7a04800ea56e7025 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Fri, 28 Jul 2023 09:24:55 +0200
Subject: [PATCH 80/96] fix: [workflow:editor] Typo in css rule

---
 app/webroot/css/workflows-editor.css | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/webroot/css/workflows-editor.css b/app/webroot/css/workflows-editor.css
index 6bf600770..4f96eac70 100644
--- a/app/webroot/css/workflows-editor.css
+++ b/app/webroot/css/workflows-editor.css
@@ -439,7 +439,7 @@
 }
 .drawflow .drawflow-framenode > .drawflow-framenode-text {
     line-height: 1.5em;
-    color: 505050;
+    color: #505050;
     font-size: 1.5em;
     font-weight: bold;
     cursor: text;

From d2bb5e7de3c42ac77e2480ebafe17f431aba1a50 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Fri, 28 Jul 2023 09:26:01 +0200
Subject: [PATCH 81/96] chg: [workflow:editor] Small refactoring of drawflow
 lib

---
 app/webroot/js/drawflow.js | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/app/webroot/js/drawflow.js b/app/webroot/js/drawflow.js
index 37a1ee627..9a3379ef9 100644
--- a/app/webroot/js/drawflow.js
+++ b/app/webroot/js/drawflow.js
@@ -38,7 +38,7 @@ class Drawflow {
 
     this.noderegister = {};
     this.render = render;
-    this.drawflow = { "drawflow": { "Home": { "data": { _frames: {}} } } };
+    this.drawflow = { "drawflow": { "Home": { "data": { "_frames": {} } } } };
     // Configurable options
     this.module = 'Home';
     this.editor_mode = 'edit';
@@ -312,15 +312,15 @@ class Drawflow {
           this.node_selected = null;
           this.dispatch('nodeUnselected', true);
         }
-        if (this.framenode_selected != null) {
-          this.framenode_selected.classList.remove("selected");
-          this.framenode_selected = null
-        }
         if (this.connection_selected != null) {
           this.connection_selected.classList.remove("selected");
           this.removeReouteConnectionSelected();
           this.connection_selected = null;
         }
+        if (this.framenode_selected != null) {
+          this.framenode_selected.classList.remove("selected");
+          this.framenode_selected = null
+        }
         this.connection_selected = this.ele_selected;
         this.connection_selected.classList.add("selected");
         const listclassConnection = this.connection_selected.parentElement.classList;
@@ -1962,7 +1962,7 @@ class Drawflow {
     if (this.module === moduleName) {
       this.container.querySelector(`#${id}`).remove();
     }
-    
+
     var _this = this
     var new_frame_to_nodes_registry = []
     if (this.nodes_to_frames_registry[id.slice(5)] !== undefined) {
@@ -2115,7 +2115,7 @@ class Drawflow {
   }
 
   addModule(name) {
-    this.drawflow.drawflow[name] = { "data": { _frames: {} } };
+    this.drawflow.drawflow[name] = { "data": { "_frames": {} } };
     this.dispatch('moduleCreated', name);
   }
   changeModule(name) {
@@ -2144,14 +2144,14 @@ class Drawflow {
 
   clearModuleSelected() {
     this.precanvas.innerHTML = "";
-    this.drawflow.drawflow[this.module] = { "data": { _frames: {}} };
+    this.drawflow.drawflow[this.module] = { "data": { "_frames": {}} };
     this.frame_to_nodes_registry = {};
     this.nodes_to_frames_registry = {};
   }
 
   clear() {
     this.precanvas.innerHTML = "";
-    this.drawflow = { "drawflow": { "Home": { "data": { _frames: {} } } } };
+    this.drawflow = { "drawflow": { "Home": { "data": { "_frames": {} } } } };
     this.frame_to_nodes_registry = {};
     this.nodes_to_frames_registry = {};
   }

From 0214ffad1be265715bd47c8786e4d86f12fcb488 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Fri, 28 Jul 2023 09:30:04 +0200
Subject: [PATCH 82/96] chg: [workflow:editor] Make frame node padding
 configurable

---
 app/webroot/js/drawflow.js | 16 +++++-----------
 1 file changed, 5 insertions(+), 11 deletions(-)

diff --git a/app/webroot/js/drawflow.js b/app/webroot/js/drawflow.js
index 9a3379ef9..91fdb362f 100644
--- a/app/webroot/js/drawflow.js
+++ b/app/webroot/js/drawflow.js
@@ -47,6 +47,7 @@ class Drawflow {
     this.zoom_min = 0.5;
     this.zoom_value = 0.1;
     this.zoom_last_value = 1;
+    this.frameNodePadding = { "top": 100, "left": 30, "right": 30, "bottom": 20 }
 
     // Mobile
     this.evCache = new Array();
@@ -1642,13 +1643,6 @@ class Drawflow {
       return
     }
 
-    var offsetPadding = {
-      top: 100,
-      left: 30,
-      right: 30,
-      bottom: 20,
-    }
-
     var _this = this
     var nodesHtml = nodeIDs.map(function (id) {
       return _this.container.querySelector('#node-' + id)
@@ -1669,10 +1663,10 @@ class Drawflow {
 
     var frameCoordinates = getFrameCoordinatesFromNodes(nodesHtml)
 
-    var pos_x = frameCoordinates.topLeft.x - offsetPadding.left / 2
-    var pos_y = frameCoordinates.topLeft.y - offsetPadding.top / 2
-    var frameNodeHeight = (frameCoordinates.bottomRight.y - frameCoordinates.topLeft.y) + (offsetPadding.top / 2 + offsetPadding.bottom)
-    var frameNodeWidth = (frameCoordinates.bottomRight.x - frameCoordinates.topLeft.x) + (offsetPadding.left / 2 + offsetPadding.right)
+    var pos_x = frameCoordinates.topLeft.x - this.frameNodePadding.left / 2
+    var pos_y = frameCoordinates.topLeft.y - this.frameNodePadding.top / 2
+    var frameNodeHeight = (frameCoordinates.bottomRight.y - frameCoordinates.topLeft.y) + (this.frameNodePadding.top / 2 + this.frameNodePadding.bottom)
+    var frameNodeWidth = (frameCoordinates.bottomRight.x - frameCoordinates.topLeft.x) + (this.frameNodePadding.left / 2 + this.frameNodePadding.right)
 
     var transpositionFactorX = (this.precanvas.clientWidth / (this.precanvas.clientWidth * this.zoom))
     var transpositionOffsetX = -(this.precanvas.getBoundingClientRect().x * (this.precanvas.clientWidth / (this.precanvas.clientWidth * this.zoom)))

From 26779f775315ccd3f5d4a13cde6391d07648f62a Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Fri, 28 Jul 2023 10:42:38 +0200
Subject: [PATCH 83/96] chg: Bumped queryVersion

---
 app/Controller/AppController.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php
index 225e9c0a2..2154496c3 100755
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@@ -33,7 +33,7 @@ class AppController extends Controller
 
     public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
 
-    private $__queryVersion = '152';
+    private $__queryVersion = '153';
     public $pyMispVersion = '2.4.173';
     public $phpmin = '7.2';
     public $phprec = '7.4';

From 1014beab391838d5aca99b56a15b749f8036bac0 Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Fri, 28 Jul 2023 11:16:40 +0200
Subject: [PATCH 84/96] chg: [warninglists] updated to the latest version

---
 app/files/warninglists | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/warninglists b/app/files/warninglists
index 1a94fcd66..11101527c 160000
--- a/app/files/warninglists
+++ b/app/files/warninglists
@@ -1 +1 @@
-Subproject commit 1a94fcd666bbf7eb505d4fbbc47ef6170c375706
+Subproject commit 11101527c0e55810613d3d753f2e770219895c39

From 4ad70965b5d3b69bfab6995ba26c0875ca452a55 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Fri, 28 Jul 2023 13:10:19 +0200
Subject: [PATCH 85/96] fix: [totp] generate a new totp secret each time a the
 totp_new endpoint is queried via a GET request, fixes #9220

---
 app/Controller/UsersController.php | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php
index dba396bfd..ee3f20cb9 100644
--- a/app/Controller/UsersController.php
+++ b/app/Controller/UsersController.php
@@ -1835,7 +1835,7 @@ class UsersController extends AppController
     public function totp_new()
     {
         if (Configure::read('LinOTPAuth.enabled')) {
-            $this->Flash->error(__("LinOTP is enabled for this instance. Build-in TOTP should not be used."));
+            $this->Flash->error(__("LinOTP is enabled for this instance. Built-in TOTP should not be used."));
             $this->redirect($this->referer());
         }
         if (!class_exists('\OTPHP\TOTP') || !class_exists('\BaconQrCode\Writer')) {
@@ -1856,17 +1856,23 @@ class UsersController extends AppController
         }
         // do not allow this page to be accessed if the current already has a TOTP. Just redirect to the users details page with a Flash->error()
         if ($user['User']['totp']) { 
-            $this->Flash->error(__("Your account already has an TOTP. Please contact your organisational administrator to change or delete it."));
+            $this->Flash->error(__("Your account already has a TOTP. Please contact your organisational administrator to change or delete it."));
             $this->redirect($this->referer());
         }
 
-        $secret = $this->Session->read('otp_secret');  // Reload secret from session.
-        if ($secret) {
-            $totp = \OTPHP\TOTP::create($secret);
-        } else {
+        if ($this->request->is('get')) {
             $totp = \OTPHP\TOTP::create();
             $secret = $totp->getSecret();
-            $this->Session->write('otp_secret', $secret);  // Store in session, this is to keep the same QR code even if the page refreshes.
+            $this->Session->write('otp_secret', $secret);  // Store in session, we want to create a new secret each time the totp_new() function is queried via a GET (this will not impede incorrect confirmation attempty)
+        } else {
+            $secret = $this->Session->read('otp_secret');  // Reload secret from session.
+            if ($secret) {
+                $totp = \OTPHP\TOTP::create($secret);
+            } else {
+                $totp = \OTPHP\TOTP::create();
+                $secret = $totp->getSecret();
+                $this->Session->write('otp_secret', $secret); // Store in session, we want to keep reusing the same QR code until the user correctly enters the generated key on their authenticator
+            }
         }
         if ($this->request->is('post') && isset($this->request->data['User']['otp'])) {
             if ($totp->verify(trim($this->request->data['User']['otp']))) {

From 7c28cee942889b7d069fb942b1fdaa780c5ef09a Mon Sep 17 00:00:00 2001
From: Luciano Righetti <luciano.righetti@gmail.com>
Date: Fri, 28 Jul 2023 15:29:46 +0200
Subject: [PATCH 86/96] fix: revert loginAction override

---
 app/Controller/AppController.php | 1 -
 1 file changed, 1 deletion(-)

diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php
index 225e9c0a2..1142b9015 100755
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@@ -111,7 +111,6 @@ class AppController extends Controller
 
         $this->_setupBaseurl();
         $this->Auth->loginRedirect = $this->baseurl . '/users/routeafterlogin';
-        $this->Auth->loginAction = $this->baseurl . '/users/login';
 
         $customLogout = Configure::read('Plugin.CustomAuth_custom_logout');
         $this->Auth->logoutRedirect = $customLogout ?: ($this->baseurl . '/users/login');

From 0204238b1ac9242c2166aba27802b9374f2034a4 Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Fri, 28 Jul 2023 15:37:03 +0200
Subject: [PATCH 87/96] chg: [misp-galaxy] updated to the latest version

---
 app/files/misp-galaxy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy
index 7028860c0..880763b0f 160000
--- a/app/files/misp-galaxy
+++ b/app/files/misp-galaxy
@@ -1 +1 @@
-Subproject commit 7028860c0aa8c471324008d3dc651b7ea9e07c0a
+Subproject commit 880763b0f40c7cb3def1ded3c925397c20504264

From 44b104ffdbf12462ab014048e2a49e8cd52eb940 Mon Sep 17 00:00:00 2001
From: Jeroen Pinoy <jeroen.pinoy@fortitude.ninja>
Date: Fri, 28 Jul 2023 13:18:35 +0200
Subject: [PATCH 88/96] new: [Authkeys] Add setting to mandate IP allowlist for
 advanced authkeys

---
 app/Model/AuthKey.php | 7 +++++++
 app/Model/Server.php  | 8 ++++++++
 2 files changed, 15 insertions(+)

diff --git a/app/Model/AuthKey.php b/app/Model/AuthKey.php
index fc2c2256e..fbbd80f1e 100644
--- a/app/Model/AuthKey.php
+++ b/app/Model/AuthKey.php
@@ -55,6 +55,7 @@ class AuthKey extends AppModel
             $this->data['AuthKey']['authkey_raw'] = $authkey;
         }
 
+        $validAllowedIpFound = false;
         if (!empty($this->data['AuthKey']['allowed_ips'])) {
             $allowedIps = &$this->data['AuthKey']['allowed_ips'];
             if (is_string($allowedIps)) {
@@ -70,12 +71,18 @@ class AuthKey extends AppModel
             if (!is_array($allowedIps)) {
                 $this->invalidate('allowed_ips', 'Allowed IPs must be array');
             }
+
             foreach ($allowedIps as $cidr) {
                 if (!CidrTool::validate($cidr)) {
                     $this->invalidate('allowed_ips', "$cidr is not valid IP range");
+                } else {
+                    $validAllowedIpFound = true;
                 }
             }
         }
+        if (!empty(Configure::read('Security.mandate_ip_allowlist_advanced_authkeys')) && $validAllowedIpFound === false){
+            $this->invalidate('allowed_ips', "Setting an ip allowlist is mandatory on this instance.");
+        }
 
         $creationTime = isset($this->data['AuthKey']['created']) ? $this->data['AuthKey']['created'] : time();
         $validity = Configure::read('Security.advanced_authkeys_validity');
diff --git a/app/Model/Server.php b/app/Model/Server.php
index cb5b787b1..b27fa1c68 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -6373,6 +6373,14 @@ class Server extends AppModel
                     'type' => 'boolean',
                     'null' => true
                 ),
+                'mandate_ip_allowlist_advanced_authkeys' => array(
+                    'level' => 2,
+                    'description' => __('If enabled, setting an ip allowlist will be mandatory when adding or editing an advanced authkey.'),
+                    'value' => false,
+                    'test' => 'testBool',
+                    'type' => 'boolean',
+                    'null' => true
+                ),
                 'disable_browser_cache' => array(
                     'level' => 0,
                     'description' => __('If enabled, HTTP headers that block browser cache will be send. Static files (like images or JavaScripts) will still be cached, but not generated pages.'),

From 72b09aecdb1568015121ec2e2c95ca6290bbef52 Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Mon, 31 Jul 2023 10:18:34 +0200
Subject: [PATCH 89/96] chg: [taxonomies] updated to the latest version 2.4.174

---
 app/files/taxonomies | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/taxonomies b/app/files/taxonomies
index bb5d823ee..59ec473a5 160000
--- a/app/files/taxonomies
+++ b/app/files/taxonomies
@@ -1 +1 @@
-Subproject commit bb5d823ee4d4cfce445bfbf89de8e013c169a3d2
+Subproject commit 59ec473a5f7a44755a6098890a1ee290487bfc53

From b1850f243fad1a57435e149fe6f24081327142e8 Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Mon, 31 Jul 2023 10:27:53 +0200
Subject: [PATCH 90/96] chg: [misp-galaxy] version 2.4.174

---
 app/files/misp-galaxy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy
index 880763b0f..f5729ac23 160000
--- a/app/files/misp-galaxy
+++ b/app/files/misp-galaxy
@@ -1 +1 @@
-Subproject commit 880763b0f40c7cb3def1ded3c925397c20504264
+Subproject commit f5729ac23a7dcb8fc9dc3194a3e125484c515742

From 8c4b5ed278369149cfb5d5015024d3d479dce490 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Mon, 31 Jul 2023 10:56:33 +0200
Subject: [PATCH 91/96] fix: [workflow] Removed trailing comma in function call

---
 app/Model/Workflow.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Model/Workflow.php b/app/Model/Workflow.php
index 27e22b865..acb10df12 100644
--- a/app/Model/Workflow.php
+++ b/app/Model/Workflow.php
@@ -684,7 +684,7 @@ class Workflow extends AppModel
             $node['data']['id'],
             $node['id'],
             $roamingData->getWorkflow()['Workflow']['name'],
-            $roamingData->getWorkflow()['Workflow']['id'],
+            $roamingData->getWorkflow()['Workflow']['id']
         );
         $this->logExecutionIfDebug($roamingData->getWorkflow(), $message);
         $this->sendRequestToDebugEndpointIfDebug($roamingData->getWorkflow(), $node, sprintf('/exec/%s?result=%s', $moduleClass->id, 'success'), $roamingData->getData());

From 2f4a49ccb2433c7bc458eea6a03645295d27a9b8 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Mon, 31 Jul 2023 11:16:49 +0200
Subject: [PATCH 92/96] fix: [event:publishSightingsRouter] Make sure to use
 correct queue for publishSightingsRouter

Fix bug introduced in 64580168622aeea59997cea5739cf0b8dbcf8bda where workers were set to default but not the queue
---
 app/Model/Event.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Model/Event.php b/app/Model/Event.php
index 7979ff0a3..06bb7bf7e 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -4511,7 +4511,7 @@ class Event extends AppModel
             }
 
             return $this->getBackgroundJobsTool()->enqueue(
-                BackgroundJobsTool::PRIO_QUEUE,
+                BackgroundJobsTool::DEFAULT_QUEUE,
                 BackgroundJobsTool::CMD_EVENT,
                 $args,
                 true,

From f22c82e87a3d038e540420a0938f8ed51a2db797 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Mon, 31 Jul 2023 12:08:29 +0200
Subject: [PATCH 93/96] chg: [PyMISP] Bump.

---
 PyMISP                           | 2 +-
 app/Controller/AppController.php | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/PyMISP b/PyMISP
index ccae32ae7..94983c01e 160000
--- a/PyMISP
+++ b/PyMISP
@@ -1 +1 @@
-Subproject commit ccae32ae716c143bea09954e860238e193bc78c6
+Subproject commit 94983c01ecced6086df28133a38a297111534142
diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php
index c46853ac5..a5212bb63 100755
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@@ -34,7 +34,7 @@ class AppController extends Controller
     public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
 
     private $__queryVersion = '153';
-    public $pyMispVersion = '2.4.173';
+    public $pyMispVersion = '2.4.174';
     public $phpmin = '7.2';
     public $phprec = '7.4';
     public $phptoonew = '8.0';

From 1fb86c047048dab8ba119b71d87f5e66afe13d7f Mon Sep 17 00:00:00 2001
From: Christian Studer <christian.studer@circl.lu>
Date: Mon, 31 Jul 2023 13:12:39 +0200
Subject: [PATCH 94/96] chg: [misp-stix] Bumped latest version

---
 app/files/scripts/misp-stix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/scripts/misp-stix b/app/files/scripts/misp-stix
index 0b5455752..a12d40a4f 160000
--- a/app/files/scripts/misp-stix
+++ b/app/files/scripts/misp-stix
@@ -1 +1 @@
-Subproject commit 0b5455752686408bd4e1c3e7c22359cf2460db04
+Subproject commit a12d40a4ff8a19c0a6b9f4d719c9549e805108c5

From e20c385a90f52ff6535effa04407f091ecc31aa4 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 31 Jul 2023 14:11:47 +0200
Subject: [PATCH 95/96] chg: [version] bump

---
 VERSION.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION.json b/VERSION.json
index 7a0dab149..1f1c3a0c9 100644
--- a/VERSION.json
+++ b/VERSION.json
@@ -1 +1 @@
-{"major":2, "minor":4, "hotfix":173}
+{"major":2, "minor":4, "hotfix":174}

From 72df2b48ab7f2a6d266c4c60ec4a59d0669bcc1b Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Mon, 31 Jul 2023 16:24:40 +0200
Subject: [PATCH 96/96] chg: [misp-objects] updated to the latest version

---
 app/files/misp-objects | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/misp-objects b/app/files/misp-objects
index da801ab14..4da05293d 160000
--- a/app/files/misp-objects
+++ b/app/files/misp-objects
@@ -1 +1 @@
-Subproject commit da801ab146fb622a6447c8d2922a95b6049bb70a
+Subproject commit 4da05293d723ad6f9db4a3e349e140daa5d2a28d