From 5b11e6b2124a1b10a555edeebec8258191d8541f Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 24 Mar 2024 18:46:02 +0100
Subject: [PATCH 01/45] chg: [internal] Log content type when JSON could not be
 parsed

---
 app/Lib/Tools/HttpSocketExtended.php | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/Lib/Tools/HttpSocketExtended.php b/app/Lib/Tools/HttpSocketExtended.php
index cfe097c28..051c0ae60 100644
--- a/app/Lib/Tools/HttpSocketExtended.php
+++ b/app/Lib/Tools/HttpSocketExtended.php
@@ -121,7 +121,8 @@ class HttpSocketResponseExtended extends HttpSocketResponse
         try {
             return JsonTool::decode($this->body);
         } catch (Exception $e) {
-            throw new HttpSocketJsonException('Could not parse response as JSON.', $this, $e);
+            $contentType = $this->getHeader('content-type');
+            throw new HttpSocketJsonException("Could not parse HTTP response as JSON. Received Content-Type $contentType.", $this, $e);
         }
     }
 }

From 2e32d22d2cfda50315fb488b07e1fc63dbf0a120 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 6 Apr 2024 14:05:44 +0200
Subject: [PATCH 02/45] chg: [sync] Move blocklist fetching out of
 ServerSyncTool and reduce sightings fetched in one fetch

---
 app/Lib/Tools/ServerSyncTool.php | 17 ++++++-----------
 app/Model/Sighting.php           | 11 +++++++++--
 2 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/app/Lib/Tools/ServerSyncTool.php b/app/Lib/Tools/ServerSyncTool.php
index 40e555d4a..11c1c9d61 100644
--- a/app/Lib/Tools/ServerSyncTool.php
+++ b/app/Lib/Tools/ServerSyncTool.php
@@ -297,29 +297,24 @@ class ServerSyncTool
 
     /**
      * @param array $eventUuids
+     * @param array $blockedOrgs Blocked organisation UUIDs
      * @return array
      * @throws HttpSocketHttpException
      * @throws HttpSocketJsonException
      * @throws JsonException
      */
-    public function fetchSightingsForEvents(array $eventUuids)
+    public function fetchSightingsForEvents(array $eventUuids, array $blockedOrgs = [])
     {
-        $SightingBlocklist = ClassRegistry::init('SightingBlocklist');
-        $blocked_sightings = $SightingBlocklist->find('column', [
-            'recursive' => -1,
-            'fields' => ['org_uuid']
-        ]);
-        foreach ($blocked_sightings as $k => $uuid) {
-            $blocked_sightings[$k] = '!' . $uuid;
-        }
         $postParams = [
             'returnFormat' => 'json',
             'last' => 0, // fetch all
             'includeUuid' => true,
             'uuid' => $eventUuids,
         ];
-        if (!empty($blocked_sightings)) {
-            $postParams['org_id'] = $blocked_sightings;
+        if (!empty($blockedOrgs)) {
+            $postParams['org_id'] = array_map(function ($uuid) {
+                return "!$uuid";
+            }, $blockedOrgs);
         }
         return $this->post('/sightings/restSearch/event', $postParams)->json()['response'];
     }
diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php
index df09b80cc..0a33a87ca 100644
--- a/app/Model/Sighting.php
+++ b/app/Model/Sighting.php
@@ -1470,12 +1470,19 @@ class Sighting extends AppModel
      */
     private function pullSightingNewWay(array $user, array $eventUuids, ServerSyncTool $serverSync)
     {
+        $SightingBlocklist = ClassRegistry::init('SightingBlocklist');
+        $blockedSightingsOrgs = $SightingBlocklist->find('column', [
+            'recursive' => -1,
+            'fields' => ['org_uuid']
+        ]);
+
         $uuids = array_keys($eventUuids);
+        shuffle($uuids); // shuffle array to avoid keeping events with a lof ot sightings in same batch all the time
         $saved = 0;
         $savedEventUuids = [];
-        foreach (array_chunk($uuids, 100) as $chunk) {
+        foreach (array_chunk($uuids, 20) as $chunk) {
             try {
-                $sightings = $serverSync->fetchSightingsForEvents($chunk);
+                $sightings = $serverSync->fetchSightingsForEvents($chunk, $blockedSightingsOrgs);
             } catch (Exception $e) {
                 $this->logException("Failed to download sightings from remote server {$serverSync->server()['Server']['name']}.", $e);
                 continue;

From 2b38de942b76b71ab8849d965e9f9577d7f0f7dc Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 30 Mar 2024 15:06:42 +0100
Subject: [PATCH 03/45] chg: [internal] Server sync debug messages

---
 app/Lib/Tools/ServerSyncTool.php | 11 +++++++++++
 app/Lib/Tools/SyncTool.php       |  1 -
 app/Model/AnalystData.php        |  4 +---
 app/Model/GalaxyCluster.php      |  3 +++
 app/Model/Server.php             | 18 ++++++++++++------
 app/Model/ShadowAttribute.php    |  2 ++
 app/Model/Sighting.php           |  4 ++++
 7 files changed, 33 insertions(+), 10 deletions(-)

diff --git a/app/Lib/Tools/ServerSyncTool.php b/app/Lib/Tools/ServerSyncTool.php
index 11c1c9d61..6f433bcba 100644
--- a/app/Lib/Tools/ServerSyncTool.php
+++ b/app/Lib/Tools/ServerSyncTool.php
@@ -506,6 +506,16 @@ class ServerSyncTool
         return $this->socket->getMetaData();
     }
 
+    /**
+     * @param string $message
+     * @return void
+     */
+    public function debug($message)
+    {
+        $memoryUsage = round(memory_get_usage() / 1024 / 1024, 2);
+        CakeLog::debug("[Server sync #{$this->serverId()}]: $message. Memory: $memoryUsage MB");
+    }
+
     /**
      * @params string $url Relative URL
      * @return HttpSocketResponseExtended
@@ -556,6 +566,7 @@ class ServerSyncTool
 
         if ($etag) {
             // Remove compression marks that adds Apache for compressed content
+            // This can be removed in future as this is already checked by MISP itself since 2024-03
             $etagWithoutQuotes = trim($etag, '"');
             $dashPos = strrpos($etagWithoutQuotes, '-');
             if ($dashPos && in_array(substr($etagWithoutQuotes, $dashPos + 1), ['br', 'gzip'], true)) {
diff --git a/app/Lib/Tools/SyncTool.php b/app/Lib/Tools/SyncTool.php
index b0ef40a41..8495dad2d 100644
--- a/app/Lib/Tools/SyncTool.php
+++ b/app/Lib/Tools/SyncTool.php
@@ -1,7 +1,6 @@
 <?php
 class SyncTool
 {
-
     const ALLOWED_CERT_FILE_EXTENSIONS = ['pem', 'crt'];
 
     /**
diff --git a/app/Model/AnalystData.php b/app/Model/AnalystData.php
index 1e1d0620c..e928216f8 100644
--- a/app/Model/AnalystData.php
+++ b/app/Model/AnalystData.php
@@ -641,9 +641,8 @@ class AnalystData extends AppModel
             return [];
         }
         $this->Server = ClassRegistry::init('Server');
-        $this->AnalystData = ClassRegistry::init('AnalystData');
 
-        $this->log("Starting Analyst Data sync with server #{$server['Server']['id']}", LOG_INFO);
+        $serverSync->debug("Starting Analyst Data sync");
 
         $analystData = $this->collectDataForPush($serverSync->server());
         $keyedAnalystData = [];
@@ -1018,7 +1017,6 @@ class AnalystData extends AppModel
         }
 
         $this->Server = ClassRegistry::init('Server');
-        $this->AnalystData = ClassRegistry::init('AnalystData');
         try {
             $filterRules = $this->buildPullFilterRules($serverSync->server());
             $remoteData = $serverSync->fetchIndexMinimal($filterRules)->json();
diff --git a/app/Model/GalaxyCluster.php b/app/Model/GalaxyCluster.php
index b02cead26..885d959ec 100644
--- a/app/Model/GalaxyCluster.php
+++ b/app/Model/GalaxyCluster.php
@@ -1845,6 +1845,9 @@ class GalaxyCluster extends AppModel
         if (!$compatible) {
             return 0;
         }
+
+        $serverSync->debug("Pulling galaxy clusters with technique $technique");
+
         $clusterIds = $this->getClusterIdListBasedOnPullTechnique($user, $technique, $serverSync);
         $successes = 0;
         // now process the $clusterIds to pull each of the events sequentially
diff --git a/app/Model/Server.php b/app/Model/Server.php
index 933fadf31..efef5f154 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -604,6 +604,7 @@ class Server extends AppModel
      * @throws HttpSocketHttpException
      * @throws HttpSocketJsonException
      * @throws JsonException
+     * @throws Exception
      */
     public function pull(array $user, $technique, array $server, $jobId = false, $force = false)
     {
@@ -619,7 +620,7 @@ class Server extends AppModel
         try {
             $server['Server']['version'] = $serverSync->info()['version'];
         } catch (Exception $e) {
-            $this->logException("Could not get remote server `{$server['Server']['name']}` version.", $e);
+            $this->logException("Could not get remote server `{$serverSync->serverName()}` version.", $e);
             if ($e instanceof HttpSocketHttpException && $e->getCode() === 403) {
                 $message = __('Not authorised. This is either due to an invalid auth key, or due to the sync user not having authentication permissions enabled on the remote server. Another reason could be an incorrect sync server setting.');
             } else {
@@ -648,6 +649,8 @@ class Server extends AppModel
             }
         }
 
+        $serverSync->debug("Pulling event list with technique $technique");
+
         try {
             $eventIds = $this->__getEventIdListBasedOnPullTechnique($technique, $serverSync, $force);
         } catch (Exception $e) {
@@ -673,26 +676,29 @@ class Server extends AppModel
                 $job->saveProgress($jobId, __n('Pulling %s event.', 'Pulling %s events.', count($eventIds), count($eventIds)));
             }
             foreach ($eventIds as $k => $eventId) {
+                $serverSync->debug("Pulling event $eventId");
                 $this->__pullEvent($eventId, $successes, $fails, $eventModel, $serverSync, $user, $jobId, $force);
                 if ($jobId && $k % 10 === 0) {
                     $job->saveProgress($jobId, null, 10 + 40 * (($k + 1) / count($eventIds)));
                 }
             }
             foreach ($fails as $eventid => $message) {
-                $this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['Server']['id'], "Failed to pull event #$eventid.", 'Reason: ' . $message);
+                $this->loadLog()->createLogEntry($user, 'pull', 'Server', $serverSync->serverId(), "Failed to pull event #$eventid.", 'Reason: ' . $message);
             }
         }
         if ($jobId) {
             $job->saveProgress($jobId, 'Pulling proposals.', 50);
         }
-        $pulledProposals = $pulledSightings = 0;
+        $pulledProposals = $pulledSightings = $pulledAnalystData = 0;
         if ($technique === 'full' || $technique === 'update') {
             $pulledProposals = $eventModel->ShadowAttribute->pullProposals($user, $serverSync);
 
             if ($jobId) {
                 $job->saveProgress($jobId, 'Pulling sightings.', 75);
             }
+
             $pulledSightings = $eventModel->Sighting->pullSightings($user, $serverSync);
+
             $this->AnalystData = ClassRegistry::init('AnalystData');
             $pulledAnalystData = $this->AnalystData->pull($user, $serverSync);
         }
@@ -819,7 +825,7 @@ class Server extends AppModel
      */
     public function getElligibleClusterIdsFromServerForPull(ServerSyncTool $serverSync, $onlyUpdateLocalCluster=true, array $eligibleClusters=array(), array $conditions=array())
     {
-        $this->log("Fetching eligible clusters from server #{$serverSync->serverId()} for pull: " . JsonTool::encode($conditions), LOG_INFO);
+        $serverSync->debug("Fetching eligible clusters for pull: " . JsonTool::encode($conditions));
 
         if ($onlyUpdateLocalCluster && empty($eligibleClusters)) {
             return []; // no clusters for update
@@ -875,7 +881,7 @@ class Server extends AppModel
      */
     private function getElligibleClusterIdsFromServerForPush(ServerSyncTool $serverSync, array $localClusters=array(), array $conditions=array())
     {
-        $this->log("Fetching eligible clusters from server #{$serverSync->serverId()} for push: " . JsonTool::encode($conditions), LOG_INFO);
+        $serverSync->debug("Fetching eligible clusters for push: " . JsonTool::encode($conditions));
         $clusterArray = $this->fetchCustomClusterIdsFromServer($serverSync, $conditions=$conditions);
         $keyedClusterArray = Hash::combine($clusterArray, '{n}.GalaxyCluster.uuid', '{n}.GalaxyCluster.version');
         if (!empty($localClusters)) {
@@ -1372,7 +1378,7 @@ class Server extends AppModel
             return []; // pushing clusters is not enabled
         }
 
-        $this->log("Starting $technique clusters sync with server #{$serverSync->serverId()}", LOG_INFO);
+        $serverSync->debug("Starting $technique clusters sync");
 
         $this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
         $this->Event = ClassRegistry::init('Event');
diff --git a/app/Model/ShadowAttribute.php b/app/Model/ShadowAttribute.php
index 645da2dec..a1e6b002b 100644
--- a/app/Model/ShadowAttribute.php
+++ b/app/Model/ShadowAttribute.php
@@ -706,6 +706,8 @@ class ShadowAttribute extends AppModel
             return 0;
         }
 
+        $serverSync->debug("Pulling proposals");
+
         $i = 1;
         $fetchedCount = 0;
         $chunkSize = 1000;
diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php
index 0a33a87ca..8170eae23 100644
--- a/app/Model/Sighting.php
+++ b/app/Model/Sighting.php
@@ -1418,6 +1418,8 @@ class Sighting extends AppModel
      */
     public function pullSightings(array $user, ServerSyncTool $serverSync)
     {
+        $serverSync->debug("Fetching event index for pulling sightings");
+
         $this->Server = ClassRegistry::init('Server');
         try {
             $remoteEvents = $this->Server->getEventIndexFromServer($serverSync);
@@ -1452,6 +1454,8 @@ class Sighting extends AppModel
             return 0;
         }
 
+        $serverSync->debug("Pulling sightings for " . count($eventUuids) . " events");
+
         if ($serverSync->isSupported(ServerSyncTool::FEATURE_SIGHTING_REST_SEARCH)) {
             return $this->pullSightingNewWay($user, $eventUuids, $serverSync);
         } else {

From e2b5e6edc38773fad3fc621e4bd5808fdbca152c Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 30 Mar 2024 15:28:37 +0100
Subject: [PATCH 04/45] chg: [CI] Split logs in CI

---
 .github/workflows/main.yml | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index d60373e3a..f9d766b50 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -269,13 +269,16 @@ jobs:
       - name: Check requirements.txt
         run: python tests/check_requirements.py
 
-      - name: Logs
+      - name: System logs
         if: ${{ always() }}
         # update logs_test.sh when adding more logsources here
         run: |
           tail -n +1 `pwd`/app/tmp/logs/*
           tail -n +1 /var/log/apache2/*.log
 
+      - name: Application logs
+        if: ${{ always() }}
+        run: |
           app/Console/cake Log export /tmp/logs.json.gz --without-changes
           zcat /tmp/logs.json.gz
 

From 8cd3cb0ef258fee08947d30cb8320787127a21db Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 30 Mar 2024 18:56:29 +0100
Subject: [PATCH 05/45] chg: [internal] Ltrim response in
 HttpSocketHttpException

---
 app/Lib/Tools/HttpSocketExtended.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Lib/Tools/HttpSocketExtended.php b/app/Lib/Tools/HttpSocketExtended.php
index cfe097c28..ee6443fff 100644
--- a/app/Lib/Tools/HttpSocketExtended.php
+++ b/app/Lib/Tools/HttpSocketExtended.php
@@ -24,7 +24,7 @@ class HttpSocketHttpException extends Exception
             $message .= " for URL $url";
         }
         if ($response->body) {
-            $message .= ': ' . substr($response->body, 0, 100);
+            $message .= ': ' . substr(ltrim($response->body), 0, 100);
         }
 
         parent::__construct($message, (int)$response->code);

From a322217cbd42d392b5707914a5e3eb523edf03f1 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Wed, 3 Apr 2024 10:47:25 +0200
Subject: [PATCH 06/45] chg: [sync] Try to save memory when fetching sightings

---
 app/Model/Server.php   | 5 +++--
 app/Model/Sighting.php | 2 +-
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/app/Model/Server.php b/app/Model/Server.php
index efef5f154..28912a878 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -924,13 +924,14 @@ class Server extends AppModel
         $indexFromCache = $redis->get("misp:event_index:{$serverSync->serverId()}");
         if ($indexFromCache) {
             list($etag, $eventIndex) = RedisTool::deserialize(RedisTool::decompress($indexFromCache));
+            unset($indexFromCache);
         } else {
             $etag = '""';  // Provide empty ETag, so MISP will compute ETag for returned data
         }
 
         $response = $serverSync->eventIndex($filterRules, $etag);
 
-        if ($response->isNotModified() && $indexFromCache) {
+        if ($response->isNotModified() && isset($eventIndex)) {
             return $eventIndex;
         }
 
@@ -946,7 +947,7 @@ class Server extends AppModel
         if ($etag) {
             $data = RedisTool::compress(RedisTool::serialize([$etag, $eventIndex]));
             $redis->setex("misp:event_index:{$serverSync->serverId()}", 3600 * 24, $data);
-        } elseif ($indexFromCache) {
+        } elseif (isset($eventIndex)) {
             RedisTool::unlink($redis, "misp:event_index:{$serverSync->serverId()}");
         }
 
diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php
index 8170eae23..fc1b1096e 100644
--- a/app/Model/Sighting.php
+++ b/app/Model/Sighting.php
@@ -1424,7 +1424,7 @@ class Sighting extends AppModel
         try {
             $remoteEvents = $this->Server->getEventIndexFromServer($serverSync);
         } catch (Exception $e) {
-            $this->logException("Could not fetch event IDs from server {$serverSync->server()['Server']['name']}", $e);
+            $this->logException("Could not fetch event IDs from server {$serverSync->serverName()}", $e);
             return 0;
         }
         // Remove events from list that do not have published sightings.

From 8a42cf460d98ab67b3fbd967498d88ebe9a31eec Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Wed, 3 Apr 2024 12:56:13 +0200
Subject: [PATCH 07/45] chg: [sync] Reduce default timeout for remote HTTP
 request to 300 seconds (5 mins)

---
 app/Lib/Tools/CurlClient.php | 19 +++++++++++++------
 app/Lib/Tools/SyncTool.php   |  9 +++++----
 app/Model/Server.php         |  4 ++--
 3 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/app/Lib/Tools/CurlClient.php b/app/Lib/Tools/CurlClient.php
index 006b58f3f..4e19b8462 100644
--- a/app/Lib/Tools/CurlClient.php
+++ b/app/Lib/Tools/CurlClient.php
@@ -6,8 +6,12 @@ class CurlClient extends HttpSocketExtended
     /** @var resource */
     private $ch;
 
-    /** @var int */
-    private $timeout = 10800;
+    /**
+     * Maximum time the transfer is allowed to complete in seconds
+     * 300 seconds is recommended timeout for MISP servers
+     * @var int
+     */
+    private $timeout = 300;
 
     /** @var string|null */
     private $caFile;
@@ -30,6 +34,9 @@ class CurlClient extends HttpSocketExtended
     /** @var array */
     private $proxy = [];
 
+    /** @var array */
+    private $defaultOptions;
+
     /**
      * @param array $params
      * @noinspection PhpMissingParentConstructorInspection
@@ -38,8 +45,6 @@ class CurlClient extends HttpSocketExtended
     {
         if (isset($params['timeout'])) {
             $this->timeout = $params['timeout'];
-        } else {
-            $this->timeout = Configure::check('MISP.curl_request_timeout') ? Configure::read('MISP.curl_request_timeout') : 10800;
         }
         if (isset($params['ssl_cafile'])) {
             $this->caFile = $params['ssl_cafile'];
@@ -59,6 +64,7 @@ class CurlClient extends HttpSocketExtended
         if (isset($params['ssl_verify_peer'])) {
             $this->verifyPeer = $params['ssl_verify_peer'];
         }
+        $this->defaultOptions = $this->generateDefaultOptions();
     }
 
     /**
@@ -166,6 +172,7 @@ class CurlClient extends HttpSocketExtended
             return;
         }
         $this->proxy = compact('host', 'port', 'method', 'user', 'pass');
+        $this->defaultOptions = $this->generateDefaultOptions(); // regenerate default options in case proxy setting is changed
     }
 
     /**
@@ -196,7 +203,7 @@ class CurlClient extends HttpSocketExtended
             $url .= '?' . http_build_query($query, '', '&', PHP_QUERY_RFC3986);
         }
 
-        $options = $this->generateOptions();
+        $options = $this->defaultOptions; // this will copy default options
         $options[CURLOPT_URL] = $url;
         $options[CURLOPT_CUSTOMREQUEST] = $method;
 
@@ -303,7 +310,7 @@ class CurlClient extends HttpSocketExtended
     /**
      * @return array
      */
-    private function generateOptions()
+    private function generateDefaultOptions()
     {
         $options = [
             CURLOPT_FOLLOWLOCATION => true, // Allows to follow redirect
diff --git a/app/Lib/Tools/SyncTool.php b/app/Lib/Tools/SyncTool.php
index b0ef40a41..46172c56d 100644
--- a/app/Lib/Tools/SyncTool.php
+++ b/app/Lib/Tools/SyncTool.php
@@ -50,7 +50,7 @@ class SyncTool
      * @return HttpSocketExtended
      * @throws Exception
      */
-    public function createHttpSocket($params = array())
+    public function createHttpSocket(array $params = [])
     {
         // Use own CA PEM file
         $caPath = Configure::read('MISP.ca_path');
@@ -82,10 +82,11 @@ class SyncTool
             }
             $params['ssl_crypto_method'] = $version;
         }
-        if (!isset($params['timeout'])) {
-            $params['timeout'] = Configure::check('MISP.curl_request_timeout') ? Configure::read('MISP.curl_request_timeout') : 10800;
-        }
+
         if (function_exists('curl_init')) {
+            if (!isset($params['timeout']) && Configure::check('MISP.curl_request_timeout')) {
+                $params['timeout'] = (int)Configure::read('MISP.curl_request_timeout');
+            }
             App::uses('CurlClient', 'Tools');
             $HttpSocket = new CurlClient($params);
         } else {
diff --git a/app/Model/Server.php b/app/Model/Server.php
index 8110755a8..3547b590d 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -5125,8 +5125,8 @@ class Server extends AppModel
                 ),
                 'curl_request_timeout' => [
                     'level' => 1,
-                    'description' => __('Control the timeout of curl requests issued by MISP (during synchronisation, feed fetching, etc.'),
-                    'value' => 10800,
+                    'description' => __('Control the default timeout in seconds of curl HTTP requests issued by MISP (during synchronisation, feed fetching, etc.)'),
+                    'value' => 300,
                     'test' => 'testForNumeric',
                     'type' => 'numeric',
                     'null' => true

From b54eec95c189cd3a70726dffd054da4117e4fee9 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Mon, 8 Apr 2024 11:36:51 +0200
Subject: [PATCH 08/45] fix: [dashboard:widgetAdd] Improved error handling for
 invalid JSON config

---
 app/webroot/js/misp.js | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/app/webroot/js/misp.js b/app/webroot/js/misp.js
index d30228064..7a3eb09ac 100644
--- a/app/webroot/js/misp.js
+++ b/app/webroot/js/misp.js
@@ -5385,6 +5385,18 @@ function submitDashboardAddWidget() {
     var height = $('#DashboardHeight').val();
     var el = null;
     var k = $('#last-element-counter').data('element-counter');
+
+    if (config === '') {
+        config = '[]'
+    }
+    try {
+        config = JSON.parse(config);
+    } catch (error) {
+        showMessage('fail', error.message)
+        return
+    }
+    config = JSON.stringify(config);
+
     $.ajax({
         url: baseurl + '/dashboards/getEmptyWidget/' + widget + '/' + (k+1),
         type: 'GET',
@@ -5398,12 +5410,6 @@ function submitDashboardAddWidget() {
                     "autoposition": 1
                 }
             );
-            if (config !== '') {
-                config = JSON.parse(config);
-                config = JSON.stringify(config);
-            } else {
-                config = '[]';
-            }
             $('#widget_' + (k+1)).attr('config', config);
             saveDashboardState();
             $('#last-element-counter').data('element-counter', (k+1));

From c4c395af3125623596ce22fb4a55350d2d14fc39 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Mon, 8 Apr 2024 14:48:04 +0200
Subject: [PATCH 09/45] new: [feed] Added unpublish_event setting to ensure
 pulled events are in the unpublished state

---
 app/Model/Feed.php     | 3 +++
 app/View/Feeds/add.ctp | 5 +++++
 2 files changed, 8 insertions(+)

diff --git a/app/Model/Feed.php b/app/Model/Feed.php
index ef448a536..74acee2ae 100644
--- a/app/Model/Feed.php
+++ b/app/Model/Feed.php
@@ -1068,6 +1068,9 @@ class Feed extends AppModel
             if (!empty($feed['Feed']['settings']['disable_correlation'])) {
                 $event['Event']['disable_correlation'] = (bool) $feed['Feed']['settings']['disable_correlation'];
             }
+            if (!empty($feed['Feed']['settings']['unpublish_event'])) {
+                $event['Event']['published'] = (bool) $feed['Feed']['settings']['unpublish_event'];
+            }
         }
         return $event;
     }
diff --git a/app/View/Feeds/add.ctp b/app/View/Feeds/add.ctp
index 936b3b49f..f1032c8c0 100755
--- a/app/View/Feeds/add.ctp
+++ b/app/View/Feeds/add.ctp
@@ -26,6 +26,11 @@ echo $this->element('genericElements/Form/genericForm', [
                 'label' => __('Disable correlation'),
                 'type' => 'checkbox'
             ],
+            [
+                'field' => 'Feed.settings.unpublish_event',
+                'label' => __('Unpublish events'),
+                'type' => 'checkbox'
+            ],
             [
                 'field' => 'name',
                 'label' => __('Name'),

From fc922910929e7bbaf2a89c2e3387c3f743910549 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Mon, 8 Apr 2024 15:38:29 +0200
Subject: [PATCH 10/45] new: [event:index] Added support of ANDed tag filtering
 in the backend

In addition of the OR filtering using searchtag:1|2, /events/index now supports AND filtering with searchtag:1&2.
The UI has not been updated yet.
---
 app/Controller/EventsController.php | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php
index 4e695ac67..418a40cce 100644
--- a/app/Controller/EventsController.php
+++ b/app/Controller/EventsController.php
@@ -497,6 +497,11 @@ class EventsController extends AppController
                         continue 2;
                     }
                     $pieces = is_array($v) ? $v : explode('|', $v);
+                    $isANDed = false;
+                    if (count($pieces) == 1 && strpos($pieces[0], '&') !== -1) {
+                        $pieces = explode('&', $v);
+                        $isANDed = count($pieces) > 1;
+                    }
                     $filterString = "";
                     $expectOR = false;
                     $tagRules = [];
@@ -563,10 +568,19 @@ class EventsController extends AppController
                     }
 
                     if (!empty($tagRules['include'])) {
-                        $include = $this->Event->EventTag->find('column', array(
-                            'conditions' => array('EventTag.tag_id' => $tagRules['include']),
-                            'fields' => ['EventTag.event_id'],
-                        ));
+                        if ($isANDed) {
+                            $include = $this->Event->EventTag->find('column', array(
+                                'conditions' => ['EventTag.tag_id' => $tagRules['include']],
+                                'fields' => ['EventTag.event_id'],
+                                'group' => ['EventTag.event_id'],
+                                'having' => ['COUNT(*) =' => count($tagRules['include'])],
+                            ));
+                        } else {
+                            $include = $this->Event->EventTag->find('column', array(
+                                'conditions' => array('EventTag.tag_id' => $tagRules['include']),
+                                'fields' => ['EventTag.event_id'],
+                            ));
+                        }
                         if (!empty($include)) {
                             $this->paginate['conditions']['AND'][] = 'Event.id IN (' . implode(",", $include) . ')';
                         } else {

From 5235b9729cc88a7f45e87267c03be319cfa0f114 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Mon, 8 Apr 2024 16:37:57 +0200
Subject: [PATCH 11/45] fix: [widget:EventEvolutionWidget] Fixed filtering on
 organisation not working as expected

---
 app/Lib/Dashboard/EventEvolutionLineWidget.php | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/app/Lib/Dashboard/EventEvolutionLineWidget.php b/app/Lib/Dashboard/EventEvolutionLineWidget.php
index a4c06e007..680e17355 100644
--- a/app/Lib/Dashboard/EventEvolutionLineWidget.php
+++ b/app/Lib/Dashboard/EventEvolutionLineWidget.php
@@ -58,9 +58,11 @@ class EventEvolutionLineWidget
             'recursive' => -1
         ];
         $eparams = [];
+        $filteringOnOrg = false;
         if (!empty($options['filter']) && is_array($options['filter'])) {
             foreach ($this->validFilterKeys as $filterKey) {
                 if (!empty($options['filter'][$filterKey])) {
+                    $filteringOnOrg = true;
                     if (!is_array($options['filter'][$filterKey])) {
                         $options['filter'][$filterKey] = [$options['filter'][$filterKey]];
                     }
@@ -87,6 +89,9 @@ class EventEvolutionLineWidget
             'conditions' => $oparams['conditions'],
             'fields' => ['id']
         ]);
+        if ($filteringOnOrg) {
+            $eparams['conditions']['AND']['Event.orgc_id IN'] = !empty($org_ids) ? $org_ids : [-1];
+        }
         $this->Event->virtualFields = [
             'published_date' => null
         ];

From 05be8033937a1fffce83a3871ca5e339fa742918 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Mon, 8 Apr 2024 16:41:46 +0200
Subject: [PATCH 12/45] fix: [dashboard:updating] Prevent sending multiple time
 the same save request[1;5D

---
 app/Controller/AppController.php | 2 +-
 app/View/Dashboards/index.ctp    | 2 +-
 app/webroot/js/misp.js           | 1 -
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php
index 56486d44f..6c2e65259 100755
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@@ -33,7 +33,7 @@ class AppController extends Controller
 
     public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
 
-    private $__queryVersion = '159';
+    private $__queryVersion = '160';
     public $pyMispVersion = '2.4.188';
     public $phpmin = '7.2';
     public $phprec = '7.4';
diff --git a/app/View/Dashboards/index.ctp b/app/View/Dashboards/index.ctp
index 278f588f3..4e0b31775 100644
--- a/app/View/Dashboards/index.ctp
+++ b/app/View/Dashboards/index.ctp
@@ -23,7 +23,7 @@ $(function () {
         saveDashboardState();
     });
     grid.on('added', function(event, items) {
-        resetDashboardGrid(grid);
+        resetDashboardGrid(grid, false);
     });
     grid.on('gsresizestop', function(event, element) {
         $(element).find('.widgetContentInner').trigger('widget-resized')
diff --git a/app/webroot/js/misp.js b/app/webroot/js/misp.js
index 7a3eb09ac..d1a56dc8a 100644
--- a/app/webroot/js/misp.js
+++ b/app/webroot/js/misp.js
@@ -5411,7 +5411,6 @@ function submitDashboardAddWidget() {
                 }
             );
             $('#widget_' + (k+1)).attr('config', config);
-            saveDashboardState();
             $('#last-element-counter').data('element-counter', (k+1));
         },
         complete: function(data) {

From e2dbc690ac4d29000e69a12e4bfc2b8139d8ff5b Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Mon, 8 Apr 2024 19:45:30 +0200
Subject: [PATCH 13/45] chg: [sync] Enable garbage collector when pulling
 events from remote server

---
 app/Console/Command/ServerShell.php | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/app/Console/Command/ServerShell.php b/app/Console/Command/ServerShell.php
index ad2d8df72..0d69a9a33 100644
--- a/app/Console/Command/ServerShell.php
+++ b/app/Console/Command/ServerShell.php
@@ -145,6 +145,10 @@ class ServerShell extends AppShell
         if (!empty($this->args[4]) && $this->args[4] === 'force') {
             $force = true;
         }
+
+        // Try to enable garbage collector as pulling events can use a lot of memory
+        gc_enable();
+
         try {
             $result = $this->Server->pull($user, $technique, $server, $jobId, $force);
             if (is_array($result)) {

From 45176f7dcdd421e5c814c516de3cf7d4332b2ee9 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Tue, 9 Apr 2024 13:41:56 +0200
Subject: [PATCH 14/45] chg: [statistics] (R)etrieval (o)f (m)ember (m)etrics
 (e)valuation (l)ist (f)or (s)tatistics changed

- will include soft deleted attributes too
---
 app/Console/Command/StatisticsShell.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/Console/Command/StatisticsShell.php b/app/Console/Command/StatisticsShell.php
index 93ca39500..a3591b928 100644
--- a/app/Console/Command/StatisticsShell.php
+++ b/app/Console/Command/StatisticsShell.php
@@ -303,10 +303,10 @@ class StatisticsShell extends AppShell {
         $this->out(json_encode([
             'events' => $this->Event->find('count'),
             'attributes' => $this->Event->Attribute->find('count',
-                ['conditions' => ['Attribute.deleted' => 0], 'recursive' => -1]
+                ['recursive' => -1]
             ),
             'objects' => $this->Event->Object->find('count',
-                ['conditions' => ['Object.deleted' => 0], 'recursive' => -1]
+                ['recursive' => -1]
             ),
             'correlations' => $this->Correlation->find('count') / 2,
             'users' => $this->User->find('count',

From 004b18e1d95a9c9f213f16965f39b5ae1cfec073 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 10 Apr 2024 12:16:49 +0200
Subject: [PATCH 15/45] fix: [component:restSearch] Restored behavior of
 searching for org and cluster metadata

---
 app/Controller/AppController.php                 | 14 +++++++++++++-
 app/Controller/Component/RestSearchComponent.php |  6 +++++-
 2 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php
index 6c2e65259..b84fac808 100755
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@@ -1033,7 +1033,19 @@ class AppController extends Controller
                     $data = array_merge($data, $temp);
                 } else {
                     foreach ($options['paramArray'] as $param) {
-                        if (isset($temp[$param])) {
+                        if (substr($param, -1) == '*') {
+                            $root = substr($param, 0, strlen($param)-1);
+                            foreach ($temp as $existingParamKey => $v) {
+                                $leftover = substr($existingParamKey, strlen($param)-1);
+                                if (
+                                    $root == substr($existingParamKey, 0, strlen($root)) &&
+                                    preg_match('/^[\w_-. ]+$/', $leftover) == 1
+                                ) {
+                                    $data[$existingParamKey] = $temp[$existingParamKey];
+                                    break;
+                                }
+                            }
+                        } else if (isset($temp[$param])) {
                             $data[$param] = $temp[$param];
                         }
                     }
diff --git a/app/Controller/Component/RestSearchComponent.php b/app/Controller/Component/RestSearchComponent.php
index ae8a4a34e..9e8ab024f 100644
--- a/app/Controller/Component/RestSearchComponent.php
+++ b/app/Controller/Component/RestSearchComponent.php
@@ -144,7 +144,11 @@ class RestSearchComponent extends Component
             'retry',
             'expiry',
             'minimum_ttl',
-            'ttl'
+            'ttl',
+            'org.sector',
+            'org.local',
+            'org.nationality',
+            'galaxy.*',
         ],
         'Object' => [
             'returnFormat',

From c2d614f87884d801a697b4d963d21fc10a0d0539 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 10 Apr 2024 15:36:09 +0200
Subject: [PATCH 16/45] fix: [tagCollection:removeTag] Fixed incorrect
 permission check

---
 app/Controller/TagCollectionsController.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Controller/TagCollectionsController.php b/app/Controller/TagCollectionsController.php
index 0131790ab..0f204e261 100644
--- a/app/Controller/TagCollectionsController.php
+++ b/app/Controller/TagCollectionsController.php
@@ -357,7 +357,7 @@ class TagCollectionsController extends AppController
             if (!$tagCollection) {
                 throw new NotFoundException(__('Invalid tag collection.'));
             }
-            if ($this->ACL->canModifyTagCollection($this->Auth->user(), $tagCollection)) {
+            if (!$this->ACL->canModifyTagCollection($this->Auth->user(), $tagCollection)) {
                 throw new ForbiddenException(__('You dont have a permission to do that'));
             }
             $tagCollectionTag = $this->TagCollection->TagCollectionTag->find('first', [

From 6e9d748f08b7f3ec5bc4934dba6dabb6a1e06e96 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 11 Apr 2024 09:40:18 +0200
Subject: [PATCH 17/45] fix: [eventreports:transformFreeTextIntoSuggestion] Add
 to_ids fallback value

---
 app/Model/EventReport.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Model/EventReport.php b/app/Model/EventReport.php
index cbf552612..2e5a592fd 100644
--- a/app/Model/EventReport.php
+++ b/app/Model/EventReport.php
@@ -710,7 +710,7 @@ class EventReport extends AppModel
                 'category' => $typeToCategoryMapping[$complexTypeToolEntry['default_type']][0],
                 'type' => $complexTypeToolEntry['default_type'],
                 'value' => $textToBeReplaced,
-                'to_ids' => $complexTypeToolEntry['to_ids'],
+                'to_ids' => $complexTypeToolEntry['to_ids'] ?? 0,
             ];
             $replacedContent = str_replace($complexTypeToolEntry['original_value'], $textToInject, $replacedContent);
         }

From 309242f358f33869bf922cf8edb9e4769816fbb1 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 11 Apr 2024 09:41:07 +0200
Subject: [PATCH 18/45] chg: [eventReports:extractAllFromReport] Expose
 functionality to API

---
 app/Controller/EventReportsController.php | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/app/Controller/EventReportsController.php b/app/Controller/EventReportsController.php
index 04199f2b5..0563beef6 100644
--- a/app/Controller/EventReportsController.php
+++ b/app/Controller/EventReportsController.php
@@ -213,10 +213,13 @@ class EventReportsController extends AppController
 
     public function extractAllFromReport($reportId)
     {
-        if (!$this->request->is('ajax')) {
+        if (!$this->request->is('ajax') && !$this->_isRest()) {
             throw new MethodNotAllowedException(__('This function can only be reached via AJAX.'));
         }
         if ($this->request->is('post')) {
+            if (!isset($this->data['EventReport'])) {
+                $this->data = ['EventReport' => $this->data];
+            }
             $report = $this->EventReport->fetchIfAuthorized($this->Auth->user(), $reportId, 'edit', $throwErrors=true, $full=false);
             $results = $this->EventReport->getComplexTypeToolResultWithReplacements($this->Auth->user(), $report);
             $report['EventReport']['content'] = $results['replacementResult']['contentWithReplacements'];

From 77a114673a7ee7f1cf95292241af946554632842 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 11 Apr 2024 10:01:52 +0200
Subject: [PATCH 19/45] chg: [analystData:API] Automatically encapsulate
 request's data into the analystType

---
 app/Controller/AnalystDataController.php | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/app/Controller/AnalystDataController.php b/app/Controller/AnalystDataController.php
index 9659555d8..c54800aae 100644
--- a/app/Controller/AnalystDataController.php
+++ b/app/Controller/AnalystDataController.php
@@ -320,6 +320,11 @@ class AnalystDataController extends AppController
                 $this->AnalystData = $this->{$vt};
                 $this->modelClass = $vt;
                 $this->{$vt}->current_user = $this->Auth->user();
+                if (!empty($this->request->data)) {
+                    if (!isset($this->request->data[$type])) {
+                        $this->request->data = [$type => $this->request->data];
+                    }
+                }
                 return $vt;
             }
         }

From ea490063c0a1e922fb4186bf0ac751286b50a3dc Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 11 Apr 2024 10:03:32 +0200
Subject: [PATCH 20/45] fix: [analystData:editableField] Made getEditableFields
 inheritance aware

---
 app/Model/AnalystData.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/Model/AnalystData.php b/app/Model/AnalystData.php
index 1e1d0620c..f7e3a01c8 100644
--- a/app/Model/AnalystData.php
+++ b/app/Model/AnalystData.php
@@ -43,7 +43,7 @@ class AnalystData extends AppModel
         'distribution',
         'sharing_group_id',
     ];
-    protected $EDITABLE_FIELDS = [];
+    public const EDITABLE_FIELDS = [];
 
     /** @var object|null */
     protected $Note;
@@ -185,7 +185,7 @@ class AnalystData extends AppModel
 
     public function getEditableFields(): array
     {
-        return array_merge(self::BASE_EDITABLE_FIELDS, $this->EDITABLE_FIELDS);
+        return array_merge(static::BASE_EDITABLE_FIELDS, static::EDITABLE_FIELDS);
     }
 
     /**

From 0808a6a23d767aa6b3adc364879c1d8a5ab47c5e Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 11 Apr 2024 10:04:53 +0200
Subject: [PATCH 21/45] fix [event:view] Missing variable definition in
 row_object

---
 app/View/Elements/Events/View/row_object.ctp | 1 +
 1 file changed, 1 insertion(+)

diff --git a/app/View/Elements/Events/View/row_object.ctp b/app/View/Elements/Events/View/row_object.ctp
index 1022b609c..9889612ee 100644
--- a/app/View/Elements/Events/View/row_object.ctp
+++ b/app/View/Elements/Events/View/row_object.ctp
@@ -36,6 +36,7 @@ $objectId = intval($object['id']);
           $notes = !empty($object['Note']) ? $object['Note'] : [];
           $opinions = !empty($object['Opinion']) ? $object['Opinion'] : [];
           $relationships = !empty($object['Relationship']) ? $object['Relationship'] : [];
+          $relationshipsInbound = !empty($object['RelationshipInbound']) ? $object['RelationshipInbound'] : [];
           echo $this->element('genericElements/Analyst_data/generic', [
               'analyst_data' => ['notes' => $notes, 'opinions' => $opinions, 'relationships_outbound' => $relationships, 'relationships_inbound' => $relationshipsInbound],
               'object_uuid' => $object['uuid'],

From 353e8c51956966b7394aaa0a9c20b95e603e9712 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 11 Apr 2024 11:20:04 +0200
Subject: [PATCH 22/45] fix: [users:statistics] Division by 0 when no events or
 no orgs

---
 app/Controller/UsersController.php | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/app/Controller/UsersController.php b/app/Controller/UsersController.php
index 09bc73493..4baa2378e 100644
--- a/app/Controller/UsersController.php
+++ b/app/Controller/UsersController.php
@@ -2071,7 +2071,7 @@ class UsersController extends AppController
 
         $stats['attribute_count'] = $this->User->Event->Attribute->find('count', array('conditions' => array('Attribute.deleted' => 0), 'recursive' => -1));
         $stats['attribute_count_month'] = $this->User->Event->Attribute->find('count', array('conditions' => array('Attribute.timestamp >' => $this_month, 'Attribute.deleted' => 0), 'recursive' => -1));
-        $stats['attributes_per_event'] = round($stats['attribute_count'] / $stats['event_count']);
+        $stats['attributes_per_event'] = $stats['event_count'] != 0 ? round($stats['attribute_count'] / $stats['event_count']) : 0;
 
         $stats['correlation_count'] = $this->User->Event->Attribute->Correlation->find('count', array('recursive' => -1));
 
@@ -2082,7 +2082,7 @@ class UsersController extends AppController
         $stats['org_count'] = count($orgs);
         $stats['local_org_count'] = $local_orgs_count;
         $stats['contributing_org_count'] = $this->User->Event->find('count', array('recursive' => -1, 'group' => array('Event.orgc_id')));
-        $stats['average_user_per_org'] = round($stats['user_count'] / $stats['local_org_count'], 1);
+        $stats['average_user_per_org'] = $stats['local_org_count'] != 0 ?  round($stats['user_count'] / $stats['local_org_count'], 1) : 0;
 
         $this->loadModel('Thread');
         $stats['thread_count'] = $this->Thread->find('count', array('conditions' => array('Thread.post_count >' => 0), 'recursive' => -1));

From b5b041202257df154da69dbf4684c5418317eb3e Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 11 Apr 2024 16:35:58 +0200
Subject: [PATCH 23/45] chg: [ui:galaxy_matrix] Resize matrix header on load

---
 app/Controller/AppController.php | 2 +-
 app/webroot/js/attack_matrix.js  | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php
index b84fac808..60c40f7be 100755
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@@ -33,7 +33,7 @@ class AppController extends Controller
 
     public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
 
-    private $__queryVersion = '160';
+    private $__queryVersion = '161';
     public $pyMispVersion = '2.4.188';
     public $phpmin = '7.2';
     public $phprec = '7.4';
diff --git a/app/webroot/js/attack_matrix.js b/app/webroot/js/attack_matrix.js
index f66b35554..ee509ec75 100644
--- a/app/webroot/js/attack_matrix.js
+++ b/app/webroot/js/attack_matrix.js
@@ -17,6 +17,8 @@
 		});
 
 		adapt_position_from_viewport();
+		var firstTabId = $('#attack-matrix-tabscontroller span[data-toggle="tab"]:first').attr('href');
+		resizeHeader(firstTabId);
 
 		$('.ajax_popover_form .btn-matrix-submit').click(function() {
 			makeTagging(pickedGalaxies);

From a0b92e4c7b073c999004540d693644f83b62b032 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Fri, 12 Apr 2024 10:31:22 +0200
Subject: [PATCH 24/45] fix: [workflow:evaluateConfition] Fixed bug in `in_and`
 operator to make it order independant

---
 app/Model/WorkflowModules/WorkflowBaseModule.php | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/app/Model/WorkflowModules/WorkflowBaseModule.php b/app/Model/WorkflowModules/WorkflowBaseModule.php
index 66defb1bf..8a4e2f668 100644
--- a/app/Model/WorkflowModules/WorkflowBaseModule.php
+++ b/app/Model/WorkflowModules/WorkflowBaseModule.php
@@ -213,6 +213,8 @@ class WorkflowBaseModule
             if ($operator == 'in_or') {
                 return !empty($matching);
             } elseif ($operator == 'in_and') {
+                sort($matching);
+                sort($value);
                 return array_values($matching) == array_values($value);
             } elseif ($operator == 'not_in_or') {
                 return empty($matching);

From a9be1561e18e386a701a674fbeec73820c55a314 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Fri, 12 Apr 2024 10:34:58 +0200
Subject: [PATCH 25/45] new: [workflowMouldes:stop-execution] Added message
 paramter to allow user to provide a reason why the execution was stopped

---
 .../action/Module_stop_execution.php             | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/app/Model/WorkflowModules/action/Module_stop_execution.php b/app/Model/WorkflowModules/action/Module_stop_execution.php
index 5266a1557..b02bce689 100644
--- a/app/Model/WorkflowModules/action/Module_stop_execution.php
+++ b/app/Model/WorkflowModules/action/Module_stop_execution.php
@@ -6,6 +6,7 @@ class Module_stop_execution extends WorkflowBaseActionModule
     public $blocking = true;
     public $id = 'stop-execution';
     public $name = 'Stop execution';
+    public $version = '0.2';
     public $description = 'Essentially stops the execution for blocking workflows. Do nothing for non-blocking ones';
     public $icon = 'ban';
     public $inputs = 1;
@@ -15,12 +16,25 @@ class Module_stop_execution extends WorkflowBaseActionModule
     public function __construct()
     {
         parent::__construct();
+
+        $this->params = [
+            [
+                'id' => 'message',
+                'label' => 'Stop message',
+                'type' => 'input',
+                'default' => __('Execution stopped'),
+                'placeholder' => __('Execution stopped'),
+                'jinja_supported' => true,
+            ],
+        ];
     }
 
     public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
     {
         parent::exec($node, $roamingData, $errors);
-        $errors[] = __('Execution stopped');
+        $rData = $roamingData->getData();
+        $params = $this->getParamsWithValues($node, $rData);
+        $errors[] = empty($params['message']['value']) ? $params['message']['default'] : $params['message']['value'];
         return false;
     }
 }

From 9060c21adf256a3dcd8a3e4ac4c1794e9f66a4bf Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Fri, 12 Apr 2024 10:35:47 +0200
Subject: [PATCH 26/45] chg: [workflowModules:distribution-if] Allow choosing
 `sharing-group` and keeping the selected sharing-group list empty

This enables users to simply check that the sharing-group distribution was used
---
 .../logic/Module_distribution_if.php          | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/app/Model/WorkflowModules/logic/Module_distribution_if.php b/app/Model/WorkflowModules/logic/Module_distribution_if.php
index 6135385f2..a4f98cecd 100644
--- a/app/Model/WorkflowModules/logic/Module_distribution_if.php
+++ b/app/Model/WorkflowModules/logic/Module_distribution_if.php
@@ -5,7 +5,7 @@ class Module_distribution_if extends WorkflowBaseLogicModule
 {
     public $id = 'distribution-if';
     public $name = 'IF :: Distribution';
-    public $version = '0.2';
+    public $version = '0.3';
     public $description = 'Distribution IF / ELSE condition block. The `then` output will be used if the encoded conditions is satisfied, otherwise the `else` output will be used.';
     public $icon = 'code-branch';
     public $inputs = 1;
@@ -103,12 +103,15 @@ class Module_distribution_if extends WorkflowBaseLogicModule
             $final_sharing_group = $this->__extractSharingGroupIDs(
                 $data['Event'],
                 $data['Event']['Attribute'][0]['Object'] ?? [],
-                $data['Event']['Attribute'][0]
+                $data['Event']['Attribute'][0],
+                $scope
             );
             if ($operator == 'equals') {
-                return !array_diff($final_sharing_group, $selected_sharing_groups); // All sharing groups are in the selection
+                return empty($selected_sharing_groups) ? !empty($final_sharing_group) :
+                    !array_diff($final_sharing_group, $selected_sharing_groups); // All sharing groups are in the selection
             } else if ($operator == 'not_equals') {
-                return count(array_diff($final_sharing_group, $selected_sharing_groups)) == count($final_sharing_group); // All sharing groups are in the selection
+                return empty($selected_sharing_groups) ? empty($final_sharing_group) :
+                    count(array_diff($final_sharing_group, $selected_sharing_groups)) == count($final_sharing_group); // All sharing groups are in the selection
             }
             $errors[] = __('Condition operator not supported for that distribution level');
             return false;
@@ -159,9 +162,15 @@ class Module_distribution_if extends WorkflowBaseLogicModule
         return min($distri1, $distri2);
     }
 
-    private function __extractSharingGroupIDs(array $event, array $object=[], array $attribute=[]): array
+    private function __extractSharingGroupIDs(array $event, array $object=[], array $attribute=[], $scope='event'): array
     {
         $sgIDs = [];
+        if ($scope == 'event') {
+            if (!empty($event) && $event['distribution'] == 4) {
+                $sgIDs[] = $event['sharing_group_id'];
+            }
+            return $sgIDs;
+        }
         if (!empty($event) && $event['distribution'] == 4) {
             $sgIDs[] = $event['sharing_group_id'];
         }

From 038c41136602669314ba977dcbf2515fa160ba05 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Fri, 12 Apr 2024 15:58:19 +0200
Subject: [PATCH 27/45] new: [feed:pullEvents] Added support of tag collection
 in feed configuration

This allow to specify a tag collection for which all the tags will be applied on the pulled Events
---
 app/Controller/FeedsController.php            | 50 +++++++++++++-
 app/Model/AppModel.php                        |  5 +-
 app/Model/Feed.php                            | 68 +++++++++++++++----
 .../IndexTable/Fields/tags.ctp                | 24 +++++++
 app/View/Feeds/index.ctp                      |  3 +-
 5 files changed, 131 insertions(+), 19 deletions(-)

diff --git a/app/Controller/FeedsController.php b/app/Controller/FeedsController.php
index 5eafbf435..d634bf65c 100644
--- a/app/Controller/FeedsController.php
+++ b/app/Controller/FeedsController.php
@@ -74,6 +74,8 @@ class FeedsController extends AppController
                 );
             }
         }
+        $loggedUser = $this->Auth->user();
+        $this->loadModel('TagCollection');
 
         $this->CRUD->index([
             'filters' => [
@@ -92,7 +94,7 @@ class FeedsController extends AppController
                 'source_format'
             ],
             'conditions' => $conditions,
-            'afterFind' => function (array $feeds) {
+            'afterFind' => function (array $feeds) use ($loggedUser) {
                 if ($this->_isSiteAdmin()) {
                     $feeds = $this->Feed->attachFeedCacheTimestamps($feeds);
                 }
@@ -106,6 +108,22 @@ class FeedsController extends AppController
                     }
                 }
 
+                foreach ($feeds as &$feed) {
+                    if (!empty($feed['Feed']['tag_id'])) {
+                        if (substr($feed['Feed']['tag_id'], 0, 11) == 'collection_') {
+                            $tagCollectionID = intval(substr($feed['Feed']['tag_id'], 11));
+                            $tagCollection = $this->TagCollection->fetchTagCollection($loggedUser, [
+                                'conditions' => [
+                                    'TagCollection.id' => $tagCollectionID,
+                                ]
+                            ]);
+                            if (!empty($tagCollection)) {
+                                $feed['TagCollection'] = $tagCollection;
+                            }
+                        }
+                    }
+                }
+
                 return $feeds;
             }
         ]);
@@ -292,7 +310,20 @@ class FeedsController extends AppController
         if (empty(Configure::read('Security.disable_local_feed_access'))) {
             $inputSources['local'] = 'Local';
         }
-        $tags = $this->Event->EventTag->Tag->find('list', array('fields' => array('Tag.name'), 'order' => array('lower(Tag.name) asc')));
+        $tags = $this->Event->EventTag->Tag->find('all', [
+            'recursive' => -1,
+            'fields' => ['Tag.name', 'Tag.id'],
+            'order' => ['lower(Tag.name) asc']
+        ]);
+        $tags = Hash::combine($tags, '{n}.Tag.id', '{n}.Tag.name');
+        $tagCollections = $this->TagCollection->fetchTagCollection($this->Auth->user());
+        $tagCollections = Hash::combine($tagCollections, '{n}.TagCollection.id', '{n}.TagCollection.name');
+        foreach ($tagCollections as $id => $name) {
+            $newID = "collection_{$id}";
+            $tagCollections[$newID] = sprintf('%s :: %s', __('Tag Collection'), $name);
+            unset($tagCollections[$id]);
+        }
+        $tags = array_merge($tags, $tagCollections);
         $tags[0] = 'None';
 
         $this->loadModel('Server');
@@ -442,7 +473,20 @@ class FeedsController extends AppController
         if (empty(Configure::read('Security.disable_local_feed_access'))) {
             $inputSources['local'] = 'Local';
         }
-        $tags = $this->Event->EventTag->Tag->find('list', array('fields' => array('Tag.name'), 'order' => array('lower(Tag.name) asc')));
+        $tags = $this->Event->EventTag->Tag->find('all', [
+            'recursive' => -1,
+            'fields' => ['Tag.name', 'Tag.id'],
+            'order' => ['lower(Tag.name) asc']
+        ]);
+        $tags = Hash::combine($tags, '{n}.Tag.id', '{n}.Tag.name');
+        $this->loadModel('TagCollection');
+        $tagCollections = $this->TagCollection->fetchTagCollection($this->Auth->user());
+        $tagCollections = Hash::combine($tagCollections, '{n}.TagCollection.id', '{n}.TagCollection.name');
+        foreach ($tagCollections as $id => $name) {
+            $newID = "collection_{$id}";
+            $tags[$newID] = sprintf('%s :: %s', __('Tag Collection'), $name);
+        }
+        unset($tagCollections);
         $tags[0] = 'None';
 
         $this->loadModel('Server');
diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php
index 738c4d3d8..f91cdadc7 100644
--- a/app/Model/AppModel.php
+++ b/app/Model/AppModel.php
@@ -91,7 +91,7 @@ class AppModel extends Model
         105 => false, 106 => false, 107 => false, 108 => false, 109 => false, 110 => false,
         111 => false, 112 => false, 113 => true, 114 => false, 115 => false, 116 => false,
         117 => false, 118 => false, 119 => false, 120 => false, 121 => false, 122 => false,
-        123 => false, 124 => false,
+        123 => false, 124 => false, 125 => false,
     );
 
     const ADVANCED_UPDATES_DESCRIPTION = array(
@@ -2176,6 +2176,9 @@ class AppModel extends Model
                     INDEX `org_name` (`org_name`)
                   ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;';
                 break;
+            case 125:
+                $sqlArray[] = "ALTER TABLE `feeds` MODIFY `tag_id` VARCHAR(40) NOT NULL DEFAULT '0';";
+                break;
             case 'fixNonEmptySharingGroupID':
                 $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
                 $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
diff --git a/app/Model/Feed.php b/app/Model/Feed.php
index 74acee2ae..862985686 100644
--- a/app/Model/Feed.php
+++ b/app/Model/Feed.php
@@ -1041,23 +1041,42 @@ class Feed extends AppModel
                     'recursive' => -1,
                     'fields' => ['Tag.name', 'Tag.colour', 'Tag.id']
                 ]);
-                $feed['Tag'] = $feed_tag['Tag'];
+                if (!empty($feed_tag)) {
+                    $feed['Tag'] = $feed_tag['Tag'];
+                }
             }
             if (!isset($event['Event']['Tag'])) {
                 $event['Event']['Tag'] = array();
             }
 
-            $feedTag = $this->Tag->find('first', array('conditions' => array('Tag.id' => $feed['Feed']['tag_id']), 'recursive' => -1, 'fields' => array('Tag.name', 'Tag.colour', 'Tag.exportable')));
-            if (!empty($feedTag)) {
-                $found = false;
-                foreach ($event['Event']['Tag'] as $tag) {
-                    if (strtolower($tag['name']) === strtolower($feedTag['Tag']['name'])) {
-                        $found = true;
-                        break;
-                    }
+            if (substr($feed['Feed']['tag_id'], 0, 11) == 'collection_') { // Tag is actually a tag collection
+                $this->TagCollection = ClassRegistry::init('TagCollection');
+                $tagCollectionID = intval(substr($feed['Feed']['tag_id'], 11));
+                $tagCollection = $this->TagCollection->find('first', [
+                    'recursive' => -1,
+                    'conditions' => [
+                        'TagCollection.id' => $tagCollectionID,
+                    ],
+                    'contain' => [
+                        'TagCollectionTag' => ['Tag'],
+                    ]
+                ]);
+                foreach ($tagCollection['TagCollectionTag'] as $collectionTag) {
+                    $event['Event']['Tag'][] = $collectionTag['Tag'];
                 }
-                if (!$found) {
-                    $event['Event']['Tag'][] = $feedTag['Tag'];
+            } else {
+                $feedTag = $this->Tag->find('first', array('conditions' => array('Tag.id' => $feed['Feed']['tag_id']), 'recursive' => -1, 'fields' => array('Tag.name', 'Tag.colour', 'Tag.exportable')));
+                if (!empty($feedTag)) {
+                    $found = false;
+                    foreach ($event['Event']['Tag'] as $tag) {
+                        if (strtolower($tag['name']) === strtolower($feedTag['Tag']['name'])) {
+                            $found = true;
+                            break;
+                        }
+                    }
+                    if (!$found) {
+                        $event['Event']['Tag'][] = $feedTag['Tag'];
+                    }
                 }
             }
         }
@@ -1131,9 +1150,13 @@ class Feed extends AppModel
      */
     private function __updateEventFromFeed(HttpSocket $HttpSocket = null, $feed, $uuid, $user, $filterRules)
     {
-        $event = $this->downloadAndParseEventFromFeed($feed, $uuid, $HttpSocket);
+         $event = $this->downloadAndParseEventFromFeed($feed, $uuid, $HttpSocket);
         $event = $this->__prepareEvent($event, $feed, $filterRules);
-        return $this->Event->_edit($event, $user, $uuid, $jobId = null);
+        if (is_array($event)) {
+            return $this->Event->_edit($event, $user, $uuid, $jobId = null);
+        } else {
+            return $event;
+        }
     }
 
     public function addDefaultFeeds($newFeeds)
@@ -1378,7 +1401,24 @@ class Feed extends AppModel
             $this->Event->publishRouter($event['Event']['id'], null, $user);
         }
         if ($feed['Feed']['tag_id']) {
-            $this->Event->EventTag->attachTagToEvent($event['Event']['id'], ['id' => $feed['Feed']['tag_id']]);
+            if (substr($feed['Feed']['tag_id'], 0, 11) == 'collection_') { // Tag is actually a tag collection
+                $this->TagCollection = ClassRegistry::init('TagCollection');
+                $tagCollectionID = intval(substr($feed['Feed']['tag_id'], 11));
+                $tagCollection = $this->TagCollection->find('first', [
+                    'recursive' => -1,
+                    'conditions' => [
+                        'TagCollection.id' => $tagCollectionID,
+                    ],
+                    'contain' => [
+                        'TagCollectionTag',
+                    ]
+                ]);
+                foreach ($tagCollection['TagCollectionTag'] as $collectionTag) {
+                    $this->Event->EventTag->attachTagToEvent($event['Event']['id'], ['id' => $collectionTag['tag_id']]);
+                }
+            } else {
+                $this->Event->EventTag->attachTagToEvent($event['Event']['id'], ['id' => $feed['Feed']['tag_id']]);
+            }
         }
         return true;
     }
diff --git a/app/View/Elements/genericElements/IndexTable/Fields/tags.ctp b/app/View/Elements/genericElements/IndexTable/Fields/tags.ctp
index 859f6fc06..1cb0a9523 100644
--- a/app/View/Elements/genericElements/IndexTable/Fields/tags.ctp
+++ b/app/View/Elements/genericElements/IndexTable/Fields/tags.ctp
@@ -25,5 +25,29 @@
                 'hide_global_scope' => isset($field['hide_global_scope']) ? $field['hide_global_scope'] : false
             ]
         );
+    } else if (!empty($field['includeTagCollection']) && empty($tags)) {
+        if (!empty($row['TagCollection'])) {
+            echo sprintf('<a class="badge" style="background-color: #fff; color: #000; border: 1px solid #000;" title="%s" href="%s">%s :: %s</a>',
+                __('Tag Collection'),
+                '/tag_collections/view/' . h($row['TagCollection'][0]['TagCollection']['id']),
+                __('Tag Collection'),
+                h($row['TagCollection'][0]['TagCollection']['name'])
+            );
+            echo '<div>';
+            echo $this->element(
+                'ajaxTags',
+                [
+                    'scope' => '',
+                    'attributeId' => 0,
+                    'tags' => Hash::extract($row['TagCollection'][0]['TagCollectionTag'], '{n}.Tag'),
+                    'tagAccess' => false,
+                    'localTagAccess' => false,
+                    'static_tags_only' => 1,
+                    'scope' => isset($field['scope']) ? $field['scope'] : 'event',
+                    'hide_global_scope' => isset($field['hide_global_scope']) ? $field['hide_global_scope'] : false
+                    ]
+                );
+            echo '</div>';
+        }
     }
 ?>
diff --git a/app/View/Feeds/index.ctp b/app/View/Feeds/index.ctp
index 9736841ec..965d13971 100644
--- a/app/View/Feeds/index.ctp
+++ b/app/View/Feeds/index.ctp
@@ -193,7 +193,8 @@
                     'class' => 'short',
                     'data_path' => 'Tag',
                     'element' => 'tags',
-                    'scope' => 'feeds'
+                    'scope' => 'feeds',
+                    'includeTagCollection' => true,
                 ),
                 array(
                     'name' => __('Visible'),

From d2176ab8bd6676b75514ea18e31a0834e093125a Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Tue, 9 Apr 2024 12:46:10 +0200
Subject: [PATCH 28/45] chg: [sync] Try to reduce memory usage when fetching
 event index from Redis

---
 app/Model/Server.php | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/app/Model/Server.php b/app/Model/Server.php
index 28912a878..045de7d0a 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -923,8 +923,10 @@ class Server extends AppModel
         $redis = RedisTool::init();
         $indexFromCache = $redis->get("misp:event_index:{$serverSync->serverId()}");
         if ($indexFromCache) {
-            list($etag, $eventIndex) = RedisTool::deserialize(RedisTool::decompress($indexFromCache));
+            $indexFromCache = RedisTool::decompress($indexFromCache);
+            list($etag, $eventIndex) = RedisTool::deserialize($indexFromCache);
             unset($indexFromCache);
+            $serverSync->debug("Event index loaded from Redis cache with etag $etag containing " . count($eventIndex) . ' events');
         } else {
             $etag = '""';  // Provide empty ETag, so MISP will compute ETag for returned data
         }
@@ -935,7 +937,9 @@ class Server extends AppModel
             return $eventIndex;
         }
 
+        unset($eventIndex);
         $eventIndex = $response->json();
+        unset($response->body); // remove response that can take a lot of memory
 
         // correct $eventArray if just one event, probably this response returns old MISP
         if (isset($eventIndex['id'])) {
@@ -945,6 +949,7 @@ class Server extends AppModel
         // Save to cache for 24 hours if ETag provided
         $etag = $response->getHeader('etag');
         if ($etag) {
+            $serverSync->debug("Event index from remote server has different etag $etag, saving to cache");
             $data = RedisTool::compress(RedisTool::serialize([$etag, $eventIndex]));
             $redis->setex("misp:event_index:{$serverSync->serverId()}", 3600 * 24, $data);
         } elseif (isset($eventIndex)) {

From 47d35dae0bc9945b852731a0e8d846151a3d70eb Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 13 Apr 2024 12:42:54 +0200
Subject: [PATCH 29/45] chg: [sync] Change way how event index is cached in
 Redis to save memory

---
 app/Model/Server.php | 42 ++++++++++++++++++++++--------------------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/app/Model/Server.php b/app/Model/Server.php
index 045de7d0a..4ac06c6f7 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -921,39 +921,41 @@ class Server extends AppModel
 
         // Fetch event index from cache if exists and is not modified
         $redis = RedisTool::init();
-        $indexFromCache = $redis->get("misp:event_index:{$serverSync->serverId()}");
+        $indexFromCache = $redis->get("misp:event_index_cache:{$serverSync->serverId()}");
         if ($indexFromCache) {
-            $indexFromCache = RedisTool::decompress($indexFromCache);
-            list($etag, $eventIndex) = RedisTool::deserialize($indexFromCache);
-            unset($indexFromCache);
-            $serverSync->debug("Event index loaded from Redis cache with etag $etag containing " . count($eventIndex) . ' events');
+            $etagPos = strpos($indexFromCache, "\n");
+            if ($etagPos === false) {
+                throw new RuntimeException("Could not find etag in cache fro server {$serverSync->serverId()}");
+            }
+            $etag = substr($indexFromCache, 0, $etagPos);
+            $serverSync->debug("Event index loaded from Redis cache with etag $etag containing");
         } else {
             $etag = '""';  // Provide empty ETag, so MISP will compute ETag for returned data
         }
 
         $response = $serverSync->eventIndex($filterRules, $etag);
 
-        if ($response->isNotModified() && isset($eventIndex)) {
-            return $eventIndex;
-        }
-
-        unset($eventIndex);
-        $eventIndex = $response->json();
-        unset($response->body); // remove response that can take a lot of memory
-
-        // correct $eventArray if just one event, probably this response returns old MISP
-        if (isset($eventIndex['id'])) {
-            $eventIndex = [$eventIndex];
+        if ($response->isNotModified() && $indexFromCache) {
+            return JsonTool::decode(RedisTool::decompress(substr($indexFromCache, $etagPos + 1)));
         }
 
         // Save to cache for 24 hours if ETag provided
         $etag = $response->getHeader('etag');
         if ($etag) {
             $serverSync->debug("Event index from remote server has different etag $etag, saving to cache");
-            $data = RedisTool::compress(RedisTool::serialize([$etag, $eventIndex]));
-            $redis->setex("misp:event_index:{$serverSync->serverId()}", 3600 * 24, $data);
-        } elseif (isset($eventIndex)) {
-            RedisTool::unlink($redis, "misp:event_index:{$serverSync->serverId()}");
+            $data = "$etag\n" . RedisTool::compress($response->body);
+            $redis->setex("misp:event_index_cache:{$serverSync->serverId()}", 3600 * 24, $data);
+        } elseif ($indexFromCache) {
+            RedisTool::unlink($redis, "misp:event_index_cache:{$serverSync->serverId()}");
+        }
+
+        unset($indexFromCache); // clean up memory
+
+        $eventIndex = $response->json();
+
+        // correct $eventArray if just one event, probably this response returns old MISP
+        if (isset($eventIndex['id'])) {
+            $eventIndex = [$eventIndex];
         }
 
         return $eventIndex;

From 8934982ff2606b89399e646fdf88b6524a4dad30 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Mon, 15 Apr 2024 07:23:03 +0200
Subject: [PATCH 30/45] fix: [eventreport] import from url api fixed

---
 app/Controller/EventReportsController.php | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/app/Controller/EventReportsController.php b/app/Controller/EventReportsController.php
index 0563beef6..efeb9ddd4 100644
--- a/app/Controller/EventReportsController.php
+++ b/app/Controller/EventReportsController.php
@@ -302,13 +302,16 @@ class EventReportsController extends AppController
 
     public function importReportFromUrl($event_id)
     {
-        if (!$this->request->is('ajax')) {
-            throw new MethodNotAllowedException(__('This function can only be reached via AJAX.'));
+        if (!$this->request->is('ajax') && !$this->_isRest()) {
+            throw new MethodNotAllowedException(__('This function can only be reached via AJAX and via the API.'));
         }
         $fetcherModule = $this->EventReport->isFetchURLModuleEnabled();
         if ($this->request->is('post')) {
+            if (empty($this->data['EventReport'])) {
+                $this->data = ['EventReport' => $this->data];
+            }
             if (empty($this->data['EventReport']['url'])) {
-                throw new MethodNotAllowedException(__('An URL must be provided'));
+                throw new MethodNotAllowedException(__('A URL must be provided'));
             }
             $url = $this->data['EventReport']['url'];
             $format = 'html';
@@ -319,7 +322,6 @@ class EventReportsController extends AppController
                     $format = $parsed_format;
                 }
             }
-            
             $content = $this->EventReport->downloadMarkdownFromURL($event_id, $url, $format);
 
             $errors = [];

From b5a60b5bfbb7b500770a23d7187832e36f5f87d4 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 17 Apr 2024 11:33:32 +0200
Subject: [PATCH 31/45] fix: [analyst-data:thread] Only render the HTML when
 opening the popover

---
 app/View/AnalystData/view.ctp                 | 50 ++++++++++++-
 .../genericElements/Analyst_data/generic.ctp  | 14 +---
 .../genericElements/Analyst_data/thread.ctp   | 75 ++++++++++++-------
 app/View/GalaxyClusters/view.ctp              | 50 ++++++++++++-
 4 files changed, 139 insertions(+), 50 deletions(-)

diff --git a/app/View/AnalystData/view.ctp b/app/View/AnalystData/view.ctp
index f56460e7a..c9715d63f 100644
--- a/app/View/AnalystData/view.ctp
+++ b/app/View/AnalystData/view.ctp
@@ -120,15 +120,57 @@ echo $this->element(
 );
 
 $object_uuid = Hash::get($data, $modelSelection . '.uuid');
+
+$notes = $data[$modelSelection]['Note'] ?? [];
+$opinions = $data[$modelSelection]['Opinion'] ?? [];
+$relationships_outbound = $data[$modelSelection]['Relationship'] ?? [];
+$relationships_inbound = $data[$modelSelection]['RelationshipInbound'] ?? [];
+$notesOpinions = array_merge($notes, $opinions);
+if(!function_exists("countNotes")) {
+    function countNotes($notesOpinions) {
+        $notesTotalCount = count($notesOpinions);
+        $notesCount = 0;
+        $relationsCount = 0;
+        foreach ($notesOpinions as $notesOpinion) {
+            if ($notesOpinion['note_type'] == 2) { // relationship
+                $relationsCount += 1;
+            } else {
+                $notesCount += 1;
+            }
+            if (!empty($notesOpinion['Note'])) {
+                $nestedCounts = countNotes($notesOpinion['Note']);
+                $notesTotalCount += $nestedCounts['total'];
+                $notesCount += $nestedCounts['notesOpinions'];
+                $relationsCount += $nestedCounts['relations'];
+            }
+            if (!empty($notesOpinion['Opinion'])) {
+                $nestedCounts = countNotes($notesOpinion['Opinion']);
+                $notesTotalCount += $nestedCounts['total'];
+                $notesCount += $nestedCounts['notesOpinions'];
+                $relationsCount += $nestedCounts['relations'];
+            }
+        }
+        return ['total' => $notesTotalCount, 'notesOpinions' => $notesCount, 'relations' => $relationsCount];
+    }
+}
+$counts = countNotes($notesOpinions);
+$notesOpinionCount = $counts['notesOpinions'];
+$allCounts = [
+    'notesOpinions' => $counts['notesOpinions'],
+    'relationships_outbound' => count($relationships_outbound),
+    'relationships_inbound' => count($relationships_inbound),
+];
+
 $options = [
     'container_id' => 'analyst_data_thread',
     'object_type' => $modelSelection,
     'object_uuid' => $object_uuid,
     'shortDist' => $shortDist,
-    'notes' => $data[$modelSelection]['Note'] ?? [],
-    'opinions' => $data[$modelSelection]['Opinion'] ?? [],
-    'relationships_outbound' => $data[$modelSelection]['Relationship'] ?? [],
-    'relationships_inbound' => $data[$modelSelection]['RelationshipInbound'] ?? [],
+    'notes' => $notes,
+    'opinions' => $opinions,
+    'relationships_outbound' => $relationships_outbound,
+    'relationships_inbound' => $relationships_inbound,
+    'allCounts' => $allCounts,
 ];
 
 echo $this->element('genericElements/assetLoader', [
diff --git a/app/View/Elements/genericElements/Analyst_data/generic.ctp b/app/View/Elements/genericElements/Analyst_data/generic.ctp
index 2ccdc4eb0..acbfae91a 100644
--- a/app/View/Elements/genericElements/Analyst_data/generic.ctp
+++ b/app/View/Elements/genericElements/Analyst_data/generic.ctp
@@ -65,20 +65,8 @@ $allCounts = [
 
 $(document).ready(function() {
     $('.node-opener-<?= $seed ?>').click(function() {
-        openNotes(this)
+        openNotes<?= $seed ?>(this)
     })
-
-    function adjustPopoverPosition() {
-        var $popover = $('.popover:last');
-        $popover.css('top', Math.max($popover.position().top, 50) + 'px')
-    }
-
-    function openNotes(clicked) {
-        openPopover(clicked, renderedNotes<?= $seed ?>, undefined, undefined, function() {
-            adjustPopoverPosition()
-            $(clicked).removeClass('have-a-popover') // avoid closing the popover if a confirm popover (like the delete one) is called
-        })
-    }
 })
 </script>
 
diff --git a/app/View/Elements/genericElements/Analyst_data/thread.ctp b/app/View/Elements/genericElements/Analyst_data/thread.ctp
index 5930f6c5d..bad0a3b8d 100644
--- a/app/View/Elements/genericElements/Analyst_data/thread.ctp
+++ b/app/View/Elements/genericElements/Analyst_data/thread.ctp
@@ -4,6 +4,7 @@
     $URL_DELETE = '/analystData/delete/';
 
     $seed = isset($seed) ? $seed : mt_rand();
+    $injectInPage = !empty($container_id) ? true : false;
 
     $notes = !empty($notes) ? $notes : [];
     $opinions = !empty($opinions) ? $opinions : [];
@@ -41,7 +42,28 @@
 if (!window.shortDist) {
     var shortDist = <?= json_encode($shortDist) ?>;
 }
-var renderedNotes<?= $seed ?> = null
+
+var container_id = false
+<?php if (isset($container_id)): ?>
+    container_id = '<?= h($container_id) ?>'
+<?php endif; ?>
+
+function adjustPopoverPosition() {
+    var $popover = $('.popover:last');
+    $popover.css('top', Math.max($popover.position().top, 50) + 'px')
+}
+
+function openNotes<?= $seed ?>(clicked) {
+    var notes = <?= json_encode($notesOpinions) ?>;
+    var relationships = <?= json_encode($relationshipsOutbound) ?>;
+    var relationships_inbound = <?= json_encode($relationshipsInbound) ?>;
+    var relationship_related_object = <?= json_encode($related_objects) ?>;
+    var renderedNotes = renderAllNotesWithForm<?= $seed ?>(notes, relationships, relationships_inbound, relationship_related_object)
+    openPopover(clicked, renderedNotes, undefined, undefined, function() {
+        adjustPopoverPosition()
+        $(clicked).removeClass('have-a-popover') // avoid closing the popover if a confirm popover (like the delete one) is called
+    })
+}
 
 function renderNotes(notes, relationship_related_object, emptyMessage='<?= __('Empty') ?>', isInbound=false) {
     var renderedNotesArray = []
@@ -406,18 +428,7 @@ function fetchMoreNotes(clicked, noteType, uuid) {
 
 }
 
-
-(function() {
-    var notes = <?= json_encode($notesOpinions) ?>;
-    var relationships = <?= json_encode($relationshipsOutbound) ?>;
-    var relationships_inbound = <?= json_encode($relationshipsInbound) ?>;
-    var relationship_related_object = <?= json_encode($related_objects) ?>;
-    var container_id = false
-    <?php if (isset($container_id)): ?>
-        container_id = '<?= h($container_id) ?>'
-    <?php endif; ?>
-
-    var nodeContainerTemplate = doT.template('\
+    var nodeContainerTemplate<?= $seed ?> = doT.template('\
         <div> \
             <ul class="nav nav-tabs" style="margin-bottom: 10px;"> \
                 <li class="active"><a href="#notes-<?= $seed ?>" data-toggle="tab"><i class="<?= $this->FontAwesome->getClass('sticky-note') ?>"></i> <?= __('Notes & Opinions') ?> <span class="label label-secondary"><?= $allCounts['notesOpinions'] ?></span></a></li> \
@@ -439,18 +450,6 @@ function fetchMoreNotes(clicked, noteType, uuid) {
         </div> \
     ')
 
-    function renderAllNotesWithForm(relationship_related_object) {
-        var buttonContainer = '<div id="add-button-container" style="margin-top: 0.5rem;">' + addNoteButton + addOpinionButton + '</div>'
-        renderedNotes<?= $seed ?> = nodeContainerTemplate({
-            content_notes: renderNotes(notes.filter(function(note) { return note.note_type != 2}), relationship_related_object, '<?= __('No notes for this UUID.') ?>') + buttonContainer,
-            content_relationships_outbound: renderNotes(relationships, relationship_related_object, '<?= __('No relationship from this UUID') ?>') + addRelationshipButton,
-            content_relationships_inbound: renderNotes(relationships_inbound, relationship_related_object, '<?= __('No element are referencing this UUID') ?>', true),
-        })
-        if (container_id) {
-            $('#' + container_id).html(renderedNotes<?= $seed ?>)
-        }
-    }
-
     var addNoteButton = '<button class="btn btn-small btn-block btn-primary" type="button" onclick="createNewNote(this, \'<?= $object_type ?>\', \'<?= $object_uuid ?>\')"> \
         <i class="<?= $this->FontAwesome->getClass('plus') ?>"></i> <?= __('Add a note') ?> \
     </button>'
@@ -461,10 +460,15 @@ function fetchMoreNotes(clicked, noteType, uuid) {
         <i class="<?= $this->FontAwesome->getClass('plus') ?>"></i> <?= __('Add a relationship') ?> \
     </button>'
 
-    $(document).ready(function() {
-        renderAllNotesWithForm(relationship_related_object)
-    })
-})()
+    function renderAllNotesWithForm<?= $seed ?>(notes, relationships, relationships_inbound, relationship_related_object) {
+        var buttonContainer = '<div id="add-button-container" style="margin-top: 0.5rem;">' + addNoteButton + addOpinionButton + '</div>'
+        var renderedNotes = nodeContainerTemplate<?= $seed ?>({
+            content_notes: renderNotes(notes.filter(function(note) { return note.note_type != 2}), relationship_related_object, '<?= __('No notes for this UUID.') ?>') + buttonContainer,
+            content_relationships_outbound: renderNotes(relationships, relationship_related_object, '<?= __('No relationship from this UUID') ?>') + addRelationshipButton,
+            content_relationships_inbound: renderNotes(relationships_inbound, relationship_related_object, '<?= __('No element are referencing this UUID') ?>', true),
+        })
+        return renderedNotes
+    }
 
     function createNewNote(clicked, object_type, object_uuid) {
         note_type = 'Note';
@@ -516,6 +520,19 @@ function fetchMoreNotes(clicked, noteType, uuid) {
         }
     }
 
+<?php if(!empty($injectInPage)): ?>
+    $(document).ready(function() {
+        var notes = <?= json_encode($notesOpinions) ?>;
+        var relationships = <?= json_encode($relationshipsOutbound) ?>;
+        var relationships_inbound = <?= json_encode($relationshipsInbound) ?>;
+        var relationship_related_object = <?= json_encode($related_objects) ?>;
+        var renderedNotes = renderAllNotesWithForm<?= $seed ?>(notes, relationships, relationships_inbound, relationship_related_object)
+        if (container_id) {
+            $('#' + container_id).html(renderedNotes)
+        }
+    })
+<?php endif; ?>
+
 </script>
 
 <style>
diff --git a/app/View/GalaxyClusters/view.ctp b/app/View/GalaxyClusters/view.ctp
index bce0ba354..0c2fea045 100755
--- a/app/View/GalaxyClusters/view.ctp
+++ b/app/View/GalaxyClusters/view.ctp
@@ -170,15 +170,57 @@ $md.html(md.render($md.text()));
 <?php
 
 $object_uuid = $cluster['GalaxyCluster']['uuid'];
+
+$notes = $cluster['GalaxyCluster']['Note'] ?? [];
+$opinions = $cluster['GalaxyCluster']['Opinion'] ?? [];
+$relationships_outbound = $cluster['GalaxyCluster']['Relationship'] ?? [];
+$relationships_inbound = $cluster['GalaxyCluster']['RelationshipInbound'] ?? [];
+$notesOpinions = array_merge($notes, $opinions);
+if(!function_exists("countNotes")) {
+    function countNotes($notesOpinions) {
+        $notesTotalCount = count($notesOpinions);
+        $notesCount = 0;
+        $relationsCount = 0;
+        foreach ($notesOpinions as $notesOpinion) {
+            if ($notesOpinion['note_type'] == 2) { // relationship
+                $relationsCount += 1;
+            } else {
+                $notesCount += 1;
+            }
+            if (!empty($notesOpinion['Note'])) {
+                $nestedCounts = countNotes($notesOpinion['Note']);
+                $notesTotalCount += $nestedCounts['total'];
+                $notesCount += $nestedCounts['notesOpinions'];
+                $relationsCount += $nestedCounts['relations'];
+            }
+            if (!empty($notesOpinion['Opinion'])) {
+                $nestedCounts = countNotes($notesOpinion['Opinion']);
+                $notesTotalCount += $nestedCounts['total'];
+                $notesCount += $nestedCounts['notesOpinions'];
+                $relationsCount += $nestedCounts['relations'];
+            }
+        }
+        return ['total' => $notesTotalCount, 'notesOpinions' => $notesCount, 'relations' => $relationsCount];
+    }
+}
+$counts = countNotes($notesOpinions);
+$notesOpinionCount = $counts['notesOpinions'];
+$allCounts = [
+    'notesOpinions' => $counts['notesOpinions'],
+    'relationships_outbound' => count($relationships_outbound),
+    'relationships_inbound' => count($relationships_inbound),
+];
+
 $options = [
     'container_id' => 'analyst_data_thread',
     'object_type' => 'GalaxyCluster',
     'object_uuid' => $object_uuid,
     'shortDist' => $shortDist,
-    'notes' => $cluster['GalaxyCluster']['Note'] ?? [],
-    'opinions' => $cluster['GalaxyCluster']['Opinion'] ?? [],
-    'relationships_outbound' => $cluster['GalaxyCluster']['Relationship'] ?? [],
-    'relationships_inbound' => $cluster['GalaxyCluster']['RelationshipInbound'] ?? [],
+    'notes' => $notes,
+    'opinions' => $opinions,
+    'relationships_outbound' => $relationships_outbound,
+    'relationships_inbound' => $relationships_inbound,
+    'allCounts' => $allCounts,
 ];
 
 echo $this->element('genericElements/Analyst_data/thread', $options);

From 4dd5d369b465013046a27419af96a0e56fe898c0 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Wed, 17 Apr 2024 12:00:53 +0200
Subject: [PATCH 32/45] chg: [attribute search] by uuid updated

- pre-checks if the passed UUID is actually an event UUID before going with the slow query against both tables
---
 app/Controller/AttributesController.php |  1 +
 app/Model/Attribute.php                 | 65 ++++++++++++++++++++-----
 2 files changed, 54 insertions(+), 12 deletions(-)

diff --git a/app/Controller/AttributesController.php b/app/Controller/AttributesController.php
index 9694cfac8..1ef438956 100644
--- a/app/Controller/AttributesController.php
+++ b/app/Controller/AttributesController.php
@@ -1580,6 +1580,7 @@ class AttributesController extends AppController
             }
 
             $this->paginate['conditions'] = $params['conditions'];
+            $this->paginate['ignoreIndexHint'] = 'deleted';
             $attributes = $this->paginate();
             $this->Attribute->attachTagsToAttributes($attributes, ['includeAllTags' => true]);
 
diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php
index 3981d0e21..a5d05dd74 100644
--- a/app/Model/Attribute.php
+++ b/app/Model/Attribute.php
@@ -3163,6 +3163,7 @@ class Attribute extends AppModel
             'includeFullModel' => !empty($filters['includeFullModel']) ? $filters['includeFullModel'] : 0,
             'allow_proposal_blocking' => !empty($filters['allow_proposal_blocking']) ? $filters['allow_proposal_blocking'] : 0
         );
+
         if (!empty($filters['attackGalaxy'])) {
             $params['attackGalaxy'] = $filters['attackGalaxy'];
         }
@@ -3388,20 +3389,60 @@ class Attribute extends AppModel
         if (!empty($params['uuid'])) {
             $params['uuid'] = $this->convert_filters($params['uuid']);
             if (!empty($params['uuid']['OR'])) {
-                $conditions['AND'][] = array(
-                    'OR' => array(
-                        'Event.uuid' => $params['uuid']['OR'],
-                        'Attribute.uuid' => $params['uuid']['OR']
-                    )
-                );
+                if ($options['scope'] == 'Attribute') {
+                    $subQuery = [
+                        'conditions' => ['uuid' => $params['uuid']['OR']],
+                        'fields' => ['id']
+                    ];
+                    $pre_lookup = $this->Event->find('first', [
+                        'conditions' => ['Event.uuid' => $params['uuid']['OR']],
+                        'recursive' => -1,
+                        'fields' => ['Event.id']
+                    ]);
+                    if (empty($pre_lookup)) {
+                        $conditions['AND'][] = array(
+                            'OR' => array(
+                                'Attribute.uuid' => $params['uuid']['OR']
+                            )
+                        );
+                    } else {
+                        $conditions['AND'][] = array(
+                            'OR' => array(
+                                $this->subQueryGenerator($this->Event, $subQuery, 'Attribute.event_id'),
+                                'Attribute.uuid' => $params['uuid']['OR']
+                            )
+                        );
+                    }
+                    
+                } else {
+                    $conditions['AND'][] = array(
+                        'OR' => array(
+                            'Event.uuid' => $params['uuid']['OR'],
+                            'Attribute.uuid' => $params['uuid']['OR']
+                        )
+                    );
+                }
             }
             if (!empty($params['uuid']['NOT'])) {
-                $conditions['AND'][] = array(
-                    'NOT' => array(
-                        'Event.uuid' => $params['uuid']['NOT'],
-                        'Attribute.uuid' =>  $params['uuid']['NOT']
-                    )
-                );
+                if ($options['scope'] == 'Attribute') {
+                    $subQuery = [
+                        'conditions' => ['uuid' => $params['uuid']['OR']],
+                        'fields' => ['id']
+                    ];
+                    $conditions['AND'][] = [
+                        'NOT' => [
+                            $this->subQueryGenerator($this->Event, $subQuery, 'Attribute.event_id'),
+                            'Attribute.uuid' =>  $params['uuid']['NOT']
+                        ]
+                    ];
+                } else {
+                    $conditions['AND'][] = array(
+                        'NOT' => array(
+                            'Event.uuid' => $params['uuid']['NOT'],
+                            'Attribute.uuid' =>  $params['uuid']['NOT']
+                        )
+                    );
+                }
             }
         }
         return $conditions;

From 7ba2b39fe1ab6280a4c76c6629ceeb93d7dcc2dc Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 17 Apr 2024 14:48:01 +0200
Subject: [PATCH 33/45] chg: [workflow:editor] Show 100 entry max in picker

---
 app/webroot/js/workflows-editor/workflows-editor.js | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/app/webroot/js/workflows-editor/workflows-editor.js b/app/webroot/js/workflows-editor/workflows-editor.js
index 928a8cedf..638dd5564 100644
--- a/app/webroot/js/workflows-editor/workflows-editor.js
+++ b/app/webroot/js/workflows-editor/workflows-editor.js
@@ -1836,10 +1836,9 @@ function genPicker(options, forNode = true) {
     var $container = genSelect(options)
     var $select = $container.find('select')
     $select.addClass('start-chosen')
-    if (options.picker_options) {
-        // $select.data('chosen_options', options.picker_options)
-        $select.attr('data-chosen_options', JSON.stringify(options.picker_options))
-    }
+    var pickerOptions = options.picker_options ?? {}
+    pickerOptions['max_shown_results'] = 100
+    $select.attr('data-chosen_options', JSON.stringify(pickerOptions))
     return $container
 }
 

From 4544ef251675e43de8a02d81ba112f9410f8b370 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Wed, 17 Apr 2024 15:08:38 +0200
Subject: [PATCH 34/45] new: [benchmarking suite] added

- collect metrics about the usage of MISP
  - stored in redis
  - per endpoint / user / user-agent collection
  - collection of execution time, php memory use, sql execution time, sql query count
  - the collection happens on a daily basis
- Searchable / filterable interface for the collected data
- Dashboard widget for the collected data
---
 app/Console/Command/AppShell.php              |  14 +-
 app/Console/Command/ServerShell.php           |  12 +-
 app/Controller/AppController.php              |  34 +++-
 app/Controller/BenchmarksController.php       | 123 ++++++++++++
 app/Controller/Component/ACLComponent.php     |   3 +
 app/Lib/Dashboard/BenchmarkTopListWidget.php  | 115 +++++++++++
 app/Lib/Tools/BenchmarkTool.php               | 181 ++++++++++++++++++
 app/Model/Benchmark.php                       |   6 +
 app/Model/Server.php                          |   7 +
 app/View/Benchmarks/index.ctp                 | 110 +++++++++++
 .../dashboard/Widgets/MultiLineChart.ctp      |   3 +-
 .../IndexTable/index_table.ctp                |   2 +-
 .../genericElements/SideMenu/side_menu.ctp    |   7 +
 app/View/Elements/global_menu.ctp             |   5 +
 app/View/Users/view.ctp                       |  11 +-
 15 files changed, 621 insertions(+), 12 deletions(-)
 create mode 100644 app/Controller/BenchmarksController.php
 create mode 100644 app/Lib/Dashboard/BenchmarkTopListWidget.php
 create mode 100644 app/Lib/Tools/BenchmarkTool.php
 create mode 100644 app/Model/Benchmark.php
 create mode 100644 app/View/Benchmarks/index.ctp

diff --git a/app/Console/Command/AppShell.php b/app/Console/Command/AppShell.php
index 6d63a94f4..869ae313d 100644
--- a/app/Console/Command/AppShell.php
+++ b/app/Console/Command/AppShell.php
@@ -18,6 +18,7 @@
 
 App::uses('AppModel', 'Model');
 App::uses('BackgroundJobsTool', 'Tools');
+App::uses('BenchmarkTool', 'Tools');
 
 require_once dirname(__DIR__) . '/../Model/Attribute.php';   // FIXME workaround bug where Vendor/symfony/polyfill-php80/Resources/stubs/Attribute.php is loaded instead
 
@@ -38,7 +39,18 @@ abstract class AppShell extends Shell
     {
         $configLoad = $this->Tasks->load('ConfigLoad');
         $configLoad->execute();
-
+        if (Configure::read('Plugin.Benchmarking_enable')) {
+            $Benchmark = new BenchmarkTool(ClassRegistry::init('User'));
+            $start_time = $Benchmark->startBenchmark();
+            register_shutdown_function(function () use ($start_time, $Benchmark) {
+                $Benchmark->stopBenchmark([
+                    'user' => 0,
+                    'controller' => 'Shell::' . $this->modelClass,
+                    'action' => $this->command,
+                    'start_time' => $start_time
+                ]);
+            });
+        }
         parent::initialize();
     }
 
diff --git a/app/Console/Command/ServerShell.php b/app/Console/Command/ServerShell.php
index ad2d8df72..6b29266f4 100644
--- a/app/Console/Command/ServerShell.php
+++ b/app/Console/Command/ServerShell.php
@@ -125,7 +125,6 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Pull'] . PHP_EOL);
         }
-
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $serverId = $this->args[1];
@@ -166,7 +165,7 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Push'] . PHP_EOL);
         }
-
+        
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $serverId = $this->args[1];
@@ -370,7 +369,7 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Fetch feeds as local data'] . PHP_EOL);
         }
-
+        
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $feedId = $this->args[1];
@@ -426,7 +425,7 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Cache server'] . PHP_EOL);
         }
-
+        
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $scope = $this->args[1];
@@ -489,7 +488,7 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Cache feeds for quick lookups'] . PHP_EOL);
         }
-
+        
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $scope = $this->args[1];
@@ -735,6 +734,7 @@ class ServerShell extends AppShell
 
     public function sendPeriodicSummaryToUsers()
     {
+        
         $periods = $this->__getPeriodsForToday();
         $start_time = time();
         echo __n('Started periodic summary generation for the %s period', 'Started periodic summary generation for periods: %s', count($periods), implode(', ', $periods)) . PHP_EOL;
@@ -800,7 +800,7 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Push Taxii'] . PHP_EOL);
         }
-
+        
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $serverId = $this->args[1];
diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php
index 56486d44f..010af593b 100755
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@@ -40,6 +40,12 @@ class AppController extends Controller
     public $phptoonew = '8.0';
     private $isApiAuthed = false;
 
+    /** @var redis */
+    private $redis = null;
+
+    /** @var benchmark_results */
+    private $benchmark_results = null;
+
     public $baseurl = '';
 
     public $restResponsePayload = null;
@@ -57,9 +63,14 @@ class AppController extends Controller
     /** @var ACLComponent */
     public $ACL;
 
+    /** @var BenchmarkComponent */
+    public $Benchmark;
+
     /** @var RestResponseComponent */
     public $RestResponse;
 
+    public $start_time;
+
     public function __construct($request = null, $response = null)
     {
         parent::__construct($request, $response);
@@ -97,6 +108,12 @@ class AppController extends Controller
 
     public function beforeFilter()
     {
+        $this->User = ClassRegistry::init('User');
+        if (Configure::read('Plugin.Benchmarking_enable')) {
+            App::uses('BenchmarkTool', 'Tools');
+            $this->Benchmark = new BenchmarkTool($this->User);
+            $this->start_time = $this->Benchmark->startBenchmark();
+        }
         $controller = $this->request->params['controller'];
         $action = $this->request->params['action'];
 
@@ -147,8 +164,6 @@ class AppController extends Controller
             Configure::write('Config.language', 'eng');
         }
 
-        $this->User = ClassRegistry::init('User');
-
         if (!empty($this->request->params['named']['disable_background_processing'])) {
             Configure::write('MISP.background_jobs', 0);
         }
@@ -863,6 +878,21 @@ class AppController extends Controller
 
     public function afterFilter()
     {
+        // benchmarking
+        if (Configure::read('Plugin.Benchmarking_enable')) {
+            $this->Benchmark->stopBenchmark([
+                'user' => $this->Auth->user('id'),
+                'controller' => $this->request->params['controller'],
+                'action' => $this->request->params['action'],
+                'start_time' => $this->start_time
+            ]);
+
+            //if ($redis && !$redis->exists('misp:auth_fail_throttling:' . $key)) {
+                //$redis->setex('misp:auth_fail_throttling:' . $key, 3600, 1);
+                //return true;
+            //}
+            
+        }
         if ($this->isApiAuthed && $this->_isRest() && !Configure::read('Security.authkey_keep_session')) {
             $this->Session->destroy();
         }
diff --git a/app/Controller/BenchmarksController.php b/app/Controller/BenchmarksController.php
new file mode 100644
index 000000000..6fe4ce773
--- /dev/null
+++ b/app/Controller/BenchmarksController.php
@@ -0,0 +1,123 @@
+<?php
+App::uses('AppController', 'Controller');
+
+class BenchmarksController extends AppController
+{
+    public $components = array('Session', 'RequestHandler');
+
+    public $paginate = [
+        'limit' => 60,
+        'maxLimit' => 9999,
+
+    ];
+
+    public function beforeFilter()
+    {
+        parent::beforeFilter();
+    }
+
+    public function index()
+    {
+        $this->set('menuData', ['menuList' => 'admin', 'menuItem' => 'index']);
+        $this->loadModel('User');
+        App::uses('BenchmarkTool', 'Tools');
+        $this->Benchmark = new BenchmarkTool($this->User);
+        $passedArgs = $this->passedArgs;
+        $this->paginate['order'] = 'value';
+        $defaults = [
+            'days' => null,
+            'average' => false,
+            'aggregate' => false,
+            'scope' => null,
+            'field' => null,
+            'key' => null,
+            'quickFilter' => null
+        ];
+        $filters = $this->IndexFilter->harvestParameters(array_keys($defaults));
+        foreach ($defaults as $key => $value) {
+            if (!isset($filters[$key])) {
+                $filters[$key] = $defaults[$key];
+            }
+        }
+        $temp = $this->Benchmark->getAllTopLists(
+            $filters['days'] ?? null,
+            $filters['limit'] ?? 100,
+            $filters['average'] ?? null,
+            $filters['aggregate'] ?? null
+        );
+        $settings = $this->Benchmark->getSettings();
+        $units = $this->Benchmark->getUnits();
+        $this->set('settings', $settings);
+        $data = [];
+        $userLookup = [];
+        foreach ($temp as $scope => $t) {
+            if (!empty($filters['scope']) && $filters['scope'] !== 'all' && $scope !== $filters['scope']) {
+                continue;
+            }
+            foreach ($t as $field => $t2) {
+                if (!empty($filters['field']) && $filters['field'] !== 'all' && $field !== $filters['field']) {
+                    continue;
+                }
+                foreach ($t2 as $date => $t3) {
+                    foreach ($t3 as $key => $value) {
+                        if ($scope == 'user') {
+                            if ($key === 'SYSTEM') {
+                                $text = 'SYSTEM';
+                            } else if (isset($userLookup[$key])) {
+                                $text = $userLookup[$key];
+                            } else {
+                                $user = $this->User->find('first', [
+                                    'fields' => ['User.id', 'User.email'],
+                                    'recursive' => -1,
+                                    'conditions' => ['User.id' => $key]
+                                ]);
+                                if (empty($user)) {
+                                    $text = '(' . $key . ') ' . __('Invalid user');
+                                } else {
+                                    $text = '(' . $key . ') ' . $user['User']['email'];
+                                }
+                                $userLookup[$key] = $text;
+                            }
+                        } else {
+                            $text = $key;
+                        }
+                        if (!empty($filters['quickFilter'])) {
+                            $q = strtolower($filters['quickFilter']);
+                            if (
+                                strpos(strtolower($scope), $q) === false &&
+                                strpos(strtolower($field), $q) === false &&
+                                strpos(strtolower($key), $q) === false &&
+                                strpos(strtolower($value), $q) === false &&
+                                strpos(strtolower($date), $q) === false &&
+                                strpos(strtolower($text), $q) === false
+                            ) {
+                                continue;
+                            }
+                        }
+                        if (empty($filters['key']) || $key == $filters['key']) {
+                            $data[] = [
+                                'scope' => $scope,
+                                'field' => $field,
+                                'date' => $date,
+                                'key' => $key,
+                                'text' => $text,
+                                'value' => $value,
+                                'unit' => $units[$field]
+                            ];    
+                        }
+                    }
+                }
+            }
+        }
+        if ($this->_isRest()) {
+            return $this->RestResponse->viewData($data, $this->response->type());
+        }
+        App::uses('CustomPaginationTool', 'Tools');
+        $customPagination = new CustomPaginationTool();
+        $customPagination->truncateAndPaginate($data, $this->params, $this->modelClass, true);
+        $this->set('data', $data);
+        $this->set('passedArgs', json_encode($passedArgs));
+        $this->set('filters', $filters);
+    }
+
+}
diff --git a/app/Controller/Component/ACLComponent.php b/app/Controller/Component/ACLComponent.php
index cfe7147a9..e121f5c5e 100644
--- a/app/Controller/Component/ACLComponent.php
+++ b/app/Controller/Component/ACLComponent.php
@@ -95,6 +95,9 @@ class ACLComponent extends Component
                 'index' => ['perm_auth'],
                 'view' => ['perm_auth'],
             ],
+            'benchmarks' => [
+                'index' => []
+            ],
             'cerebrates' => [
                 'add' => [],
                 'delete' => [],
diff --git a/app/Lib/Dashboard/BenchmarkTopListWidget.php b/app/Lib/Dashboard/BenchmarkTopListWidget.php
new file mode 100644
index 000000000..1fd36ec1b
--- /dev/null
+++ b/app/Lib/Dashboard/BenchmarkTopListWidget.php
@@ -0,0 +1,115 @@
+<?php
+
+
+class BenchmarkTopListWidget
+{
+    public $title = 'Benchmark top list';
+    public $render = 'MultiLineChart';
+    public $width = 3;
+    public $height = 3;
+    public $description = 'A graph showing the top list for a given scope and field in the captured metrics.';
+    public $cacheLifetime = false;
+    public $autoRefreshDelay = 30;
+    public $params = array(
+        'days' => 'Number of days to consider for the graph. There will be a data entry for each day (assuming the benchmarking has been enabled). Defaults to returning all data.',
+        'weeks' => 'Number of weeks to consider for the graph. There will be a data entry for each day (assuming the benchmarking has been enabled). Defaults to returning all data.',
+        'months' => 'Number of months to consider for the graph. There will be a data entry for each day (assuming the benchmarking has been enabled). Defaults to returning all data.',
+        'scope' => 'The scope of the benchmarking refers to what was being tracked. The following scopes are valid: user, endpoint, user_agent',
+        'field' => 'The individual metric to be queried from the benchmark results. Valid values are: time, sql_time, sql_queries, memory, endpoint',
+        'average' => 'If you wish to view the averages per scope/field, set this variable to true. It will divide the result by the number of executions recorded for the scope/field combination for the given day.'
+    );
+    public $Benchmark;
+    public $User;
+
+    public $placeholder =
+        '{
+    "days": "30",
+    "scope": "endpoints",
+    "field": "sql_time"
+}';
+
+    public function handler($user, $options = array())
+    {
+        $this->User = ClassRegistry::init('User');
+        $currentTime = strtotime("now");
+        $endOfDay = strtotime("tomorrow", $currentTime) - 1;
+        if (!empty($options['days'])) {
+            $limit = (int)($options['days']);
+            $delta = 'day';
+        } else if (!empty($options['weeks'])) {
+            $limit = (int)($options['weeks']);
+            $delta = 'week';
+        } else if (!empty($options['months'])) {
+            $limit = (int)($options['months']);
+            $delta = 'month';
+        } else {
+            $limit = 30;
+            $delta = 'day';
+        }
+        $axis_info = [
+            'time' => 'Total time taken (ms)',
+            'sql_time' => 'SQL time taken (ms)',
+            'sql_queries' => 'Queries (#)',
+            'memory' => 'Memory (MB)',
+            'endpoint' => 'Queries to endpoint (#)'
+        ];
+        $y_axis = $axis_info[isset($options['field']) ? $options['field'] : 'time'];
+        $data = ['y-axis' => $y_axis];
+        $data['data'] = array();
+        // Add total users data for all timestamps
+        
+        for ($i = 0; $i < $limit; $i++) {
+            $itemTime = strtotime('- ' . $i . $delta, $endOfDay);
+            $item = array();
+            $date = strftime('%Y-%m-%d', $itemTime);
+            $item = $this->getData($date, $options);
+            if (!empty($item)) {
+                $item['date'] = $date;
+                $data['data'][] = $item;
+            }
+        }
+        $keys = [];
+        foreach ($data['data'] as $day_data) {
+            foreach ($day_data as $key => $temp) {
+                $keys[$key] = 1;
+            }
+        }
+        $keys = array_keys($keys);
+        foreach ($data['data'] as $k => $day_data) {
+            foreach ($keys as $key) {
+                if (!isset($day_data[$key])) {
+                    $data['data'][$k][$key] = 0;
+                }
+            }
+            foreach ($day_data as $key => $temp) {
+                $keys[$key] = 1;
+            }
+        }
+        return $data;
+    }
+
+    private function getData($time, $options)
+    {
+        $dates = [$time];
+        $this->Benchmark = new BenchmarkTool($this->User);
+        $result = $this->Benchmark->getTopList(
+            isset($options['scope']) ? $options['scope'] : 'endpoint',
+            isset($options['field']) ? $options['field'] : 'memory',
+            $dates,
+            isset($options['limit']) ? $options['limit'] : 5,
+            isset($options['average']) ? $options['average'] : false,
+        );
+        if (!empty($result)) {
+            return $result[$time];
+        }
+        return false;
+    }
+
+    public function checkPermissions($user)
+    {
+        if (empty($user['Role']['perm_site_admin'])) {
+            return false;
+        }
+        return true;
+    }
+}
diff --git a/app/Lib/Tools/BenchmarkTool.php b/app/Lib/Tools/BenchmarkTool.php
new file mode 100644
index 000000000..8f741b854
--- /dev/null
+++ b/app/Lib/Tools/BenchmarkTool.php
@@ -0,0 +1,181 @@
+<?php
+
+/**
+ * Get filter parameters from index searches
+ */
+
+class BenchmarkTool
+{
+    /** @var Model */
+    public $Model;
+    
+    /** @var redis */
+    public $redis;
+
+    /** @var retention */
+    private $retention = 0;
+
+    /** @var start_time */
+    public $start_timexxx;
+
+    const BENCHMARK_SCOPES = ['user', 'endpoint', 'user_agent'];
+    const BENCHMARK_FIELDS = ['time', 'sql_time', 'sql_queries', 'memory'];
+    const BENCHMARK_UNITS = [
+        'time' => 's',
+        'sql_time' => 'ms',
+        'sql_queries' => '',
+        'memory' => 'MB'
+    ];
+
+    public $namespace = 'misp:benchmark:';
+
+    function __construct(Model $model) {
+        $this->Model = $model;
+    }
+
+    public function getSettings()
+    {
+        return [
+            'scope' => self::BENCHMARK_SCOPES,
+            'field' => self::BENCHMARK_FIELDS,
+            'average' => [0, 1],
+            'aggregate' => [0, 1]
+        ];
+    }
+
+    public function getUnits()
+    {
+        return self::BENCHMARK_UNITS;
+    }
+
+    public function startBenchmark()
+    {
+        $start_time = microtime(true);
+        $this->redis = $this->Model->setupRedis();
+        $this->retention = Configure::check('Plugin.benchmark_retention') ? Configure::read('Plugin.benchmark_retention') : 0;
+        return $start_time;
+    }
+
+    public function stopBenchmark(array $options)
+    {
+        $start_time = $options['start_time'];
+        if (!empty($options['user'])) {
+            $sql = $this->Model->getDataSource()->getLog(false, false);
+            $benchmarkData = [
+                'user' => $options['user'],
+                'endpoint' => $options['controller'] . '/' . $options['action'],
+                'user_agent' => $_SERVER['HTTP_USER_AGENT'],
+                'sql_queries' => $sql['count'],
+                'sql_time' => $sql['time'],
+                'time' => (microtime(true) - $start_time),
+                'memory' => (int)(memory_get_peak_usage(true) / 1024 / 1024),
+                //'date' => date('Y-m-d', strtotime("-3 days"))
+                'date' => date('Y-m-d')
+            ];
+            $this->pushBenchmarkDataToRedis($benchmarkData);
+        } else {
+            $sql = $this->Model->getDataSource()->getLog(false, false);
+            $benchmarkData = [
+                'user' => 'SYSTEM',
+                'endpoint' => $options['controller'] . '/' . $options['action'],
+                'user_agent' => 'CLI',
+                'sql_queries' => $sql['count'],
+                'sql_time' => $sql['time'],
+                'time' => (microtime(true) - $start_time),
+                'memory' => (int)(memory_get_peak_usage(true) / 1024 / 1024),
+                //'date' => date('Y-m-d', strtotime("-3 days"))
+                'date' => date('Y-m-d')
+            ];
+            $this->pushBenchmarkDataToRedis($benchmarkData);
+        }
+    }
+
+    private function pushBenchmarkDataToRedis($benchmarkData)
+    {
+        $this->redis = $this->Model->setupRedis();
+        $this->redis->pipeline();
+        $this->redis->sAdd(
+            $this->namespace . 'days',
+            $benchmarkData['date']
+        );
+        foreach (self::BENCHMARK_SCOPES as $scope) {
+            $this->redis->sAdd(
+                $this->namespace . $scope . ':list',
+                $benchmarkData[$scope]
+            );
+            $this->redis->zIncrBy(
+                $this->namespace . $scope . ':count:' . $benchmarkData['date'],
+                1,
+                $benchmarkData[$scope]
+            );
+            foreach (self::BENCHMARK_FIELDS as $field) {
+                $this->redis->zIncrBy(
+                    $this->namespace . $scope . ':' . $field . ':' . $benchmarkData['date'],
+                    $benchmarkData[$field],
+                    $benchmarkData[$scope]
+                );
+            }
+            $this->redis->zIncrBy(
+                $this->namespace . $scope . ':endpoint:' . $benchmarkData['date'] . ':' . $benchmarkData['user'],
+                1,
+                $benchmarkData['endpoint']
+            );
+        }
+        $this->redis->exec();
+    }
+
+    public function getTopList(string $scope, string $field, array $days = [], $limit = 10, $average = false, $aggregate = false)
+    {
+        if (empty($this->redis)) {
+            $this->redis = $this->Model->setupRedis();
+        }
+        $results = [];
+        if (is_string($days)) {
+            $days = [$days];
+        }
+        foreach ($days as $day) {
+            $temp = $this->redis->zrevrange($this->namespace . $scope . ':' . $field . ':' . $day, 0, $limit, true);
+            foreach ($temp as $k => $v) {
+                if ($average) {
+                    $divisor = $this->redis->zscore($this->namespace . $scope . ':count:' . $day, $k);
+                    if ($aggregate) {
+                        $results['aggregate'][$k] = empty($results['aggregate'][$k]) ? ($v / $divisor) : ($results['aggregate'][$k] + ($v / $divisor));
+                    } else {
+                        $results[$day][$k] = (int)($v / $divisor);
+                    }
+                } else {
+                    if ($aggregate) {
+                        $results['aggregate'][$k] = empty($results['aggregate'][$k]) ? $v : ($results['aggregate'][$k] + $v);
+                    } else {
+                        $results[$day][$k] = $v;
+                    }
+                }
+            }
+        }
+        if ($aggregate && $average) {
+            $count_days = count($days);
+            foreach ($results['aggregate'] as $k => $result) {
+                $results['aggregate'][$k] = (int)($result / $count_days);
+            }
+        }
+        return $results;
+    }
+
+    public function getAllTopLists(array $days = null, $limit = 10, $average = false, $aggregate = false, $scope_filter = [])
+    {
+        if (empty($this->redis)) {
+            $this->redis = $this->Model->setupRedis();
+        }
+        if ($days === null) {
+            $days = $this->redis->smembers($this->namespace . 'days');
+        }
+        foreach (self::BENCHMARK_SCOPES as $scope) {
+            if (empty($scope_filter) || in_array($scope, $scope_filter)) {
+                foreach (self::BENCHMARK_FIELDS as $field) {
+                    $results[$scope][$field] = $this->getTopList($scope, $field, $days, $limit, $average, $aggregate);
+                }
+            }
+        }
+        return $results;
+    }
+}
diff --git a/app/Model/Benchmark.php b/app/Model/Benchmark.php
new file mode 100644
index 000000000..c350c6398
--- /dev/null
+++ b/app/Model/Benchmark.php
@@ -0,0 +1,6 @@
+<?php
+App::uses('AppModel', 'Model');
+
+class Benchmark extends AppModel
+{
+}
diff --git a/app/Model/Server.php b/app/Model/Server.php
index 933fadf31..8354e2683 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -7536,6 +7536,13 @@ class Server extends AppModel
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
+                'Benchmarking_enable' => [
+                    'level' => 2,
+                    'description' => __('Enable the benchmarking functionalities to capture information about execution times, SQL query loads and more per user and per endpoint.'),
+                    'value' => false,
+                    'test' => 'testBool',
+                    'type' => 'boolean'
+                ],
                 'Enrichment_services_enable' => array(
                     'level' => 0,
                     'description' => __('Enable/disable the enrichment services'),
diff --git a/app/View/Benchmarks/index.ctp b/app/View/Benchmarks/index.ctp
new file mode 100644
index 000000000..df2c9e282
--- /dev/null
+++ b/app/View/Benchmarks/index.ctp
@@ -0,0 +1,110 @@
+<?php
+    $passedArgsArray = json_decode($passedArgs, true);
+    $fields = [
+        [
+            'name' => __('Date'),
+            'sort' => 'date',
+            'data_path' => 'date'
+        ],
+        [
+            'name' => __('scope'),
+            'sort' => 'scope',
+            'data_path' => 'scope'
+        ],
+        [
+            'name' => __('Key'),
+            'sort' => 'key',
+            'data_path' => 'text'
+        ],
+        [
+            'name' => __('field'),
+            'sort' => 'field',
+            'data_path' => 'field'
+        ],
+        [
+            'name' => __('Value'),
+            'element' => 'custom',
+            'function' => function($row) {
+                return empty($row['unit']) ? h($row['value']) : h($row['value'] . ' ' . $row['unit']);
+            },
+            'sort' => 'value'
+        ]
+    ];
+    $quick_filters = [];
+    foreach ($settings as $key => $setting_data) {
+        $temp = $filters;
+        $url = $baseurl . '/benchmarks/index';
+        foreach ($temp as $s => $v) {
+            if ($v && $s != $key) {
+                if (is_array($v)) {
+                    foreach ($v as $multi_v) {
+                        $url .= '/' . $s . '[]:' . $multi_v;
+                    }
+                } else {
+                    $url .= '/' . $s . ':' . $v;
+                }
+                
+            }
+        }
+        if ($key != 'average' && $key != 'aggregate') {
+            $quick_filters[$key]['all'] = [
+                'url' => h($url),
+                'text' => __('All'),
+                'active' => !$filters[$key],
+                'style' => 'display:inline;'
+            ];
+        }
+        foreach ($setting_data as $setting_element) {
+            $text = $setting_element;
+            if ($key == 'average') {
+                $text = $setting_element ? 'average / request' : 'total';
+            }
+            if ($key == 'aggregate') {
+                $text = $setting_element ? 'aggregate' : 'daily';
+            }
+            $quick_filters[$key][] = [
+                'url' => h($url . '/' . $key . ':' . $setting_element),
+                'text' => $text,
+                'active' => $filters[$key] == $setting_element,
+                'style' => 'display:inline;'
+            ];
+        }
+    }
+    echo $this->element('genericElements/IndexTable/scaffold', [
+        'scaffold_data' => [
+            'passedArgsArray' => $passedArgsArray,
+            'data' => [
+                'persistUrlParams' => array_keys($settings),
+                'data' => $data,
+                'top_bar' => [
+                    'pull' => 'right',
+                    'children' => [
+                        [
+                            'children' => $quick_filters['scope']
+                        ],
+                        [
+                            'children' => $quick_filters['field']
+                        ],
+                        [
+                            'children' => $quick_filters['average']
+                        ],
+                        [
+                            'children' => $quick_filters['aggregate']
+                        ],
+                        [
+                            'type' => 'search',
+                            'button' => __('Filter'),
+                            'placeholder' => __('Enter value to search'),
+                            'data' => '',
+                            'searchKey' => 'quickFilter'
+                        ]
+                    ]
+                ],
+                'fields' => $fields,
+                'title' => empty($ajax) ? __('Benchmark results') : false,
+                'description' => empty($ajax) ? __('Results of the collected benchmarks. You can filter it further by passing the limit, scope, field parameters.') : false,
+            ]
+        ]
+    ]);
+
+?>
diff --git a/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp b/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp
index a6057be90..10115e48d 100644
--- a/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp
+++ b/app/View/Elements/dashboard/Widgets/MultiLineChart.ctp
@@ -6,6 +6,7 @@
             h($data['formula'])
         );
     }
+    $y_axis = $data['y-axis'] ?? 'Count';
 ?>
 <div id="chartContainer-<?= $seed ?>" style="flex-grow: 1; position:relative;"></div>
 <script>
@@ -50,7 +51,7 @@ function init<?= $seed ?>() { // variables and functions have their own scope (n
         show_legend: true,
         style: {
             xlabel: "Date",
-            ylabel: "Count",
+            ylabel: "<?= h($y_axis) ?>",
             hideXAxis: false,
             hideYAxis: false,
         },
diff --git a/app/View/Elements/genericElements/IndexTable/index_table.ctp b/app/View/Elements/genericElements/IndexTable/index_table.ctp
index 004371ebc..7ed01ef7c 100644
--- a/app/View/Elements/genericElements/IndexTable/index_table.ctp
+++ b/app/View/Elements/genericElements/IndexTable/index_table.ctp
@@ -35,7 +35,7 @@
     if (!empty($data['persistUrlParams'])) {
         foreach ($data['persistUrlParams'] as $persistedParam) {
             if (!empty($passedArgsArray[$persistedParam])) {
-                $data['paginatorOptions']['url'][] = $passedArgsArray[$persistedParam];
+                $data['paginatorOptions']['url'][$persistedParam] = $passedArgsArray[$persistedParam];
             }
         }
     }
diff --git a/app/View/Elements/genericElements/SideMenu/side_menu.ctp b/app/View/Elements/genericElements/SideMenu/side_menu.ctp
index b39ee9a95..1032944d2 100644
--- a/app/View/Elements/genericElements/SideMenu/side_menu.ctp
+++ b/app/View/Elements/genericElements/SideMenu/side_menu.ctp
@@ -1100,6 +1100,13 @@ $divider = '<li class="divider"></li>';
                             'url' => $baseurl . '/servers/updateProgress',
                             'text' => __('Update Progress')
                         ));
+                        if (Configure::read('Plugin.Benchmarking_enable')) {
+                            echo $divider;
+                            echo $this->element('/genericElements/SideMenu/side_menu_link', array(
+                                'url' => $baseurl . '/benchmarks/index',
+                                'text' => __('Benchmarks')
+                            ));
+                        }
                         echo $divider;
                         if (Configure::read('MISP.background_jobs')) {
                             echo $this->element('/genericElements/SideMenu/side_menu_link', array(
diff --git a/app/View/Elements/global_menu.ctp b/app/View/Elements/global_menu.ctp
index 0e7f67c6c..6f22751fb 100755
--- a/app/View/Elements/global_menu.ctp
+++ b/app/View/Elements/global_menu.ctp
@@ -408,6 +408,11 @@ if (!empty($me)) {
                     'url' => $baseurl . '/servers/serverSettings',
                     'requirement' => $isSiteAdmin
                 ),
+                [
+                    'text' => __('Benchmarking'),
+                    'url' => $baseurl . '/benchmarks/index',
+                    'requirement' => $isSiteAdmin && Configure::read('Plugin.Benchmarking_enable')
+                ],
                 array(
                     'type' => 'separator',
                     'requirement' => $isSiteAdmin
diff --git a/app/View/Users/view.ctp b/app/View/Users/view.ctp
index e46c81b15..7d1a733c3 100755
--- a/app/View/Users/view.ctp
+++ b/app/View/Users/view.ctp
@@ -177,7 +177,7 @@ if ($isAdmin && $isTotp) {
         'js' => array('vis', 'jquery-ui.min', 'network-distribution-graph')
     ));
     echo sprintf(
-        '<div class="users view"><div class="row-fluid"><div class="span8" style="margin:0px;">%s</div></div>%s%s%s<div style="margin-top:20px;">%s%s</div></div>',
+        '<div class="users view"><div class="row-fluid"><div class="span8" style="margin:0px;">%s</div></div>%s%s%s<div style="margin-top:20px;">%s%s%s</div></div>',
         sprintf(
             '<h2>%s</h2>%s',
             __('User %s', h($user['User']['email'])),
@@ -210,6 +210,15 @@ if ($isAdmin && $isTotp) {
             __('Review user logins')
         ),
         $me['Role']['perm_auth'] ? $this->element('/genericElements/accordion', array('title' => __('Auth keys'), 'url' => '/auth_keys/index/' . h($user['User']['id']))) : '',
+        $me['Role']['perm_site_admin'] ?
+            $this->element(
+                '/genericElements/accordion',
+                [
+                    'title' => __('Benchmarks'),
+                    'url' => '/benchmarks/index/scope:user/average:1/aggregate:1/key:' . h($user['User']['id'])
+                ]
+            ) :
+            '',
         $this->element('/genericElements/accordion', array('title' => 'Events', 'url' => '/events/index/searchemail:' . urlencode(h($user['User']['email']))))
     );
     $current_menu = [

From a2ea6ae0c09aa058a7a8de4446e2abde746f7c91 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 17 Apr 2024 15:15:49 +0200
Subject: [PATCH 35/45] fix: [feed] Added tag_collection_id as column

---
 app/Model/AppModel.php | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php
index f91cdadc7..cb8232d2e 100644
--- a/app/Model/AppModel.php
+++ b/app/Model/AppModel.php
@@ -2177,7 +2177,7 @@ class AppModel extends Model
                   ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;';
                 break;
             case 125:
-                $sqlArray[] = "ALTER TABLE `feeds` MODIFY `tag_id` VARCHAR(40) NOT NULL DEFAULT '0';";
+                $sqlArray[] = "ALTER TABLE `feeds` ADD COLUMN `tag_collection_id` INT(11) NOT NULL DEFAULT 0;";
                 break;
             case 'fixNonEmptySharingGroupID':
                 $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';

From 00991bda271e8dfe7f2ae77a9b3b378118ceb4b1 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Wed, 17 Apr 2024 15:59:10 +0200
Subject: [PATCH 36/45] chg: [feed] Added support of tag_collection_id when
 dealing with feeds

---
 app/Controller/FeedsController.php | 49 +++++++++++-------------------
 app/Model/Feed.php                 | 12 ++++----
 app/View/Feeds/add.ctp             |  8 +++++
 3 files changed, 32 insertions(+), 37 deletions(-)

diff --git a/app/Controller/FeedsController.php b/app/Controller/FeedsController.php
index d634bf65c..954bf95f6 100644
--- a/app/Controller/FeedsController.php
+++ b/app/Controller/FeedsController.php
@@ -109,17 +109,14 @@ class FeedsController extends AppController
                 }
 
                 foreach ($feeds as &$feed) {
-                    if (!empty($feed['Feed']['tag_id'])) {
-                        if (substr($feed['Feed']['tag_id'], 0, 11) == 'collection_') {
-                            $tagCollectionID = intval(substr($feed['Feed']['tag_id'], 11));
-                            $tagCollection = $this->TagCollection->fetchTagCollection($loggedUser, [
-                                'conditions' => [
-                                    'TagCollection.id' => $tagCollectionID,
-                                ]
-                            ]);
-                            if (!empty($tagCollection)) {
-                                $feed['TagCollection'] = $tagCollection;
-                            }
+                    if (!empty($feed['Feed']['tag_collection_id'])) {
+                        $tagCollection = $this->TagCollection->fetchTagCollection($loggedUser, [
+                            'conditions' => [
+                                'TagCollection.id' => $feed['Feed']['tag_collection_id'],
+                            ]
+                        ]);
+                        if (!empty($tagCollection)) {
+                            $feed['TagCollection'] = $tagCollection;
                         }
                     }
                 }
@@ -310,21 +307,12 @@ class FeedsController extends AppController
         if (empty(Configure::read('Security.disable_local_feed_access'))) {
             $inputSources['local'] = 'Local';
         }
-        $tags = $this->Event->EventTag->Tag->find('all', [
-            'recursive' => -1,
-            'fields' => ['Tag.name', 'Tag.id'],
-            'order' => ['lower(Tag.name) asc']
-        ]);
-        $tags = Hash::combine($tags, '{n}.Tag.id', '{n}.Tag.name');
+        $tags = $this->Event->EventTag->Tag->find('list', array('fields' => array('Tag.name'), 'order' => array('lower(Tag.name) asc')));
+        $tags[0] = 'None';
+        $this->loadModel('TagCollection');
         $tagCollections = $this->TagCollection->fetchTagCollection($this->Auth->user());
         $tagCollections = Hash::combine($tagCollections, '{n}.TagCollection.id', '{n}.TagCollection.name');
-        foreach ($tagCollections as $id => $name) {
-            $newID = "collection_{$id}";
-            $tagCollections[$newID] = sprintf('%s :: %s', __('Tag Collection'), $name);
-            unset($tagCollections[$id]);
-        }
-        $tags = array_merge($tags, $tagCollections);
-        $tags[0] = 'None';
+        $tagCollections[0] = 'None';
 
         $this->loadModel('Server');
         $allTypes = $this->Server->getAllTypes();
@@ -335,6 +323,7 @@ class FeedsController extends AppController
                 'order' => 'LOWER(name)'
             )),
             'tags' => $tags,
+            'tag_collections' => $tagCollections,
             'feedTypes' => $this->Feed->getFeedTypesOptions(),
             'sharingGroups' => $sharingGroups,
             'distributionLevels' => $distributionLevels,
@@ -371,6 +360,7 @@ class FeedsController extends AppController
                 'distribution',
                 'sharing_group_id',
                 'tag_id',
+                'tag_collection_id',
                 'event_id',
                 'publish',
                 'delta_merge',
@@ -478,16 +468,12 @@ class FeedsController extends AppController
             'fields' => ['Tag.name', 'Tag.id'],
             'order' => ['lower(Tag.name) asc']
         ]);
-        $tags = Hash::combine($tags, '{n}.Tag.id', '{n}.Tag.name');
+        $tags = $this->Event->EventTag->Tag->find('list', array('fields' => array('Tag.name'), 'order' => array('lower(Tag.name) asc')));
+        $tags[0] = 'None';
         $this->loadModel('TagCollection');
         $tagCollections = $this->TagCollection->fetchTagCollection($this->Auth->user());
         $tagCollections = Hash::combine($tagCollections, '{n}.TagCollection.id', '{n}.TagCollection.name');
-        foreach ($tagCollections as $id => $name) {
-            $newID = "collection_{$id}";
-            $tags[$newID] = sprintf('%s :: %s', __('Tag Collection'), $name);
-        }
-        unset($tagCollections);
-        $tags[0] = 'None';
+        $tagCollections[0] = 'None';
 
         $this->loadModel('Server');
         $allTypes = $this->Server->getAllTypes();
@@ -501,6 +487,7 @@ class FeedsController extends AppController
                 'order' => 'LOWER(name)'
             )),
             'tags' => $tags,
+            'tag_collections' => $tagCollections,
             'feedTypes' => $this->Feed->getFeedTypesOptions(),
             'sharingGroups' => $sharingGroups,
             'distributionLevels' => $distributionLevels,
diff --git a/app/Model/Feed.php b/app/Model/Feed.php
index 862985686..e17fa4b45 100644
--- a/app/Model/Feed.php
+++ b/app/Model/Feed.php
@@ -1032,7 +1032,7 @@ class Feed extends AppModel
                 }
             }
         }
-        if ($feed['Feed']['tag_id']) {
+        if ($feed['Feed']['tag_id'] || $feed['Feed']['tag_collection_id']) {
             if (empty($feed['Tag']['name'])) {
                 $feed_tag = $this->Tag->find('first', [
                     'conditions' => [
@@ -1049,9 +1049,9 @@ class Feed extends AppModel
                 $event['Event']['Tag'] = array();
             }
 
-            if (substr($feed['Feed']['tag_id'], 0, 11) == 'collection_') { // Tag is actually a tag collection
+            if (!empty($feed['Feed']['tag_collection_id'])) {
                 $this->TagCollection = ClassRegistry::init('TagCollection');
-                $tagCollectionID = intval(substr($feed['Feed']['tag_id'], 11));
+                $tagCollectionID = $feed['Feed']['tag_collection_id'];
                 $tagCollection = $this->TagCollection->find('first', [
                     'recursive' => -1,
                     'conditions' => [
@@ -1400,10 +1400,10 @@ class Feed extends AppModel
         if ($feed['Feed']['publish']) {
             $this->Event->publishRouter($event['Event']['id'], null, $user);
         }
-        if ($feed['Feed']['tag_id']) {
-            if (substr($feed['Feed']['tag_id'], 0, 11) == 'collection_') { // Tag is actually a tag collection
+        if ($feed['Feed']['tag_id'] || $feed['Feed']['tag_collection_id']) {
+            if (!empty($feed['Feed']['tag_collection_id'])) {
                 $this->TagCollection = ClassRegistry::init('TagCollection');
-                $tagCollectionID = intval(substr($feed['Feed']['tag_id'], 11));
+                $tagCollectionID = $feed['Feed']['tag_collection_id'];
                 $tagCollection = $this->TagCollection->find('first', [
                     'recursive' => -1,
                     'conditions' => [
diff --git a/app/View/Feeds/add.ctp b/app/View/Feeds/add.ctp
index f1032c8c0..6000130d7 100755
--- a/app/View/Feeds/add.ctp
+++ b/app/View/Feeds/add.ctp
@@ -166,6 +166,14 @@ echo $this->element('genericElements/Form/genericForm', [
                 'type' => 'dropdown',
                 'searchable' => 1
             ],
+            [
+                'field' => 'tag_collection_id',
+                'label' => __('Default Tag Collection'),
+                'options' => $dropdownData['tag_collections'],
+                'selected' => isset($entity['Feed']['tag_collection_id']) ? $entity['Feed']['tag_collection_id'] : '0',
+                'type' => 'dropdown',
+                'searchable' => 1
+            ],
             [
                 'field' => 'rules',
                 'label' => __('Filter rules'),

From 7f3db27667363cf64297be84ddb53548d908f0e4 Mon Sep 17 00:00:00 2001
From: Sami Mokaddem <mokaddem.sami@gmail.com>
Date: Thu, 18 Apr 2024 09:25:08 +0200
Subject: [PATCH 37/45] chg: [db_schema] Bumped version

---
 db_schema.json | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/db_schema.json b/db_schema.json
index e720ef18c..e2c975312 100644
--- a/db_schema.json
+++ b/db_schema.json
@@ -3206,6 +3206,17 @@
                 "column_type": "int(11)",
                 "column_default": "0",
                 "extra": ""
+            },
+            {
+                "column_name": "tag_collection_id",
+                "is_nullable": "NO",
+                "data_type": "int",
+                "character_maximum_length": null,
+                "numeric_precision": "10",
+                "collation_name": null,
+                "column_type": "int(11)",
+                "column_default": "0",
+                "extra": ""
             }
         ],
         "fuzzy_correlate_ssdeep": [
@@ -10603,5 +10614,5 @@
             "uuid": false
         }
     },
-    "db_version": "124"
+    "db_version": "125"
 }
\ No newline at end of file

From be9ad95905fdc0051292f099fc03fdbea044b700 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Thu, 18 Apr 2024 12:46:11 +0200
Subject: [PATCH 38/45] chg: [syslog] output slightly changed

- always have a consistent number of fields conveyed, include delimited ( -- ) fields even if no data is passed to a field
- Avoid linebreaks in content
---
 app/Model/Log.php | 17 ++++++++---------
 1 file changed, 8 insertions(+), 9 deletions(-)

diff --git a/app/Model/Log.php b/app/Model/Log.php
index 2155a2e99..6c810db1d 100644
--- a/app/Model/Log.php
+++ b/app/Model/Log.php
@@ -431,15 +431,14 @@ class Log extends AppModel
                 }
             }
 
-            $entry = $data['Log']['action'];
-            if (!empty($data['Log']['title'])) {
-                $entry .= " -- {$data['Log']['title']}";
-            }
-            if (!empty($data['Log']['description'])) {
-                $entry .= " -- {$data['Log']['description']}";
-            } else if (!empty($data['Log']['change'])) {
-                $entry .= " -- " . JsonTool::encode($data['Log']['change']);
-            }
+            $entry = sprintf(
+                '%s -- %s -- %s',
+                $data['Log']['action'],
+                empty($data['Log']['title']) ? '' : $formatted_title = preg_replace('/\s+/', " ", $data['Log']['title']),
+                empty($data['Log']['description']) ? 
+                    (empty($data['Log']['change']) ? '' : preg_replace('/\s+/', " ", $data['Log']['change'])) :
+                    preg_replace('/\s+/', " ", $data['Log']['description'])
+            );
             $this->syslog->write($action, $entry);
         }
     }

From 35fe93fc0267b8d903655789fe9ebba23354d1d1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Thu, 18 Apr 2024 13:03:10 +0200
Subject: [PATCH 39/45] chg: Bump PyMISP

---
 PyMISP                           | 2 +-
 app/Controller/AppController.php | 2 +-
 requirements.txt                 | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/PyMISP b/PyMISP
index 60aa6b9a0..0ed35fbc1 160000
--- a/PyMISP
+++ b/PyMISP
@@ -1 +1 @@
-Subproject commit 60aa6b9a0fce69507776429fcaaf3e0e3962a36c
+Subproject commit 0ed35fbc19274609583e2086a8b7f87a0b97c165
diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php
index 38e8df162..97660006d 100755
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@@ -34,7 +34,7 @@ class AppController extends Controller
     public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
 
     private $__queryVersion = '161';
-    public $pyMispVersion = '2.4.188';
+    public $pyMispVersion = '2.4.190';
     public $phpmin = '7.2';
     public $phprec = '7.4';
     public $phptoonew = '8.0';
diff --git a/requirements.txt b/requirements.txt
index b1f58a418..8127efbd3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,7 +6,7 @@ misp-lib-stix2>=3.0.1.1
 mixbox>=1.0.5
 plyara>=2.1.1
 pydeep2>=0.5.1
-pymisp==2.4.188
+pymisp==2.4.190
 python-magic>=0.4.27
 pyzmq>=25.1.1
 redis>=5.0.1

From c78641ef8567d4d99bf599c439f1f5afa0dd8d43 Mon Sep 17 00:00:00 2001
From: iglocska <andras.iklody@gmail.com>
Date: Thu, 18 Apr 2024 14:48:16 +0200
Subject: [PATCH 40/45] chg: [version] bump

---
 VERSION.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION.json b/VERSION.json
index e7997ba86..01970df01 100644
--- a/VERSION.json
+++ b/VERSION.json
@@ -1 +1 @@
-{"major":2, "minor":4, "hotfix":189}
+{"major":2, "minor":4, "hotfix":190}

From 4f6e4360e4007c4454282ac4d7f0022c94fdd06e Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Thu, 18 Apr 2024 14:52:13 +0200
Subject: [PATCH 41/45] chg: [misp-objects] updated

---
 app/files/misp-objects | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/misp-objects b/app/files/misp-objects
index dc52c1084..96492b9c9 160000
--- a/app/files/misp-objects
+++ b/app/files/misp-objects
@@ -1 +1 @@
-Subproject commit dc52c10844cbed9e2f39f0429665b4f9b1caef3e
+Subproject commit 96492b9c932a4b307216550abeadddc727e17cec

From 1819cece53dea354fc45285bdda16a09c670e292 Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Thu, 18 Apr 2024 14:52:35 +0200
Subject: [PATCH 42/45] chg: [misp-galaxy] updated to the latest version

---
 app/files/misp-galaxy | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy
index c953d8ee5..8e8c3fa93 160000
--- a/app/files/misp-galaxy
+++ b/app/files/misp-galaxy
@@ -1 +1 @@
-Subproject commit c953d8ee5d1ec43bf8a418f957ad6f920fe1c0e8
+Subproject commit 8e8c3fa93de383c86b27c00830bfb6314d2c8087

From 89fd016e46dd23eefc366cf54eb6bd620928775e Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Thu, 18 Apr 2024 14:53:06 +0200
Subject: [PATCH 43/45] chg: [taxonomy] updated to the latest version

---
 app/files/taxonomies | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/taxonomies b/app/files/taxonomies
index 55e0f57d5..272192fe3 160000
--- a/app/files/taxonomies
+++ b/app/files/taxonomies
@@ -1 +1 @@
-Subproject commit 55e0f57d5d2953417e4084fe311a80bae99270f5
+Subproject commit 272192fe3223a2a5f382b1f24540e49cf6a945ff

From 2bb12095d54c4122efaa3970df3692d56b5ac824 Mon Sep 17 00:00:00 2001
From: Alexandre Dulaunoy <a@foo.be>
Date: Thu, 18 Apr 2024 14:53:52 +0200
Subject: [PATCH 44/45] chg: [warninglists] updated to the latest version

---
 app/files/warninglists | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/app/files/warninglists b/app/files/warninglists
index 7c69f4f98..c282baa10 160000
--- a/app/files/warninglists
+++ b/app/files/warninglists
@@ -1 +1 @@
-Subproject commit 7c69f4f987ee753901e952071556d5ad0fafd115
+Subproject commit c282baa102adf07c10dec8ca94488a1ffd933e32

From 9f3e6ce20ef9424bed41edab4ea4f27cdbdc8f9f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= <raphael@vinot.info>
Date: Thu, 18 Apr 2024 14:57:57 +0200
Subject: [PATCH 45/45] chg: [PyMISP] Bump

---
 PyMISP | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/PyMISP b/PyMISP
index 0ed35fbc1..8b4f98ac4 160000
--- a/PyMISP
+++ b/PyMISP
@@ -1 +1 @@
-Subproject commit 0ed35fbc19274609583e2086a8b7f87a0b97c165
+Subproject commit 8b4f98ac4c2e6c8cc1dba064f937dac816b67d0f