From f3f50c14fd1663d9ae5fae44318c129f3cd5d18f Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 17 Oct 2021 14:20:32 +0200
Subject: [PATCH 01/21] chg: [internal] Remove unused method

---
 app/Controller/EventsController.php | 5 +----
 app/Model/AppModel.php              | 5 -----
 2 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php
index 7d6f43ca8..b0ad6e4a8 100644
--- a/app/Controller/EventsController.php
+++ b/app/Controller/EventsController.php
@@ -3202,10 +3202,7 @@ class EventsController extends AppController
             $iocData = FileAccessTool::readFromFile($this->data['Event']['submittedioc']['tmp_name'], $this->data['Event']['submittedioc']['size']);
 
         // write
-        $attachments_dir = Configure::read('MISP.attachments_dir');
-            if (empty($attachments_dir)) {
-            $attachments_dir = $this->Event->getDefaultAttachments_dir();
-        }
+        $attachments_dir = Configure::read('MISP.attachments_dir') ?: (APP . 'files');
         $rootDir = $attachments_dir . DS . $id . DS;
             App::uses('Folder', 'Utility');
             $dir = new Folder($rootDir . 'ioc', true);
diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php
index bf6963c24..d77aadbd4 100644
--- a/app/Model/AppModel.php
+++ b/app/Model/AppModel.php
@@ -2923,11 +2923,6 @@ class AppModel extends Model
         return $val / (1024 * 1024);
     }
 
-    public function getDefaultAttachments_dir()
-    {
-        return APP . 'files';
-    }
-
     private function __bumpReferences()
     {
         $this->Event = ClassRegistry::init('Event');

From 3b69b528cc4e8d6c65579b3fbf859722282f03c3 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 17 Oct 2021 14:20:59 +0200
Subject: [PATCH 02/21] chg: [internal] Fix validation for UserSetting value

---
 app/Model/EventGraph.php  | 12 +-----------
 app/Model/UserSetting.php |  6 +-----
 2 files changed, 2 insertions(+), 16 deletions(-)

diff --git a/app/Model/EventGraph.php b/app/Model/EventGraph.php
index 1e6a203ac..b2861c7da 100644
--- a/app/Model/EventGraph.php
+++ b/app/Model/EventGraph.php
@@ -30,7 +30,7 @@ class EventGraph extends AppModel
 
     public $validate = array(
         'network_json' => array(
-            'rule' => array('isValidJson'),
+            'rule' => 'valueIsJson',
             'message' => 'The provided eventGraph is not a valid json format',
             'required' => true,
         ),
@@ -44,16 +44,6 @@ class EventGraph extends AppModel
         return true;
     }
 
-    public function isValidJson($fields)
-    {
-        $text = $fields['network_json'];
-        $check = json_decode($text);
-        if ($check === null) {
-            return false;
-        }
-        return true;
-    }
-
     public function getPictureData($eventGraph)
     {
         $b64 = str_replace('data:image/png;base64,', '', $eventGraph['EventGraph']['preview_img']);
diff --git a/app/Model/UserSetting.php b/app/Model/UserSetting.php
index 2ab3bcfd8..dbcedc4c3 100644
--- a/app/Model/UserSetting.php
+++ b/app/Model/UserSetting.php
@@ -16,11 +16,7 @@ class UserSetting extends AppModel
     );
 
     public $validate = array(
-        'json' => array(
-            'isValidJson' => array(
-                'rule' => array('isValidJson'),
-            )
-        )
+        'value' => 'valueIsJson',
     );
 
     public $belongsTo = array(

From a1c22e9fe5c3fda3d3bcd136a4d346ffa358aa94 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 17 Oct 2021 14:21:18 +0200
Subject: [PATCH 03/21] chg: [internal] Remove unused validation rule

---
 app/Model/AppModel.php | 13 -------------
 1 file changed, 13 deletions(-)

diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php
index d77aadbd4..5981b97ee 100644
--- a/app/Model/AppModel.php
+++ b/app/Model/AppModel.php
@@ -1951,19 +1951,6 @@ class AppModel extends Model
         return true;
     }
 
-    public function valueIsJsonOrNull($value)
-    {
-        $field = array_keys($value);
-        $field = $field[0];
-        if (!is_null($value[$field])) {
-            $json_decoded = json_decode($value[$field]);
-            if ($json_decoded === null) {
-                return __('Invalid JSON.');
-            }
-        }
-        return true;
-    }
-
     public function valueIsID($value)
     {
         $field = array_keys($value);

From 1ffdfed0b1f154bbb01b8ede4637a97161ffc402 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 17 Oct 2021 14:21:47 +0200
Subject: [PATCH 04/21] chg: [internal] Optimise validators

---
 app/Model/AppModel.php | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php
index 5981b97ee..922e6adff 100644
--- a/app/Model/AppModel.php
+++ b/app/Model/AppModel.php
@@ -1933,8 +1933,8 @@ class AppModel extends Model
     {
         $field = array_keys($value);
         $field = $field[0];
-        $value[$field] = trim($value[$field]);
-        if (!empty($value[$field])) {
+        $value = trim($value[$field]);
+        if (!empty($value)) {
             return true;
         }
         return ucfirst($field) . ' cannot be empty.';
@@ -1942,9 +1942,9 @@ class AppModel extends Model
 
     public function valueIsJson($value)
     {
-        $field = array_keys($value);
-        $field = $field[0];
-        $json_decoded = json_decode($value[$field]);
+        $value = array_values($value);
+        $value = $value[0];
+        $json_decoded = json_decode($value);
         if ($json_decoded === null) {
             return __('Invalid JSON.');
         }
@@ -1965,8 +1965,8 @@ class AppModel extends Model
     {
         $field = array_keys($value);
         $field = $field[0];
-        $value[$field] = trim($value[$field]);
-        if (!isset($value[$field]) || ($value[$field] == false && $value[$field] !== "0")) {
+        $value = trim($value[$field]);
+        if (!isset($value) || ($value == false && $value !== "0")) {
             return ucfirst($field) . ' cannot be empty.';
         }
         return true;

From f481e30fe6059a1c724bd850626aa99330ebcf72 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 17 Oct 2021 14:47:22 +0200
Subject: [PATCH 05/21] chg: [pubsub] Optimise

---
 app/Lib/Tools/FileAccessTool.php | 15 ++++++++
 app/Lib/Tools/PubSubTool.php     | 61 ++++++++++++++------------------
 2 files changed, 42 insertions(+), 34 deletions(-)

diff --git a/app/Lib/Tools/FileAccessTool.php b/app/Lib/Tools/FileAccessTool.php
index 438d471f1..48ecda90a 100644
--- a/app/Lib/Tools/FileAccessTool.php
+++ b/app/Lib/Tools/FileAccessTool.php
@@ -2,6 +2,21 @@
 
 class FileAccessTool
 {
+    /**
+     * @param string $path
+     * @param int $permissions
+     * @throws Exception
+     */
+    public static function createFile($path, $permissions = 0600)
+    {
+        if (!file_exists($path)) {
+            if (!touch($path)) {
+                throw new Exception("Could not create file `$path`.");
+            }
+        }
+        @chmod($path, $permissions); // hide error if current user is not file owner
+    }
+
     /**
      * Creates temporary file, but you have to delete it after use.
      * @param string|null $dir
diff --git a/app/Lib/Tools/PubSubTool.php b/app/Lib/Tools/PubSubTool.php
index 078995f20..b7a7e65e5 100644
--- a/app/Lib/Tools/PubSubTool.php
+++ b/app/Lib/Tools/PubSubTool.php
@@ -1,4 +1,7 @@
 <?php
+App::uses('FileAccessTool', 'Tools');
+App::uses('JsonTool', 'Tools');
+
 class PubSubTool
 {
     const SCRIPTS_TMP = APP . 'files' . DS . 'scripts' . DS . 'tmp' . DS;
@@ -9,18 +12,12 @@ class PubSubTool
      */
     private $redis;
 
-    /**
-     * @var array
-     */
-    private $settings;
-
     public function initTool()
     {
         if (!$this->redis) {
             $settings = $this->getSetSettings();
             $this->setupPubServer($settings);
             $this->redis = $this->createRedisConnection($settings);
-            $this->settings = $settings;
         }
     }
 
@@ -57,8 +54,8 @@ class PubSubTool
     {
         $settings = $this->getSetSettings();
         $redis = $this->createRedisConnection($settings);
-        $redis->rPush($settings['redis_namespace'] . ':command', 'status');
-        $response = $redis->blPop($settings['redis_namespace'] . ':status', 5);
+        $redis->rPush( 'command', 'status');
+        $response = $redis->blPop('status', 5);
         if ($response === null) {
             throw new Exception("No response from status command returned after 5 seconds.");
         }
@@ -80,7 +77,7 @@ class PubSubTool
         App::uses('JSONConverterTool', 'Tools');
         $jsonTool = new JSONConverterTool();
         $json = $jsonTool->convert($event);
-        return $this->pushToRedis(':data:misp_json', $json);
+        return $this->pushToRedis('data:misp_json', $json);
     }
 
     public function event_save(array $event, $action)
@@ -88,7 +85,7 @@ class PubSubTool
         if (!empty($action)) {
             $event['action'] = $action;
         }
-        return $this->pushToRedis(':data:misp_json_event', $event);
+        return $this->pushToRedis('data:misp_json_event', $event);
     }
 
     public function object_save(array $object, $action)
@@ -96,7 +93,7 @@ class PubSubTool
         if (!empty($action)) {
             $object['action'] = $action;
         }
-        return $this->pushToRedis(':data:misp_json_object', $object);
+        return $this->pushToRedis('data:misp_json_object', $object);
     }
 
     public function object_reference_save(array $object_reference, $action)
@@ -104,12 +101,12 @@ class PubSubTool
         if (!empty($action)) {
             $object_reference['action'] = $action;
         }
-        return $this->pushToRedis(':data:misp_json_object_reference', $object_reference);
+        return $this->pushToRedis('data:misp_json_object_reference', $object_reference);
     }
 
     public function publishConversation(array $message)
     {
-        return $this->pushToRedis(':data:misp_json_conversation', $message);
+        return $this->pushToRedis('data:misp_json_conversation', $message);
     }
 
     public function attribute_save(array $attribute, $action = false)
@@ -117,7 +114,7 @@ class PubSubTool
         if (!empty($action)) {
             $attribute['action'] = $action;
         }
-        return $this->pushToRedis(':data:misp_json_attribute', $attribute);
+        return $this->pushToRedis('data:misp_json_attribute', $attribute);
     }
 
     public function tag_save(array $tag, $action = false)
@@ -125,7 +122,7 @@ class PubSubTool
         if (!empty($action)) {
             $tag['action'] = $action;
         }
-        return $this->pushToRedis(':data:misp_json_tag', $tag);
+        return $this->pushToRedis('data:misp_json_tag', $tag);
     }
 
     public function sighting_save(array $sighting, $action = false)
@@ -133,7 +130,7 @@ class PubSubTool
         if (!empty($action)) {
             $sighting['action'] = $action;
         }
-        return $this->pushToRedis(':data:misp_json_sighting', $sighting);
+        return $this->pushToRedis('data:misp_json_sighting', $sighting);
     }
 
     public function warninglist_save(array $warninglist, $action = false)
@@ -141,7 +138,7 @@ class PubSubTool
         if (!empty($action)) {
             $warninglist['action'] = $action;
         }
-        return $this->pushToRedis(':data:misp_json_warninglist', $warninglist);
+        return $this->pushToRedis('data:misp_json_warninglist', $warninglist);
     }
 
     /**
@@ -155,7 +152,7 @@ class PubSubTool
         if (!empty($action)) {
             $data['action'] = $action;
         }
-        return $this->pushToRedis(':data:misp_json_' . $type, $data);
+        return $this->pushToRedis('data:misp_json_' . $type, $data);
     }
 
     public function publish($data, $type, $action = false)
@@ -163,7 +160,7 @@ class PubSubTool
         if (!empty($action)) {
             $data['action'] = $action;
         }
-        return $this->pushToRedis(':data:misp_json_' . $type, $data);
+        return $this->pushToRedis('data:misp_json_' . $type, $data);
     }
 
     public function killService()
@@ -171,7 +168,7 @@ class PubSubTool
         if ($this->checkIfRunning()) {
             $settings = $this->getSetSettings();
             $redis = $this->createRedisConnection($settings);
-            $redis->rPush($settings['redis_namespace'] . ':command', 'kill');
+            $redis->rPush('command', 'kill');
             sleep(1);
             if ($this->checkIfRunning()) {
                 // Still running.
@@ -194,7 +191,7 @@ class PubSubTool
 
         if ($this->checkIfRunning()) {
             $redis = $this->createRedisConnection($settings);
-            $redis->rPush($settings['redis_namespace'] . ':command', 'reload');
+            $redis->rPush( 'command', 'reload');
         } else {
             return 'Setting saved, but something is wrong with the ZeroMQ server. Please check the diagnostics page for more information.';
         }
@@ -226,7 +223,7 @@ class PubSubTool
             if ($this->checkIfRunning(self::OLD_PID_LOCATION)) {
                 // Old version is running, kill it and start again new one.
                 $redis = $this->createRedisConnection($settings);
-                $redis->rPush($settings['redis_namespace'] . ':command', 'kill');
+                $redis->rPush('command', 'kill');
                 sleep(1);
             }
 
@@ -244,10 +241,10 @@ class PubSubTool
     private function pushToRedis($ns, $data)
     {
         if (is_array($data)) {
-            $data = json_encode($data, JSON_UNESCAPED_UNICODE);
+            $data = JsonTool::encode($data);
         }
 
-        $this->redis->rPush($this->settings['redis_namespace'] . $ns, $data);
+        $this->redis->rPush($ns, $data);
         return true;
     }
 
@@ -264,6 +261,7 @@ class PubSubTool
             $redis->auth($redisPassword);
         }
         $redis->select($settings['redis_database']);
+        $redis->setOption(Redis::OPT_PREFIX, $settings['redis_namespace'] . ':');
         return $redis;
     }
 
@@ -274,18 +272,12 @@ class PubSubTool
     private function saveSettingToFile(array $settings)
     {
         $settingFilePath = self::SCRIPTS_TMP . 'mispzmq_settings.json';
-        $settingsFile = new File($settingFilePath, true, 0644);
-        if (!$settingsFile->exists()) {
-            throw new Exception("Could not create zmq config file '$settingFilePath'.");
-        }
+
         // Because setting file contains secrets, it should be readable just by owner. But because in Travis test,
         // config file is created under one user and then changed under other user, file must be readable and writable
         // also by group.
-        @chmod($settingsFile->pwd(), 0660); // hide error if current user is not file owner
-        if (!$settingsFile->write(json_encode($settings))) {
-            throw new Exception("Could not write zmq config file '$settingFilePath'.");
-        }
-        $settingsFile->close();
+        FileAccessTool::createFile($settingFilePath, 0660);
+        FileAccessTool::writeToFile($settingFilePath, JsonTool::encode($settings));
     }
 
     private function getSetSettings()
@@ -302,8 +294,9 @@ class PubSubTool
             'password' => null,
         );
 
+        $pluginConfig = Configure::read('Plugin');
         foreach ($settings as $key => $setting) {
-            $temp = Configure::read('Plugin.ZeroMQ_' . $key);
+            $temp = isset($pluginConfig['ZeroMQ_' . $key]) ? $pluginConfig['ZeroMQ_' . $key] : null;
             if ($temp) {
                 $settings[$key] = $temp;
             }

From a6c5089ebab4569992d560d6dfa033c2fb613426 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 17 Oct 2021 15:05:22 +0200
Subject: [PATCH 06/21] chg: [stix-export] Use JsonTool

---
 app/Lib/Export/StixExport.php | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/Lib/Export/StixExport.php b/app/Lib/Export/StixExport.php
index 75712f615..6c178714e 100644
--- a/app/Lib/Export/StixExport.php
+++ b/app/Lib/Export/StixExport.php
@@ -1,6 +1,7 @@
 <?php
 App::uses('JSONConverterTool', 'Tools');
 App::uses('TmpFileTool', 'Tools');
+App::uses('JsonTool', 'Tools');
 
 abstract class StixExport
 {
@@ -44,7 +45,7 @@ abstract class StixExport
         }
 
         $converter = new JSONConverterTool();
-        $event = json_encode($converter->convert($data, false, true)); // we don't need pretty printed JSON
+        $event = JsonTool::encode($converter->convert($data, false, true)); // we don't need pretty printed JSON
         if ($this->__n_attributes + $attributesCount < $this->__attributes_limit) {
             $this->__tmp_file->append($this->__n_attributes === 0 ? $event : ',' . $event);
             $this->__n_attributes += $attributesCount;

From 8b219c73b5eef2168d8d60b28fa2facc1311c195 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 17 Oct 2021 15:24:54 +0200
Subject: [PATCH 07/21] chg: [modules] Use JsonTool

---
 app/Model/Module.php | 45 +++++++++++++++++++++-----------------------
 1 file changed, 21 insertions(+), 24 deletions(-)

diff --git a/app/Model/Module.php b/app/Model/Module.php
index fc7611e18..003e050cd 100644
--- a/app/Model/Module.php
+++ b/app/Model/Module.php
@@ -1,5 +1,6 @@
 <?php
 App::uses('AppModel', 'Model');
+App::uses('JsonTool', 'Tools');
 
 class Module extends AppModel
 {
@@ -234,28 +235,27 @@ class Module extends AppModel
      * @param array|null $postData
      * @param string $moduleFamily
      * @return array
-     * @throws JsonException
+     * @throws HttpSocketJsonException
+     * @throws Exception
      */
     public function sendRequest($uri, $timeout, $postData = null, $moduleFamily = 'Enrichment')
     {
-        $url = $this->__getModuleServer($moduleFamily);
-        if (!$url) {
+        $serverUrl = $this->__getModuleServer($moduleFamily);
+        if (!$serverUrl) {
             throw new Exception("Module type $moduleFamily is not enabled.");
         }
-        App::uses('HttpSocket', 'Network/Http');
+        App::uses('HttpSocketExtended', 'Tools');
+        $httpSocketSetting = ['timeout' => $timeout];
         $sslSettings = array('ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_verify_peer', 'ssl_cafile');
         foreach ($sslSettings as $sslSetting) {
-            if (Configure::check('Plugin.' . $moduleFamily . '_' . $sslSetting) && Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting) !== '') {
-                $settings[$sslSetting] = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
+            $value = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
+            if ($value && $value !== '') {
+                $httpSocketSetting[$sslSetting] = $value;
             }
         }
-        $httpSocket = new HttpSocket(['timeout' => $timeout]);
-        $request = array(
-            'header' => array(
-                'Content-Type' => 'application/json',
-            )
-        );
-        if ($moduleFamily == 'Cortex') {
+        $httpSocket = new HttpSocketExtended($httpSocketSetting);
+        $request = [];
+        if ($moduleFamily === 'Cortex') {
             if (!empty(Configure::read('Plugin.' . $moduleFamily . '_authkey'))) {
                 $request['header']['Authorization'] = 'Bearer ' . Configure::read('Plugin.' . $moduleFamily . '_authkey');
             }
@@ -264,26 +264,23 @@ class Module extends AppModel
             if (!is_array($postData)) {
                 throw new InvalidArgumentException("Post data must be array, " . gettype($postData) . " given.");
             }
-            $post = json_encode($postData);
-            $response = $httpSocket->post($url . $uri, $post, $request);
+            $post = JsonTool::encode($postData);
+            $request['header']['Content-Type'] = 'application/json';
+            $response = $httpSocket->post($serverUrl . $uri, $post, $request);
         } else {
-            if ($moduleFamily == 'Cortex') {
-                unset($request['header']['Content-Type']);
-            }
-            $response = $httpSocket->get($url . $uri, false, $request);
+            $response = $httpSocket->get($serverUrl . $uri, false, $request);
         }
         if (!$response->isOk()) {
-            if ($httpSocket->lastError()) {
-                throw new Exception("Failed to get response from $moduleFamily module: " . $httpSocket->lastError['str']);
-            }
-            throw new Exception("Failed to get response from $moduleFamily module: HTTP $response->reasonPhrase", (int)$response->code);
+            $e = new HttpSocketHttpException($response, $serverUrl . $uri);
+            throw new Exception("Failed to get response from `$moduleFamily` module", 0, $e);
         }
-        return $this->jsonDecode($response->body);
+        return $response->json();
     }
 
     /**
      * @param string $moduleFamily
      * @return array
+     * @throws JsonException
      */
     public function getModuleSettings($moduleFamily = 'Enrichment')
     {

From 4cba4b33370e2bc38d291c93c55ed2e58f39c835 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 17 Oct 2021 15:25:19 +0200
Subject: [PATCH 08/21] chg: [internal] Optimise JSONConverterTool

---
 app/Lib/Tools/JSONConverterTool.php | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/app/Lib/Tools/JSONConverterTool.php b/app/Lib/Tools/JSONConverterTool.php
index 24cf8f60b..024bf880a 100644
--- a/app/Lib/Tools/JSONConverterTool.php
+++ b/app/Lib/Tools/JSONConverterTool.php
@@ -48,7 +48,7 @@ class JSONConverterTool
             }
         }
 
-        if (isset($event['Event']['SharingGroup']) && empty($event['Event']['SharingGroup'])) {
+        if (empty($event['Event']['SharingGroup'])) {
             unset($event['Event']['SharingGroup']);
         }
 
@@ -86,11 +86,6 @@ class JSONConverterTool
         }
         unset($tempSightings);
         unset($event['Event']['RelatedAttribute']);
-        if (isset($event['Event']['RelatedEvent'])) {
-            foreach ($event['Event']['RelatedEvent'] as $key => $value) {
-                unset($event['Event']['RelatedEvent'][$key]['Event']['user_id']);
-            }
-        }
         $result = array('Event' => $event['Event']);
         if (isset($event['errors'])) {
             $result = array_merge($result, array('errors' => $event['errors']));
@@ -143,7 +138,7 @@ class JSONConverterTool
     {
         // remove value1 and value2 from the output and remove invalid utf8 characters for the xml parser
         foreach ($attributes as $key => $attribute) {
-            if (isset($attribute['SharingGroup']) && empty($attribute['SharingGroup'])) {
+            if (empty($attribute['SharingGroup'])) {
                 unset($attributes[$key]['SharingGroup']);
             }
             unset($attributes[$key]['value1']);

From f179ec0db6e33ca0ad8b19d3821e5d5fb710f8c9 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 17 Oct 2021 16:33:35 +0200
Subject: [PATCH 09/21] chg: [internal] Optimise datetimeOrNull method

---
 app/Model/Attribute.php  | 22 ++++++++--------------
 app/Model/MispObject.php | 22 +++++++++-------------
 2 files changed, 17 insertions(+), 27 deletions(-)

diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php
index 91b2d5c5b..13c6ef281 100644
--- a/app/Model/Attribute.php
+++ b/app/Model/Attribute.php
@@ -742,21 +742,17 @@ class Attribute extends AppModel
     // check whether the variable is null or datetime
     public function datetimeOrNull($fields)
     {
-        $k = array_keys($fields)[0];
-        $seen = $fields[$k];
-        try {
-            new DateTime($seen);
-            $returnValue = true;
-        } catch (Exception $e) {
-            $returnValue = false;
+        $seen = array_values($fields)[0];
+        if ($seen === null) {
+            return true;
         }
-        return $returnValue || is_null($seen);
+        return strtotime($seen) !== false;
     }
 
     public function validateLastSeenValue($fields)
     {
         $ls = $fields['last_seen'];
-        if (!isset($this->data['Attribute']['first_seen']) || is_null($ls)) {
+        if (!isset($this->data['Attribute']['first_seen']) || $ls === null) {
             return true;
         }
         $converted = $this->ISODatetimeToUTC(['Attribute' => [
@@ -1410,7 +1406,7 @@ class Attribute extends AppModel
                 break;
             case 'datetime':
                 try {
-                    $value = (new DateTime($value))->setTimezone(new DateTimeZone('GMT'))->format('Y-m-d\TH:i:s.uO'); // ISO8601 formating with microseconds
+                    $value = (new DateTime($value, new DateTimeZone('GMT')))->format('Y-m-d\TH:i:s.uO'); // ISO8601 formating with microseconds
                 } catch (Exception $e) {
                     // silently skip. Rejection will be done in runValidation()
                 }
@@ -1698,8 +1694,7 @@ class Attribute extends AppModel
     {
         // convert into utc and micro sec
         if (!empty($data[$alias]['first_seen'])) {
-            $d = new DateTime($data[$alias]['first_seen']);
-            $d->setTimezone(new DateTimeZone('GMT'));
+            $d = new DateTime($data[$alias]['first_seen'], new DateTimeZone('GMT'));
             $fs_sec = $d->format('U');
             $fs_micro = $d->format('u');
             $fs_micro = str_pad($fs_micro, 6, "0", STR_PAD_LEFT);
@@ -1707,8 +1702,7 @@ class Attribute extends AppModel
             $data[$alias]['first_seen'] = $fs;
         }
         if (!empty($data[$alias]['last_seen'])) {
-            $d = new DateTime($data[$alias]['last_seen']);
-            $d->setTimezone(new DateTimeZone('GMT'));
+            $d = new DateTime($data[$alias]['last_seen'], new DateTimeZone('GMT'));
             $ls_sec = $d->format('U');
             $ls_micro = $d->format('u');
             $ls_micro = str_pad($ls_micro, 6, "0", STR_PAD_LEFT);
diff --git a/app/Model/MispObject.php b/app/Model/MispObject.php
index bd7856bd5..f1bb226f8 100644
--- a/app/Model/MispObject.php
+++ b/app/Model/MispObject.php
@@ -216,23 +216,19 @@ class MispObject extends AppModel
     }
 
      // check whether the variable is null or datetime
-     public function datetimeOrNull($fields)
-     {
-         $k = array_keys($fields)[0];
-         $seen = $fields[$k];
-         try {
-             new DateTime($seen);
-             $returnValue = true;
-         } catch (Exception $e) {
-             $returnValue = false;
-         }
-         return $returnValue || is_null($seen);
-     }
+    public function datetimeOrNull($fields)
+    {
+        $seen = array_values($fields)[0];
+        if ($seen === null) {
+            return true;
+        }
+        return strtotime($seen) !== false;
+    }
 
      public function validateLastSeenValue($fields)
      {
          $ls = $fields['last_seen'];
-         if (!isset($this->data['Object']['first_seen']) || is_null($ls)) {
+         if (!isset($this->data['Object']['first_seen']) || $ls === null) {
              return true;
          }
          $converted = $this->Attribute->ISODatetimeToUTC(['Object' => [

From ea9360e680aa70b3b8eb846590ae0bbf1eae7cea Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 17 Oct 2021 16:44:15 +0200
Subject: [PATCH 10/21] chg: [internal] Optimise beforeValidate for object

---
 app/Model/MispObject.php | 39 +++++++++++++++++++--------------------
 1 file changed, 19 insertions(+), 20 deletions(-)

diff --git a/app/Model/MispObject.php b/app/Model/MispObject.php
index f1bb226f8..943ec7258 100644
--- a/app/Model/MispObject.php
+++ b/app/Model/MispObject.php
@@ -255,38 +255,37 @@ class MispObject extends AppModel
 
     public function beforeValidate($options = array())
     {
-        parent::beforeValidate();
-        if (empty($this->data[$this->alias]['comment'])) {
-            $this->data[$this->alias]['comment'] = "";
+        $object = &$this->data['Object'];
+        if (empty($object['comment'])) {
+            $object['comment'] = "";
         }
         // generate UUID if it doesn't exist
-        if (empty($this->data[$this->alias]['uuid'])) {
-            $this->data[$this->alias]['uuid'] = CakeText::uuid();
+        if (empty($object['uuid'])) {
+            $object['uuid'] = CakeText::uuid();
         }
         // generate timestamp if it doesn't exist
-        if (empty($this->data[$this->alias]['timestamp'])) {
-            $date = new DateTime();
-            $this->data[$this->alias]['timestamp'] = $date->getTimestamp();
+        if (empty($object['timestamp'])) {
+            $object['timestamp'] = time();
         }
         // parse first_seen different formats
-        if (isset($this->data[$this->alias]['first_seen'])) {
-            $this->data[$this->alias]['first_seen'] = $this->data[$this->alias]['first_seen'] === '' ? null : $this->data[$this->alias]['first_seen'];
+        if (isset($object['first_seen'])) {
+            $object['first_seen'] = $object['first_seen'] === '' ? null : $object['first_seen'];
         }
         // parse last_seen different formats
-        if (isset($this->data[$this->alias]['last_seen'])) {
-            $this->data[$this->alias]['last_seen'] = $this->data[$this->alias]['last_seen'] === '' ? null : $this->data[$this->alias]['last_seen'];
+        if (isset($object['last_seen'])) {
+            $object['last_seen'] = $object['last_seen'] === '' ? null : $object['last_seen'];
         }
-        if (empty($this->data[$this->alias]['template_version'])) {
-            $this->data[$this->alias]['template_version'] = 1;
+        if (empty($object['template_version'])) {
+            $object['template_version'] = 1;
         }
-        if (isset($this->data[$this->alias]['deleted']) && empty($this->data[$this->alias]['deleted'])) {
-            $this->data[$this->alias]['deleted'] = 0;
+        if (isset($object['deleted']) && empty($object['deleted'])) {
+            $object['deleted'] = 0;
         }
-        if (!isset($this->data[$this->alias]['distribution']) || $this->data['Object']['distribution'] != 4) {
-            $this->data['Object']['sharing_group_id'] = 0;
+        if (!isset($object['distribution']) || $object['distribution'] != 4) {
+            $object['sharing_group_id'] = 0;
         }
-        if (!isset($this->data[$this->alias]['distribution'])) {
-            $this->data['Object']['distribution'] = 5;
+        if (!isset($object['distribution'])) {
+            $object['distribution'] = 5;
         }
         return true;
     }

From 6fe3a895c8e100dcfcf2d7dd2a70747b047f77a3 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 17 Oct 2021 16:49:26 +0200
Subject: [PATCH 11/21] chg: [internal] Simplify validation

---
 app/Model/AppModel.php | 12 ++++--------
 1 file changed, 4 insertions(+), 8 deletions(-)

diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php
index 922e6adff..549adfcce 100644
--- a/app/Model/AppModel.php
+++ b/app/Model/AppModel.php
@@ -1931,8 +1931,7 @@ class AppModel extends Model
     // alternative to the build in notempty/notblank validation functions, compatible with cakephp <= 2.6 and cakephp and cakephp >= 2.7
     public function valueNotEmpty($value)
     {
-        $field = array_keys($value);
-        $field = $field[0];
+        $field = array_keys($value)[0];
         $value = trim($value[$field]);
         if (!empty($value)) {
             return true;
@@ -1942,8 +1941,7 @@ class AppModel extends Model
 
     public function valueIsJson($value)
     {
-        $value = array_values($value);
-        $value = $value[0];
+        $value = array_values($value)[0];
         $json_decoded = json_decode($value);
         if ($json_decoded === null) {
             return __('Invalid JSON.');
@@ -1953,8 +1951,7 @@ class AppModel extends Model
 
     public function valueIsID($value)
     {
-        $field = array_keys($value);
-        $field = $field[0];
+        $field = array_keys($value)[0];
         if (!is_numeric($value[$field]) || $value[$field] < 0) {
             return 'Invalid ' . ucfirst($field) . ' ID';
         }
@@ -1963,8 +1960,7 @@ class AppModel extends Model
 
     public function stringNotEmpty($value)
     {
-        $field = array_keys($value);
-        $field = $field[0];
+        $field = array_keys($value)[0];
         $value = trim($value[$field]);
         if (!isset($value) || ($value == false && $value !== "0")) {
             return ucfirst($field) . ' cannot be empty.';

From 0043148ca5f2b262d5155a08758f2f6a7608fe82 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 17 Oct 2021 17:12:38 +0200
Subject: [PATCH 12/21] chg: [internal] Simplify Attribute::beforeValidate

---
 app/Model/Attribute.php | 65 +++++++++++++++++++++--------------------
 1 file changed, 33 insertions(+), 32 deletions(-)

diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php
index 13c6ef281..c90a49820 100644
--- a/app/Model/Attribute.php
+++ b/app/Model/Attribute.php
@@ -563,81 +563,82 @@ class Attribute extends AppModel
 
     public function beforeValidate($options = array())
     {
-        if (empty($this->data['Attribute']['type'])) {
+        $attribute = &$this->data['Attribute'];
+        if (empty($attribute['type'])) {
             $this->validationErrors['type'] = ['No type set.'];
             return false;
         }
-        $type = $this->data['Attribute']['type'];
-        if (is_array($this->data['Attribute']['value'])) {
+        $type = $attribute['type'];
+        if (is_array($attribute['value'])) {
             $this->validationErrors['value'] = ['Value is an array.'];
             return false;
         }
 
-        if (!empty($this->data['Attribute']['object_id']) && empty($this->data['Attribute']['object_relation'])) {
+        if (!empty($attribute['object_id']) && empty($attribute['object_relation'])) {
             $this->validationErrors['object_relation'] = ['Object attribute sent, but no object_relation set.'];
             return false;
         }
 
         // If `value1` or `value2` provided and `value` is empty, merge them into `value` because of validation
-        if (empty($this->data['Attribute']['value'])) {
-            if (!empty($this->data['Attribute']['value1']) && !empty($this->data['Attribute']['value2'])) {
-                $this->data['Attribute']['value'] = "{$this->data['Attribute']['value1']}|{$this->data['Attribute']['value2']}";
-            } else if (!empty($this->data['Attribute']['value1'])) {
-                $this->data['Attribute']['value'] = $this->data['Attribute']['value1'];
+        if (empty($attribute['value'])) {
+            if (!empty($attribute['value1']) && !empty($attribute['value2'])) {
+                $attribute['value'] = "{$attribute['value1']}|{$attribute['value2']}";
+            } else if (!empty($attribute['value1'])) {
+                $attribute['value'] = $attribute['value1'];
             }
         }
 
         // remove leading and trailing blanks and refang value and
-        $this->data['Attribute']['value'] = ComplexTypeTool::refangValue(trim($this->data['Attribute']['value']), $type);
+        $attribute['value'] = ComplexTypeTool::refangValue(trim($attribute['value']), $type);
         // make some changes to the inserted value
-        $this->data['Attribute']['value'] = $this->modifyBeforeValidation($type, $this->data['Attribute']['value']);
+        $attribute['value'] = $this->modifyBeforeValidation($type, $attribute['value']);
         // Run user defined regexp to attribute value
-        $result = $this->runRegexp($type, $this->data['Attribute']['value']);
+        $result = $this->runRegexp($type, $attribute['value']);
         if ($result === false) {
             $this->invalidate('value', 'This value is blocked by a regular expression in the import filters.');
         } else {
-            $this->data['Attribute']['value'] = $result;
+            $attribute['value'] = $result;
         }
 
-        if (empty($this->data['Attribute']['comment'])) {
-            $this->data['Attribute']['comment'] = "";
+        if (empty($attribute['comment'])) {
+            $attribute['comment'] = "";
         }
         // generate UUID if it doesn't exist
-        if (empty($this->data['Attribute']['uuid'])) {
-            $this->data['Attribute']['uuid'] = CakeText::uuid();
+        if (empty($attribute['uuid'])) {
+            $attribute['uuid'] = CakeText::uuid();
         } else {
-            $this->data['Attribute']['uuid'] = strtolower($this->data['Attribute']['uuid']);
+            $attribute['uuid'] = strtolower($attribute['uuid']);
         }
         // generate timestamp if it doesn't exist
-        if (empty($this->data['Attribute']['timestamp'])) {
-            $this->data['Attribute']['timestamp'] = time();
+        if (empty($attribute['timestamp'])) {
+            $attribute['timestamp'] = time();
         }
 
         // parse first_seen different formats
-        if (isset($this->data['Attribute']['first_seen'])) {
-            $this->data['Attribute']['first_seen'] = $this->data['Attribute']['first_seen'] === '' ? null : $this->data['Attribute']['first_seen'];
+        if (isset($attribute['first_seen'])) {
+            $attribute['first_seen'] = $attribute['first_seen'] === '' ? null : $attribute['first_seen'];
         }
         // parse last_seen different formats
-        if (isset($this->data['Attribute']['last_seen'])) {
-            $this->data['Attribute']['last_seen'] = $this->data['Attribute']['last_seen'] === '' ? null : $this->data['Attribute']['last_seen'];
+        if (isset($attribute['last_seen'])) {
+            $attribute['last_seen'] = $attribute['last_seen'] === '' ? null : $attribute['last_seen'];
         }
 
         // Set defaults for when some of the mandatory fields don't have defaults
         // These fields all have sane defaults either based on another field, or due to server settings
-        if (!isset($this->data['Attribute']['distribution'])) {
-            $this->data['Attribute']['distribution'] = $this->defaultDistribution();
+        if (!isset($attribute['distribution'])) {
+            $attribute['distribution'] = $this->defaultDistribution();
         }
         // If category is not provided, assign default category by type
-        if (empty($this->data['Attribute']['category'])) {
-            $this->data['Attribute']['category'] = $this->typeDefinitions[$type]['default_category'];
+        if (empty($attribute['category'])) {
+            $attribute['category'] = $this->typeDefinitions[$type]['default_category'];
         }
 
-        if (!isset($this->data['Attribute']['to_ids'])) {
-            $this->data['Attribute']['to_ids'] = $this->typeDefinitions[$type]['to_ids'];
+        if (!isset($attribute['to_ids'])) {
+            $attribute['to_ids'] = $this->typeDefinitions[$type]['to_ids'];
         }
 
-        if ($this->data['Attribute']['distribution'] != 4) {
-            $this->data['Attribute']['sharing_group_id'] = 0;
+        if ($attribute['distribution'] != 4) {
+            $attribute['sharing_group_id'] = 0;
         }
         // return true, otherwise the object cannot be saved
         return true;

From 7fae06785bd3234a545ce08ed897cc2f4aef7ea8 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 17 Oct 2021 18:39:54 +0200
Subject: [PATCH 13/21] chg: [internal] Simplify Event::beforeValidate

---
 app/Model/Event.php | 64 ++++++++++++++++++++++++---------------------
 1 file changed, 34 insertions(+), 30 deletions(-)

diff --git a/app/Model/Event.php b/app/Model/Event.php
index c4876e525..6f0950dee 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -442,65 +442,69 @@ class Event extends AppModel
 
     public function beforeValidate($options = array())
     {
+        $event = &$this->data['Event'];
         // analysis - setting correct vars
-        if (isset($this->data['Event']['analysis'])) {
-            switch ($this->data['Event']['analysis']) {
+        if (isset($event['analysis'])) {
+            switch ($event['analysis']) {
                 case 'Initial':
-                    $this->data['Event']['analysis'] = 0;
+                    $event['analysis'] = 0;
                     break;
                 case 'Ongoing':
-                    $this->data['Event']['analysis'] = 1;
+                    $event['analysis'] = 1;
                     break;
                 case 'Completed':
-                    $this->data['Event']['analysis'] = 2;
+                    $event['analysis'] = 2;
                     break;
             }
         } else {
-            $this->data['Event']['analysis'] = 0;
+            $event['analysis'] = 0;
         }
 
-        if (!isset($this->data['Event']['threat_level_id'])) {
-            $this->data['Event']['threat_level_id'] = Configure::read('MISP.default_event_threat_level') ?: 4;
+        if (!isset($event['threat_level_id'])) {
+            $event['threat_level_id'] = Configure::read('MISP.default_event_threat_level') ?: 4;
         }
 
         // generate UUID if it doesn't exist
-        if (empty($this->data['Event']['uuid'])) {
-            $this->data['Event']['uuid'] = CakeText::uuid();
+        if (empty($event['uuid'])) {
+            $event['uuid'] = CakeText::uuid();
         } else {
-            $this->data['Event']['uuid'] = strtolower($this->data['Event']['uuid']);
+            $event['uuid'] = strtolower($event['uuid']);
         }
 
         // Convert event ID to uuid if needed
-        if (!empty($this->data['Event']['extends_uuid']) && is_numeric($this->data['Event']['extends_uuid'])) {
-            $extended_event = $this->find('first', array(
-                'recursive' => -1,
-                'conditions' => array('Event.id' => $this->data['Event']['extends_uuid']),
-                'fields' => array('Event.uuid')
-            ));
-            if (empty($extended_event)) {
-                $this->data['Event']['extends_uuid'] = '';
+        if (!empty($event['extends_uuid'])) {
+            if (is_numeric($event['extends_uuid'])) {
+                $extended_event = $this->find('first', array(
+                    'recursive' => -1,
+                    'conditions' => array('Event.id' => $event['extends_uuid']),
+                    'fields' => array('Event.uuid')
+                ));
+                if (empty($extended_event)) {
+                    $event['extends_uuid'] = '';
+                    $this->invalidate('extends_uuid', 'Invalid event ID provided.');
+                } else {
+                    $event['extends_uuid'] = $extended_event['Event']['uuid'];
+                }
             } else {
-                $this->data['Event']['extends_uuid'] = $extended_event['Event']['uuid'];
+                $event['extends_uuid'] = strtolower($event['extends_uuid']);
             }
-        } else if (!empty($this->data['Event']['extends_uuid'])) {
-            $this->data['Event']['extends_uuid'] = strtolower($this->data['Event']['extends_uuid']);
         }
 
         // generate timestamp if it doesn't exist
-        if (empty($this->data['Event']['timestamp'])) {
-            $this->data['Event']['timestamp'] = time();
+        if (empty($event['timestamp'])) {
+            $event['timestamp'] = time();
         }
 
-        if (isset($this->data['Event']['publish_timestamp']) && empty($this->data['Event']['publish_timestamp'])) {
-            $this->data['Event']['publish_timestamp'] = 0;
+        if (isset($event['publish_timestamp']) && empty($event['publish_timestamp'])) {
+            $event['publish_timestamp'] = 0;
         }
 
-        if (empty($this->data['Event']['date'])) {
-            $this->data['Event']['date'] = date('Y-m-d');
+        if (empty($event['date'])) {
+            $event['date'] = date('Y-m-d');
         }
 
-        if (!isset($this->data['Event']['distribution']) || $this->data['Event']['distribution'] != 4) {
-            $this->data['Event']['sharing_group_id'] = 0;
+        if (!isset($event['distribution']) || $event['distribution'] != 4) {
+            $event['sharing_group_id'] = 0;
         }
     }
 

From a1a9ee9e6e4d7ee981617c329bf979d7da504290 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 17 Oct 2021 18:47:46 +0200
Subject: [PATCH 14/21] chg: [internal] Optimise fetching correlations

---
 app/Model/Event.php | 57 ++++++++++++++++++++++++---------------------
 1 file changed, 30 insertions(+), 27 deletions(-)

diff --git a/app/Model/Event.php b/app/Model/Event.php
index 6f0950dee..28c3d7193 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -550,10 +550,10 @@ class Event extends AppModel
             'order' => false
         ));
         $tags = array_column(array_column($tags, 'Tag'), null, 'id');
-        foreach ($events as $k => $event) {
+        foreach ($events as &$event) {
             if (!empty($event['EventTag'])) {
-                foreach ($event['EventTag'] as $k2 => $et) {
-                    $events[$k]['EventTag'][$k2]['Tag'] = $tags[$et['tag_id']];
+                foreach ($event['EventTag'] as &$et) {
+                    $et['Tag'] = $tags[$et['tag_id']];
                 }
             }
         }
@@ -776,51 +776,53 @@ class Event extends AppModel
     public function getRelatedAttributes(array $user, $id, $shadowAttribute = false, $scope = 'event')
     {
         if ($shadowAttribute) {
-            $settings = array('correlationModel' => 'ShadowAttributeCorrelation', 'parentIdField' => '1_shadow_attribute_id');
+            $parentIdField = '1_shadow_attribute_id';
+            $correlationModelName = 'ShadowAttributeCorrelation';
         } else {
-            $settings = array('correlationModel' => 'Correlation', 'parentIdField' => '1_attribute_id');
+            $parentIdField = '1_attribute_id';
+            $correlationModelName = 'Correlation';
         }
-        if (!isset($this->{$settings['correlationModel']})) {
-            $this->{$settings['correlationModel']} = ClassRegistry::init($settings['correlationModel']);
+        if (!isset($this->{$correlationModelName})) {
+            $this->{$correlationModelName} = ClassRegistry::init($correlationModelName);
         }
         if (!$user['Role']['perm_site_admin']) {
             $sgids = $this->cacheSgids($user, true);
             $conditionsCorrelation = array(
                 'AND' => array(
-                    $settings['correlationModel'] . '.1_' . $scope . '_id' => $id,
+                    $correlationModelName . '.1_' . $scope . '_id' => $id,
                     array(
                         'OR' => array(
-                            $settings['correlationModel'] . '.org_id' => $user['org_id'],
+                            $correlationModelName . '.org_id' => $user['org_id'],
                             'AND' => array(
                                 array(
                                     'OR' => array(
                                         array(
                                             'AND' => array(
-                                                $settings['correlationModel'] . '.distribution >' => 0,
-                                                $settings['correlationModel'] . '.distribution <' => 4,
+                                                $correlationModelName . '.distribution >' => 0,
+                                                $correlationModelName . '.distribution <' => 4,
                                             ),
                                         ),
                                         array(
                                             'AND' => array(
-                                                $settings['correlationModel'] . '.distribution' => 4,
-                                                $settings['correlationModel'] . '.sharing_group_id' => $sgids
+                                                $correlationModelName . '.distribution' => 4,
+                                                $correlationModelName . '.sharing_group_id' => $sgids
                                             ),
                                         ),
                                     ),
                                 ),
                                 array(
                                     'OR' => array(
-                                        $settings['correlationModel'] . '.a_distribution' => 5,
+                                        $correlationModelName . '.a_distribution' => 5,
                                         array(
                                             'AND' => array(
-                                                $settings['correlationModel'] . '.a_distribution >' => 0,
-                                                $settings['correlationModel'] . '.a_distribution <' => 4,
+                                                $correlationModelName . '.a_distribution >' => 0,
+                                                $correlationModelName . '.a_distribution <' => 4,
                                             ),
                                         ),
                                         array(
                                             'AND' => array(
-                                                $settings['correlationModel'] . '.a_distribution' => 4,
-                                                $settings['correlationModel'] . '.a_sharing_group_id' => $sgids
+                                                $correlationModelName . '.a_distribution' => 4,
+                                                $correlationModelName . '.a_sharing_group_id' => $sgids
                                             ),
                                         ),
                                     ),
@@ -832,11 +834,11 @@ class Event extends AppModel
                 )
             );
         } else {
-            $conditionsCorrelation = array($settings['correlationModel'] . '.1_' . $scope . '_id' => $id);
+            $conditionsCorrelation = array($correlationModelName . '.1_' . $scope . '_id' => $id);
         }
         $max_correlations = Configure::read('MISP.max_correlations_per_event') ?: 5000;
-        $correlations = $this->{$settings['correlationModel']}->find('all', array(
-            'fields' => ['event_id', 'attribute_id', 'value', $settings['parentIdField']],
+        $correlations = $this->{$correlationModelName}->find('all', array(
+            'fields' => ['event_id', 'attribute_id', 'value', $parentIdField],
             'conditions' => $conditionsCorrelation,
             'recursive' => -1,
             'order' => false,
@@ -846,11 +848,11 @@ class Event extends AppModel
             return array();
         }
 
-        $correlations = array_column($correlations, $settings['correlationModel']);
+        $correlations = array_column($correlations, $correlationModelName);
         $eventIds = array_unique(array_column($correlations, 'event_id'));
 
         $conditions = $this->createEventConditions($user);
-        $conditions['AND']['Event.id'] = $eventIds;
+        $conditions['Event.id'] = $eventIds;
         $events = $this->find('all', array(
             'recursive' => -1,
             'conditions' => $conditions,
@@ -862,20 +864,21 @@ class Event extends AppModel
         $relatedAttributes = [];
         foreach ($correlations as $correlation) {
             // User don't have access to correlated attribute event, skip.
-            if (!isset($events[$correlation['event_id']])) {
+            $eventId = $correlation['event_id'];
+            if (!isset($events[$eventId])) {
                 continue;
             }
 
-            $event = $events[$correlation['event_id']];
+            $event = $events[$eventId];
             $current = array(
-                'id' => $correlation['event_id'],
+                'id' => $eventId,
                 'attribute_id' => $correlation['attribute_id'],
                 'value' => $correlation['value'],
                 'org_id' => $event['orgc_id'],
                 'info' => $event['info'],
                 'date' => $event['date'],
             );
-            $parentId = $correlation[$settings['parentIdField']];
+            $parentId = $correlation[$parentIdField];
             $relatedAttributes[$parentId][] = $current;
         }
         return $relatedAttributes;

From 9e0c36ff2a5ed97f83d6857f50a618c7ea84fb29 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 17 Oct 2021 18:57:29 +0200
Subject: [PATCH 15/21] chg: [internal] Simplify Event::__attachReferences

---
 app/Model/Event.php | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/app/Model/Event.php b/app/Model/Event.php
index 28c3d7193..c748cd437 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -2131,13 +2131,6 @@ class Event extends AppModel
         // Precache current user email
         $userEmails = empty($user['id']) ? [] : [$user['id'] => $user['email']];
 
-        // Do some refactoring with the event
-        $fields = array(
-            'common' => array('distribution', 'sharing_group_id', 'uuid'),
-            'Attribute' => array('value', 'type', 'category', 'to_ids'),
-            'Object' => array('name', 'meta-category')
-        );
-
         if (!$options['includeAllTags']) {
             $justExportableTags = true;
         } else {
@@ -2169,7 +2162,7 @@ class Event extends AppModel
                 $this->Warninglist->attachWarninglistToAttributes($event['ShadowAttribute']);
                 $event['warnings'] = $eventWarnings;
             }
-            $this->__attachReferences($event, $fields);
+            $this->__attachReferences($event);
             $this->__attachTags($event, $justExportableTags);
             $this->__attachGalaxies($event, $user, $options['excludeGalaxy'], $options['fetchFullClusters']);
             $event = $this->Orgc->attachOrgs($event, $fieldsOrg);
@@ -6940,13 +6933,19 @@ class Event extends AppModel
      * we just find proper element in event.
      *
      * @param array $event
-     * @param array $fields
      */
-    private function __attachReferences(array &$event, array $fields)
+    private function __attachReferences(array &$event)
     {
         if (!isset($event['Object'])) {
             return;
         }
+
+        $fieldsToCopy = array(
+            'common' => array('distribution', 'sharing_group_id', 'uuid'),
+            'Attribute' => array('value', 'type', 'category', 'to_ids'),
+            'Object' => array('name', 'meta-category')
+        );
+
         foreach ($event['Object'] as $k => $object) {
             foreach ($object['ObjectReference'] as $k2 => $reference) {
                 // find referenced object in current event
@@ -6962,7 +6961,7 @@ class Event extends AppModel
                 if ($found) {
                     // copy requested fields
                     $reference = [];
-                    foreach (array_merge($fields['common'], $fields[$type]) as $field) {
+                    foreach (array_merge($fieldsToCopy['common'], $fieldsToCopy[$type]) as $field) {
                         $reference[$field] = $found[$field];
                     }
                     $event['Object'][$k]['ObjectReference'][$k2][$type] = $reference;

From f99f82fa34f5ce216dad21b856466357f9c95754 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 17 Oct 2021 19:07:30 +0200
Subject: [PATCH 16/21] chg: [internal] SaveMany for Event::add_original_file

---
 app/Model/Event.php | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/app/Model/Event.php b/app/Model/Event.php
index c748cd437..1a019052c 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -7218,9 +7218,8 @@ class Event extends AppModel
                 'disable_correlation' => true
             )
         );
-        foreach ($attributes as $attribute) {
-            $this->Attribute->create();
-            $this->Attribute->save($attribute);
+        if (!$this->Attribute->saveMany($attributes)) {
+            throw new Exception("Could not save attributes for original file because of validation errors:" . json_encode($this->Attribute->validationErrors));
         }
         return true;
     }

From 8a0532d902a54b4d847393860885a500f7932693 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 17 Oct 2021 21:48:38 +0200
Subject: [PATCH 17/21] chg: [internal] Move UUID generation to beforeSave
 method

---
 app/Model/Attribute.php  | 15 +++++++--------
 app/Model/Event.php      | 14 ++++++++++----
 app/Model/MispObject.php | 12 ++++++------
 3 files changed, 23 insertions(+), 18 deletions(-)

diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php
index c90a49820..8043931c8 100644
--- a/app/Model/Attribute.php
+++ b/app/Model/Attribute.php
@@ -362,6 +362,9 @@ class Attribute extends AppModel
 
     public function beforeSave($options = array())
     {
+        if (empty($this->data['Attribute']['uuid'])) {
+            $this->data['Attribute']['uuid'] = CakeText::uuid();
+        }
         if (!empty($this->data['Attribute']['id'])) {
             $this->old = $this->find('first', array(
                 'recursive' => -1,
@@ -603,10 +606,7 @@ class Attribute extends AppModel
         if (empty($attribute['comment'])) {
             $attribute['comment'] = "";
         }
-        // generate UUID if it doesn't exist
-        if (empty($attribute['uuid'])) {
-            $attribute['uuid'] = CakeText::uuid();
-        } else {
+        if (!empty($attribute['uuid'])) {
             $attribute['uuid'] = strtolower($attribute['uuid']);
         }
         // generate timestamp if it doesn't exist
@@ -628,6 +628,9 @@ class Attribute extends AppModel
         if (!isset($attribute['distribution'])) {
             $attribute['distribution'] = $this->defaultDistribution();
         }
+        if ($attribute['distribution'] != 4) {
+            $attribute['sharing_group_id'] = 0;
+        }
         // If category is not provided, assign default category by type
         if (empty($attribute['category'])) {
             $attribute['category'] = $this->typeDefinitions[$type]['default_category'];
@@ -636,10 +639,6 @@ class Attribute extends AppModel
         if (!isset($attribute['to_ids'])) {
             $attribute['to_ids'] = $this->typeDefinitions[$type]['to_ids'];
         }
-
-        if ($attribute['distribution'] != 4) {
-            $attribute['sharing_group_id'] = 0;
-        }
         // return true, otherwise the object cannot be saved
         return true;
     }
diff --git a/app/Model/Event.php b/app/Model/Event.php
index 1a019052c..6dec35aae 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -142,7 +142,6 @@ class Event extends AppModel
                 //'required' => true,
                 //'allowEmpty' => true
         ),
-
         'analysis' => array(
             'rule' => array('inList', array('0', '1', '2')),
                 'message' => 'Options : 0, 1, 2 (for Initial, Ongoing, Completed)',
@@ -465,9 +464,7 @@ class Event extends AppModel
         }
 
         // generate UUID if it doesn't exist
-        if (empty($event['uuid'])) {
-            $event['uuid'] = CakeText::uuid();
-        } else {
+        if (!empty($event['uuid'])) {
             $event['uuid'] = strtolower($event['uuid']);
         }
 
@@ -508,6 +505,15 @@ class Event extends AppModel
         }
     }
 
+    public function beforeSave($options = [])
+    {
+        // generate UUID if not provided
+        if (empty($this->data['Event']['uuid'])) {
+            $this->data['Event']['uuid'] = CakeText::uuid();
+        }
+        return true;
+    }
+
     public function afterSave($created, $options = array())
     {
         if (!Configure::read('MISP.completely_disable_correlation') && !$created) {
diff --git a/app/Model/MispObject.php b/app/Model/MispObject.php
index 943ec7258..1b69c2d6b 100644
--- a/app/Model/MispObject.php
+++ b/app/Model/MispObject.php
@@ -69,7 +69,6 @@ class MispObject extends AppModel
             'unique' => array(
                 'rule' => 'isUnique',
                 'message' => 'The UUID provided is not unique',
-                'required' => true,
                 'on' => 'create'
             ),
         ),
@@ -249,7 +248,12 @@ class MispObject extends AppModel
         return $results;
     }
 
-    public function beforeSave($options = array()) {
+    public function beforeSave($options = array())
+    {
+        // generate UUID if it doesn't exist
+        if (empty($this->data['Object']['uuid'])) {
+            $this->data['Object']['uuid'] = CakeText::uuid();
+        }
         $this->data = $this->Attribute->ISODatetimeToUTC($this->data, $this->alias);
     }
 
@@ -259,10 +263,6 @@ class MispObject extends AppModel
         if (empty($object['comment'])) {
             $object['comment'] = "";
         }
-        // generate UUID if it doesn't exist
-        if (empty($object['uuid'])) {
-            $object['uuid'] = CakeText::uuid();
-        }
         // generate timestamp if it doesn't exist
         if (empty($object['timestamp'])) {
             $object['timestamp'] = time();

From 03e903a41d0cedb104a6e31f8143d476eea5fc21 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 17 Oct 2021 22:31:57 +0200
Subject: [PATCH 18/21] chg: [internal] Simplify validation for Event org_id
 and orgc_id fields

---
 app/Model/Event.php | 36 +++++++++++++++---------------------
 1 file changed, 15 insertions(+), 21 deletions(-)

diff --git a/app/Model/Event.php b/app/Model/Event.php
index 6dec35aae..4b671ee01 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -96,20 +96,14 @@ class Event extends AppModel
 
     public $validate = array(
         'org_id' => array(
-            'valueNotEmpty' => array(
-                'rule' => array('valueNotEmpty'),
-            ),
-            'numeric' => array(
-                'rule' => array('numeric'),
-            ),
+            'rule' => 'numeric',
+            'required' => true,
+            'allowEmpty' => false,
         ),
         'orgc_id' => array(
-            'valueNotEmpty' => array(
-                'rule' => array('valueNotEmpty'),
-            ),
-            'numeric' => array(
-                    'rule' => array('numeric'),
-            ),
+            'rule' => 'numeric',
+            'required' => true,
+            'allowEmpty' => false,
         ),
         'date' => array(
             'date' => array(
@@ -134,21 +128,21 @@ class Event extends AppModel
                 'required' => true,
                 //'last' => false, // Stop validation after this rule
                 //'on' => 'create', // Limit validation to 'create' or 'update' operations
-                )
+            )
         ),
         'sharing_group_id' => array(
             'rule' => array('sharingGroupRequired'),
-                'message' => 'If the distribution is set to "Sharing Group", a sharing group has to be selected.',
-                //'required' => true,
-                //'allowEmpty' => true
+            'message' => 'If the distribution is set to "Sharing Group", a sharing group has to be selected.',
+            //'required' => true,
+            //'allowEmpty' => true
         ),
         'analysis' => array(
             'rule' => array('inList', array('0', '1', '2')),
-                'message' => 'Options : 0, 1, 2 (for Initial, Ongoing, Completed)',
-                //'allowEmpty' => false,
-                'required' => true,
-                //'last' => false, // Stop validation after this rule
-                //'on' => 'create', // Limit validation to 'create' or 'update' operations
+            'message' => 'Options : 0, 1, 2 (for Initial, Ongoing, Completed)',
+            //'allowEmpty' => false,
+            'required' => true,
+            //'last' => false, // Stop validation after this rule
+            //'on' => 'create', // Limit validation to 'create' or 'update' operations
         ),
         'info' => array(
             'valueNotEmpty' => array(

From 5c6a7a29397f0436a00a9d2d5e479eda7f85009b Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 17 Oct 2021 23:06:41 +0200
Subject: [PATCH 19/21] chg: [internal] Event::unpublishEvent method

---
 app/Model/Event.php | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/app/Model/Event.php b/app/Model/Event.php
index 4b671ee01..107927136 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -5846,19 +5846,21 @@ class Event extends AppModel
     {
         $event = $this->find('first', array(
             'recursive' => -1,
-            'conditions' => array('Event.id' => $id)
+            'conditions' => array('Event.id' => $id),
+            'fields' => ['id', 'info'], // info is required because of SysLogLogableBehavior
         ));
         if (empty($event)) {
             return false;
         }
+        $fields = ['published', 'timestamp'];
         $event['Event']['published'] = 0;
-        $date = new DateTime();
-        $event['Event']['timestamp'] = $date->getTimestamp();
+        $event['Event']['timestamp'] = time();
         if ($proposalLock) {
             $event['Event']['proposal_email_lock'] = 0;
+            $fields[] = 'proposal_email_lock';
         }
         $event['Event']['unpublishAction'] = true;
-        return $this->save($event);
+        return $this->save($event, true, $fields);
     }
 
     /**

From db5962a6800778466794ab915048281aead698b7 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 17 Oct 2021 23:26:26 +0200
Subject: [PATCH 20/21] chg: [internal] Update correlations just when necessary

---
 app/Model/Event.php | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/app/Model/Event.php b/app/Model/Event.php
index 107927136..d62abaddd 100755
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@@ -510,27 +510,28 @@ class Event extends AppModel
 
     public function afterSave($created, $options = array())
     {
+        $event = $this->data['Event'];
         if (!Configure::read('MISP.completely_disable_correlation') && !$created) {
             $updateCorrelation = [];
-            if (isset($this->data['Event']['distribution'])) {
-                $updateCorrelation['Correlation.distribution'] = (int)$this->data['Event']['distribution'];
+            if (isset($event['distribution']) && (empty($options['fieldList']) || in_array('distribution', $options['fieldList']))) {
+                $updateCorrelation['Correlation.distribution'] = (int)$event['distribution'];
             }
-            if (isset($this->data['Event']['sharing_group_id'])) {
-                $updateCorrelation['Correlation.sharing_group_id'] = (int)$this->data['Event']['sharing_group_id'];
+            if (isset($event['sharing_group_id']) && (empty($options['fieldList']) || in_array('sharing_group_id', $options['fieldList']))) {
+                $updateCorrelation['Correlation.sharing_group_id'] = (int)$event['sharing_group_id'];
             }
             if (!empty($updateCorrelation)) {
-                $this->Attribute->Correlation->updateAll($updateCorrelation, ['Correlation.event_id' => (int)$this->data['Event']['id']]);
+                $this->Attribute->Correlation->updateAll($updateCorrelation, ['Correlation.event_id' => (int)$event['id']]);
             }
         }
-        if (empty($this->data['Event']['unpublishAction']) && empty($this->data['Event']['skip_zmq']) && Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) {
+        if (empty($event['unpublishAction']) && empty($event['skip_zmq']) && Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) {
             $pubSubTool = $this->getPubSubTool();
-            $event = $this->quickFetchEvent($this->data['Event']['id']);
+            $eventForZmq = $this->quickFetchEvent($event['id']);
             if (!empty($event)) {
-                $pubSubTool->event_save($event, $created ? 'add' : 'edit');
+                $pubSubTool->event_save($eventForZmq, $created ? 'add' : 'edit');
             }
         }
-        if (empty($this->data['Event']['unpublishAction']) && empty($this->data['Event']['skip_kafka'])) {
-            $this->publishKafkaNotification('event', $this->quickFetchEvent($this->data['Event']['id']), $created ? 'add' : 'edit');
+        if (empty($event['unpublishAction']) && empty($event['skip_kafka'])) {
+            $this->publishKafkaNotification('event', $this->quickFetchEvent($event['id']), $created ? 'add' : 'edit');
         }
     }
 

From 9720a9772c046e5b855bb71f0abc5cc444d17499 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sun, 17 Oct 2021 23:37:40 +0200
Subject: [PATCH 21/21] chg: [auditlog] Optimise fetching old records

---
 app/Model/Behavior/AuditLogBehavior.php | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/app/Model/Behavior/AuditLogBehavior.php b/app/Model/Behavior/AuditLogBehavior.php
index 6e14c97d4..3b20a475c 100644
--- a/app/Model/Behavior/AuditLogBehavior.php
+++ b/app/Model/Behavior/AuditLogBehavior.php
@@ -81,11 +81,26 @@ class AuditLogBehavior extends ModelBehavior
         if (!$this->enabled) {
             return true;
         }
+
+        // Do not fetch old version when just few fields will be fetched
+        $fieldToFetch = [];
+        if (!empty($options['fieldList'])) {
+            foreach ($options['fieldList'] as $field) {
+                if (!isset($this->skipFields[$field])) {
+                    $fieldToFetch[] = $field;
+                }
+            }
+            if (empty($fieldToFetch))  {
+                $this->old = null;
+                return true;
+            }
+        }
         if ($model->id) {
             $this->old = $model->find('first', [
                 'conditions' => [$model->alias . '.' . $model->primaryKey => $model->id],
                 'recursive' => -1,
                 'callbacks' => false,
+                'fields' => $fieldToFetch,
             ]);
         } else {
             $this->old = null;