Merge branch 'develop' of github.com:MISP/MISP into develop

2022-08-24 09:54:46 +02:00 · 2022-08-24 09:54:46 +02:00 · f0913e1c52
parent d551d0533a 1693af4a61
commit f0913e1c52
40 changed files with 5799 additions and 5311 deletions
--- a/2
+++ b/2
@ -1 +1 @@
-Subproject commit 962b296f0c2de3366deaf69ce3484187f255f5e0
+Subproject commit 3ca8717e6c7780718cd20328040799261a39a281
--- a/VERSION.json
+++ b/VERSION.json
@ -1 +1 @@
-{"major":2, "minor":4, "hotfix":159}
+{"major":2, "minor":4, "hotfix":161}
--- a/app/Config/config.default.php
+++ b/app/Config/config.default.php
@ -71,6 +71,7 @@ $config = array(
        'enableOrgBlocklisting'          => true,
        'log_client_ip'                  => false,
        'log_auth'                       => false,
+        'store_api_access_time'          => false,
        'disableUserSelfManagement'      => false,
        'disable_user_login_change'      => false,
        'disable_user_password_change'   => false,
--- a/app/Console/Command/AdminShell.php
+++ b/app/Console/Command/AdminShell.php
@ -16,8 +16,13 @@ App::uses('JsonTool', 'Tools');
 */
 class AdminShell extends AppShell
 {
-    public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Allowedlist', 'Server', 'Organisation', 'AdminSetting', 'Galaxy', 'Taxonomy', 'Warninglist', 'Noticelist', 'ObjectTemplate', 'Bruteforce', 'Role', 'Feed', 'SharingGroupBlueprint', 'Correlation');
-    public $tasks = array('ConfigLoad');
+    public $uses = [
+        'Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Allowedlist', 'Server', 'Organisation', 
+        'AdminSetting', 'Galaxy', 'Taxonomy', 'Warninglist', 'Noticelist', 'ObjectTemplate', 'Bruteforce',
+        'Role', 'Feed', 'SharingGroupBlueprint', 'Correlation', 'OverCorrelatingValue'
+    ];
+
+    public $tasks = ['ConfigLoad'];
    
    public function getOptionParser()
    {
@ -109,6 +114,17 @@ class AdminShell extends AppShell
        $this->Attribute->generateCorrelation($jobId);
    }

+    public function jobGenerateOccurrences()
+    {
+        $this->ConfigLoad->execute();
+        if (empty($this->args[0])) {
+            die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Generate over-correlation occurrences'] . PHP_EOL);
+        }
+
+        $jobId = $this->args[0];
+        $this->OverCorrelatingValue->generateOccurrences($jobId);
+    }
+
    public function jobPurgeCorrelation()
    {
        if (empty($this->args[0])) {
--- a/app/Controller/AppController.php
+++ b/app/Controller/AppController.php
@ -435,6 +435,10 @@ class AppController extends Controller
                        );
                        $this->Log->save($log);
                    }
+                    $storeAPITime = Configure::read('MISP.store_api_access_time');
+                    if (!empty($storeAPITime) && $storeAPITime) {
+                        $this->User->updateAPIAccessTime($user);
+                    }
                    $this->Session->renew();
                    $this->Session->write(AuthComponent::$sessionKey, $user);
                    $this->isApiAuthed = true;
--- a/app/Controller/Component/ACLComponent.php
+++ b/app/Controller/Component/ACLComponent.php
@ -97,6 +97,7 @@ class ACLComponent extends Component
                'view' => []
            ],
            'correlations' => [
+                'generateOccurrences' => [],
                'generateTopCorrelations' => [],
                'overCorrelations' => [],
                'switchEngine' => [],
--- a/app/Controller/CorrelationsController.php
+++ b/app/Controller/CorrelationsController.php
@ -200,4 +200,22 @@ class CorrelationsController extends AppController
            $this->render('ajax/truncate_confirmation');
        }
    }
+
+    public function generateOccurrences()
+    {
+        $this->loadModel('OverCorrelatingValue');
+        $this->OverCorrelatingValue->generateOccurrencesRouter();
+        $message = __('Job queued.');
+        if (Configure::read('MISP.background_jobs')) {
+            $message = __('Job queued.');
+        } else {
+            $message = __('Over-correlations counted successfully.');
+        }
+        if (!$this->_isRest()) {
+            $this->Flash->info($message);
+            $this->redirect(['controller' => 'correlations', 'action' => 'overCorrelations']);
+        } else {
+            return $this->RestResponse->saveSuccessResponse('Correlations', 'generateOccurrences', false, $this->response->type(), $message);
+        }
+    }
 }
--- a/app/Controller/GalaxiesController.php
+++ b/app/Controller/GalaxiesController.php
@ -437,10 +437,11 @@ class GalaxiesController extends AppController
        )), 'GalaxyCluster');
        $synonyms = $this->Galaxy->GalaxyCluster->GalaxyElement->find('all', array(
            'conditions' => array(
-                'GalaxyElement.galaxy_cluster_id' => array_column($data, 'id'),
-                'GalaxyElement.key' => 'synonyms'
+                'GalaxyElement.key' => 'synonyms',
+                $conditions
            ),
            'fields' => ['GalaxyElement.galaxy_cluster_id', 'GalaxyElement.value'],
+            'contain' => 'GalaxyCluster',
            'recursive' => -1
        ));
        $sortedSynonyms = array();
--- a/app/Controller/UsersController.php
+++ b/app/Controller/UsersController.php
@ -445,6 +445,7 @@ class UsersController extends AppController
            'expiration',
            'current_login',
            'last_login',
+            'last_api_access',
            'force_logout',
            'date_created',
            'date_modified'
--- a/app/Lib/EventWarning/DefaultWarning.php
+++ b/app/Lib/EventWarning/DefaultWarning.php
@ -63,7 +63,7 @@ class DefaultWarning
    {
        $lowerTagName = trim(strtolower($tagName));
        if (substr($lowerTagName, 0, 4) === 'tlp:') {
-            if (!in_array($lowerTagName, ['tlp:white', 'tlp:green', 'tlp:amber', 'tlp:red', 'tlp:ex:chr'], true)) {
+            if (!in_array($lowerTagName, ['tlp:white', 'tlp:green', 'tlp:amber', 'tlp:red', 'tlp:ex:chr', 'tlp:clear', 'tlp:amber+strict'], true)) {
                $warnings['TLP'][] = __('Unknown TLP tag, please refer to the TLP taxonomy as to what is valid, otherwise filtering rules created by your partners may miss your intent.');
            } else if ($lowerTagName !== $tagName) {
                $warnings['TLP'][] = __('TLP tag with invalid formatting: Make sure that you only use TLP tags from the taxonomy. Custom tags with invalid capitalisation, white spaces or other artifacts will break synchronisation and filtering rules intended for the correct taxonomy derived tags.');
--- a/app/Lib/Export/NidsExport.php
+++ b/app/Lib/Export/NidsExport.php
@ -204,6 +204,7 @@ class NidsExport
                        break;
                    case 'email':
                        $this->emailSrcRule($ruleFormat, $item['Attribute'], $sid);
+			$sid++;
                        $this->emailDstRule($ruleFormat, $item['Attribute'], $sid);
                        break;
                    case 'email-src':
@ -868,4 +869,4 @@ class NidsExport
        }
        return $ipport;
    }
-}
+}
--- a/app/Model/AppModel.php
+++ b/app/Model/AppModel.php
@ -82,7 +82,8 @@ class AppModel extends Model
        69 => false, 70 => false, 71 => true, 72 => true, 73 => false, 74 => false,
        75 => false, 76 => true, 77 => false, 78 => false, 79 => false, 80 => false,
        81 => false, 82 => false, 83 => false, 84 => false, 85 => false, 86 => false,
-        87 => false, 88 => false, 89 => false, 90 => false,
+        87 => false, 88 => false, 89 => false, 90 => false, 91 => false, 92 => false,
+        93 => false,
    );

    const ADVANCED_UPDATES_DESCRIPTION = array(
@ -237,6 +238,29 @@ class AppModel extends Model
                $this->Workflow = Classregistry::init('Workflow');
                $this->Workflow->enableDefaultModules();
                break;
+            case 91:
+                $existing_index = $this->query(
+                    "SHOW INDEX FROM default_correlations WHERE Key_name = 'unique_correlation';"
+                );
+                if (empty($existing_index)) {
+                    $this->query(
+                        "ALTER TABLE default_correlations
+                        ADD CONSTRAINT unique_correlation
+                        UNIQUE KEY(attribute_id, 1_attribute_id, value_id);"
+                    );
+                }
+                $existing_index = $this->query(
+                    "SHOW INDEX FROM no_acl_correlations WHERE Key_name = 'unique_correlation';"
+                );
+                if (empty($existing_index)) {
+                    $this->query(
+                        "ALTER TABLE no_acl_correlations
+                        ADD CONSTRAINT unique_correlation
+                        UNIQUE KEY(attribute_id, 1_attribute_id, value_id);"
+                    );
+                }
+                $dbUpdateSuccess = true;
+                break;
            default:
                $dbUpdateSuccess = $this->updateDatabase($command);
                break;
@ -1808,6 +1832,25 @@ class AppModel extends Model
                      INDEX `timestamp` (`timestamp`)
                    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;";
                    break;
+            case 92:
+                $sqlArray[] = "ALTER TABLE users ADD `last_api_access` INT(11) DEFAULT 0;";
+                break;
+            case 93:
+                $this->__dropIndex('default_correlations', 'distribution');
+                $this->__dropIndex('default_correlations', 'object_distribution');
+                $this->__dropIndex('default_correlations', 'event_distribution');
+                $this->__dropIndex('default_correlations', 'sharing_group_id');
+                $this->__dropIndex('default_correlations', 'object_sharing_group_id');
+                $this->__dropIndex('default_correlations', 'event_sharing_group_id');
+                $this->__dropIndex('default_correlations', 'org_id');
+                $this->__dropIndex('default_correlations', '1_distribution');
+                $this->__dropIndex('default_correlations', '1_object_distribution');
+                $this->__dropIndex('default_correlations', '1_event_distribution');
+                $this->__dropIndex('default_correlations', '1_sharing_group_id');
+                $this->__dropIndex('default_correlations', '1_object_sharing_group_id');
+                $this->__dropIndex('default_correlations', '1_event_sharing_group_id');
+                $this->__dropIndex('default_correlations', '1_org_id');
+                break;
            case 'fixNonEmptySharingGroupID':
                $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
                $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
--- a/app/Model/Attribute.php
+++ b/app/Model/Attribute.php
@ -897,7 +897,8 @@ class Attribute extends AppModel
        $defaultMaxSize = $outputFormat === 'webp' ? 400 : 200;
        $maxWidth = $maxWidth ?: $defaultMaxSize;
        $maxHeight = $maxHeight ?: $defaultMaxSize;
-
+        $suffix = null;
+        
        if ($maxWidth == $defaultMaxSize && $maxHeight == $defaultMaxSize) {
            $thumbnailInRedis = Configure::read('MISP.thumbnail_in_redis');
            if ($thumbnailInRedis) {
@ -1522,6 +1523,8 @@ class Attribute extends AppModel
        $this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep');
        $this->FuzzyCorrelateSsdeep->purge($eventId, $attributeId);

+        $this->query('TRUNCATE TABLE over_correlating_values');
+
        // get all attributes..
        if (!$eventId) {
            $eventIds = $this->Event->find('column', [
@ -1601,9 +1604,6 @@ class Attribute extends AppModel
            $attributes = $this->find('all', $query);
            foreach ($attributes as $attribute) {
                $attribute['Attribute']['event_id'] = $eventId;
-                if ($full) {
-                    $this->Correlation->beforeSaveCorrelation($attribute['Attribute']);
-                }
                $this->Correlation->afterSaveCorrelation($attribute['Attribute'], $full);
            }
            $fetchedAttributes = count($attributes);
@ -3143,6 +3143,8 @@ class Attribute extends AppModel
                'Attribute' => array(
                    'sharinggroup' => array('function' => 'set_filter_sharing_group'),
                    'value' => array('function' => 'set_filter_value'),
+                    'value1' => array('function' => 'set_filter_simple_attribute'),
+                    'value2' => array('function' => 'set_filter_simple_attribute'),
                    'category' => array('function' => 'set_filter_simple_attribute'),
                    'type' => array('function' => 'set_filter_type'),
                    'object_relation' => array('function' => 'set_filter_simple_attribute'),
--- a/app/Model/Behavior/DefaultCorrelationBehavior.php
+++ b/app/Model/Behavior/DefaultCorrelationBehavior.php
@ -1,17 +1,14 @@
 <?php
-
 App::uses('AppModel', 'Model');
-App::uses('RandomTool', 'Tools');

 /**
 * Default correlation behaviour
 */
 class DefaultCorrelationBehavior extends ModelBehavior
 {
+    const TABLE_NAME = 'default_correlations';

-    private $__tableName = 'default_correlations';
-
-    private $__config = [
+    const CONFIG = [
        'AttributeFetcher' => [
            'fields' =>  [
                'Attribute.event_id',
@ -44,26 +41,29 @@ class DefaultCorrelationBehavior extends ModelBehavior
        ]
    ];

-    public $Correlation = null;
+    /** @var Correlation */
+    public $Correlation;

    private $deadlockAvoidance = false;

-    public function setup(Model $Model, $settings = []) {
-        $Model->useTable = $this->__tableName;
+    public function setup(Model $Model, $settings = [])
+    {
+        $Model->useTable = self::TABLE_NAME;
        $this->Correlation = $Model;
        $this->deadlockAvoidance = $settings['deadlockAvoidance'];
    }

    public function getTableName(Model $Model)
    {
-        return $this->__tableName;
+        return self::TABLE_NAME;
    }

-    public function createCorrelationEntry(Model $Model, $value, $a, $b) {
-        $value_id = $this->Correlation->CorrelationValue->getValueId($value);
+    public function createCorrelationEntry(Model $Model, $value, $a, $b)
+    {
+        $valueId = $this->Correlation->CorrelationValue->getValueId($value);
        if ($this->deadlockAvoidance) {
            return [
-                'value_id' => $value_id,
+                'value_id' => $valueId,
                '1_event_id' => $a['Event']['id'],
                '1_object_id' => $a['Attribute']['object_id'],
                '1_attribute_id' => $a['Attribute']['id'],
@ -87,7 +87,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
            ];
        } else {
            return [
-                (int) $value_id,
+                (int) $valueId,
                (int) $a['Event']['id'],
                (int) $a['Attribute']['object_id'],
                (int) $a['Attribute']['id'],
@ -160,7 +160,8 @@ class DefaultCorrelationBehavior extends ModelBehavior
        }
    }

-    public function runBeforeSaveCorrelation(Model $Model, $attribute) {
+    public function runBeforeSaveCorrelation(Model $Model, $attribute)
+    {
        // (update-only) clean up the relation of the old value: remove the existing relations related to that attribute, we DO have a reference, the id
        // ==> DELETE FROM default_correlations WHERE 1_attribute_id = $a_id OR attribute_id = $a_id; */
        // first check if it's an update
@ -181,20 +182,28 @@ class DefaultCorrelationBehavior extends ModelBehavior
    public function getContainRules(Model $Model, $filter = null)
    {
        if (empty($filter)) {
-            return $this->__config['AttributeFetcher']['contain'];
+            return self::CONFIG['AttributeFetcher']['contain'];
        } else {
-            return empty($this->__config['AttributeFetcher']['contain'][$filter]) ? false : $this->__config['AttributeFetcher']['contain'][$filter];
+            return empty(self::CONFIG['AttributeFetcher']['contain'][$filter]) ? false : self::CONFIG['AttributeFetcher']['contain'][$filter];
        }
    }

    public function getFieldRules(Model $Model)
    {
-        return $this->__config['AttributeFetcher']['fields'];
+        return self::CONFIG['AttributeFetcher']['fields'];
    }

-    private function __collectCorrelations($user, $id, $sgids, $primary)
+    /**
+     * Fetch correlations for given event.
+     * @param array $user
+     * @param int $eventId
+     * @param array $sgids
+     * @param bool $primary
+     * @return array
+     */
+    private function __collectCorrelations(array $user, $eventId, $sgids, $primary)
    {
-        $max_correlations = Configure::read('MISP.max_correlations_per_event') ?: 5000;
+        $maxCorrelations = Configure::read('MISP.max_correlations_per_event') ?: 5000;
        $source = $primary ? '' : '1_';
        $prefix = $primary ? '1_' : '';
        $correlations = $this->Correlation->find('all', array(
@ -213,7 +222,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
            ],
            'conditions' => [
                'OR' => [
-                    $source . 'event_id' => $id
+                    $source . 'event_id' => $eventId
                ],
                'AND' => [
                    [
@ -234,46 +243,50 @@ class DefaultCorrelationBehavior extends ModelBehavior
                ]
            ],
            'order' => false,
-            'limit' => $max_correlations
+            'limit' => $maxCorrelations
        ));
-        foreach ($correlations as $k => &$correlation) {
+        foreach ($correlations as $k => $correlation) {
            if (!$this->checkCorrelationACL($user, $correlation['Correlation'], $sgids, $prefix)) {
                unset($correlations[$k]);
            }
        }
-        $correlations = array_values($correlations);
        return $correlations;
    }

+    /**
+     * @param Correlation $Model
+     * @param array $user
+     * @param int $id Event ID
+     * @param array $sgids
+     * @return array
+     */
    public function runGetAttributesRelatedToEvent(Model $Model, $user, $id, $sgids)
    {
-        $temp_correlations = $this->__collectCorrelations($user, $id, $sgids, false);
-        $temp_correlations_1 = $this->__collectCorrelations($user, $id, $sgids, true);
        $correlations = [];
-        $event_ids = [];
-        foreach ($temp_correlations as $temp_correlation) {
+        $eventIds = [];
+        foreach ($this->__collectCorrelations($user, $id, $sgids, false) as $correlation) {
            $correlations[] = [
-                'id' => $temp_correlation['Correlation']['event_id'],
-                'attribute_id' => $temp_correlation['Correlation']['attribute_id'],
-                'parent_id' => $temp_correlation['Correlation']['1_attribute_id'],
-                'value' => $temp_correlation['CorrelationValue']['value']
+                'id' => $correlation['Correlation']['event_id'],
+                'attribute_id' => $correlation['Correlation']['attribute_id'],
+                'parent_id' => $correlation['Correlation']['1_attribute_id'],
+                'value' => $correlation['CorrelationValue']['value']
            ];
-            $event_ids[$temp_correlation['Correlation']['event_id']] = true;
+            $eventIds[$correlation['Correlation']['event_id']] = true;
        }
-        foreach ($temp_correlations_1 as $temp_correlation) {
+        foreach ($this->__collectCorrelations($user, $id, $sgids, true) as $correlation) {
            $correlations[] = [
-                'id' => $temp_correlation['Correlation']['1_event_id'],
-                'attribute_id' => $temp_correlation['Correlation']['1_attribute_id'],
-                'parent_id' => $temp_correlation['Correlation']['attribute_id'],
-                'value' => $temp_correlation['CorrelationValue']['value']
+                'id' => $correlation['Correlation']['1_event_id'],
+                'attribute_id' => $correlation['Correlation']['1_attribute_id'],
+                'parent_id' => $correlation['Correlation']['attribute_id'],
+                'value' => $correlation['CorrelationValue']['value']
            ];
-            $event_ids[$temp_correlation['Correlation']['1_event_id']] = true;
+            $eventIds[$correlation['Correlation']['1_event_id']] = true;
        }
        if (empty($correlations)) {
            return [];
        }
        $conditions = $Model->Event->createEventConditions($user);
-        $conditions['Event.id'] = array_keys($event_ids);
+        $conditions['Event.id'] = array_keys($eventIds);
        $events = $Model->Event->find('all', [
            'recursive' => -1,
            'conditions' => $conditions,
@ -288,9 +301,9 @@ class DefaultCorrelationBehavior extends ModelBehavior
                continue;
            }
            $event = $events[$eventId];
-            $correlation['org_id'] = $events[$eventId]['orgc_id'];
-            $correlation['info'] = $events[$eventId]['info'];
-            $correlation['date'] = $events[$eventId]['date'];
+            $correlation['org_id'] = $event['orgc_id'];
+            $correlation['info'] = $event['info'];
+            $correlation['date'] = $event['date'];
            $parentId = $correlation['parent_id'];
            unset($correlation['parent_id']);
            $relatedAttributes[$parentId][] = $correlation;
@ -388,7 +401,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
        ]);
        if (!empty($includeEventData)) {
            $results = [];
-            foreach ($relatedAttributes as $k => $attribute) {
+            foreach ($relatedAttributes as $attribute) {
                $temp = $attribute['Attribute'];
                $temp['Event'] = $attribute['Event'];
                $results[] = $temp;
@ -410,63 +423,81 @@ class DefaultCorrelationBehavior extends ModelBehavior
        //        ii. Event has a sharing group that the user is accessible to view
        //    b.  Attribute:
        //        i. Attribute has a distribution of 5 (inheritance of the event, for this the event check has to pass anyway)
-        //        ii. Atttibute has a distribution between 1-3 (community only, connected communities, all orgs)
+        //        ii. Attribute has a distribution between 1-3 (community only, connected communities, all orgs)
        //        iii. Attribute has a sharing group that the user is accessible to view
        $primaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, $sgids, true);
        $secondaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, $sgids, false);
-        return array_unique(array_merge($primaryEventIds,$secondaryEventIds));
-
+        return array_unique(array_merge($primaryEventIds,$secondaryEventIds), SORT_REGULAR);
    }

+    /**
+     * @param Model $Model
+     * @param array $user
+     * @param int $eventId
+     * @param array $sgids
+     * @param bool $primary
+     * @return array|int[]
+     */
    private function __filterRelatedEvents(Model $Model, array $user, int $eventId, array $sgids, bool $primary)
    {
        $current = $primary ? '' : '1_';
        $prefix = $primary ? '1_' : '';
-        $correlations = $Model->find('all', [
-            'recursive' => -1,
-            'fields' => [
-                $prefix . 'org_id',
-                $prefix . 'event_id',
-                $prefix . 'event_distribution',
-                $prefix . 'event_sharing_group_id',
-                $prefix . 'object_id',
-                $prefix . 'object_distribution',
-                $prefix . 'object_sharing_group_id',
-                $prefix . 'distribution',
-                $prefix . 'sharing_group_id'
-            ],
+
+        if (empty($user['Role']['perm_site_admin'])) {
+            $correlations = $Model->find('all', [
+                'recursive' => -1,
+                'fields' => [
+                    $prefix . 'org_id',
+                    $prefix . 'event_id',
+                    $prefix . 'event_distribution',
+                    $prefix . 'event_sharing_group_id',
+                    $prefix . 'object_id',
+                    $prefix . 'object_distribution',
+                    $prefix . 'object_sharing_group_id',
+                    $prefix . 'distribution',
+                    $prefix . 'sharing_group_id'
+                ],
+                'conditions' => [
+                    $current . 'event_id' => $eventId
+                ],
+            ]);
+
+            $eventIds = [];
+            foreach ($correlations as $correlation) {
+                $correlation = $correlation['Correlation'];
+                // if we have already added this event as a valid target, no need to check again.
+                if (isset($eventIds[$correlation[$prefix . 'event_id']])) {
+                    continue;
+                }
+                if ($this->checkCorrelationACL($user, $correlation, $sgids, $prefix)) {
+                    $eventIds[$correlation[$prefix . 'event_id']] = true;
+                }
+            }
+            return array_keys($eventIds);
+        }
+
+        return $Model->find('column', [
+            'fields' => [$prefix . 'event_id'],
            'conditions' => [
                $current . 'event_id' => $eventId
            ],
            'unique' => true,
        ]);
-        $eventIds = [];
-        if (empty($user['Role']['perm_site_admin'])) {
-            foreach ($correlations as $k => $correlation) {
-                // if we have already added this event as a valid target, no need to check again.
-                if (isset($eventIds[$correlation['Correlation'][$prefix . 'event_id']])) {
-                    continue;
-                }
-                $correlation = $correlation['Correlation'];
-                if (!$this->checkCorrelationACL($user, $correlation, $sgids, $prefix)) {
-                    unset($correlations[$k]);
-                    continue;
-                }
-                $eventIds[$correlation[$prefix . 'event_id']] = true;
-            }
-            return array_keys($eventIds);
-        } else {
-            $eventIds = Hash::extract($correlations, '{n}.Correlation.' . $prefix . 'event_id');
-            return $eventIds;
-        }
    }

-    private function checkCorrelationACL($user, $correlation, $sgids, $prefix)
+    /**
+     * @param array $user
+     * @param array $correlation
+     * @param array $sgids
+     * @param string $prefix
+     * @return bool
+     */
+    private function checkCorrelationACL(array $user, $correlation, $sgids, $prefix)
    {
        if ($user['Role']['perm_site_admin']) {
            return true;
        }
-        // check if user can see the event
+        // Check if user can see the event
        if (isset($correlation['Correlation'])) {
            $correlation = $correlation['Correlation'];
        }
@ -483,7 +514,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
            return false;
        }

-        //check if the user can see the object, if we're looking at an object attribute
+        // Check if the user can see the object, if we're looking at an object attribute
        if (
            $correlation[$prefix . 'object_id'] &&
            (
@ -498,7 +529,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
            return false;
        }

-        //check if the user can see the attribute
+        // Check if the user can see the attribute
        if (
            (
                $correlation[$prefix . 'distribution'] == 0 ||
@ -518,7 +549,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
        array $data,
        string $type = 'event',
        array $options = []
-    ): bool
+    )
    {
        $updateCorrelation = [];
        $updateFields = [
@ -550,14 +581,15 @@ class DefaultCorrelationBehavior extends ModelBehavior
                $Model->updateAll(
                    $side,
                    [
-                        $updateFields[$k] => (int)$data['id']]
+                        $updateFields[$k] => (int)$data['id']
+                    ]
                );
            }
        }
        return true;
    }

-    public function purgeCorrelations(Model $Model, $eventId = null): void
+    public function purgeCorrelations(Model $Model, $eventId = null)
    {
        if (!$eventId) {
            $Model->query('TRUNCATE TABLE default_correlations;');
@ -573,7 +605,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
        }
    }

-    public function purgeByValue(Model $Model, string $value): void
+    public function purgeByValue(Model $Model, string $value)
    {
        $valueIds = $Model->CorrelationValue->find('column', [
            'recursive' => -1,
--- a/app/Model/Behavior/NoAclCorrelationBehavior.php
+++ b/app/Model/Behavior/NoAclCorrelationBehavior.php
@ -352,7 +352,7 @@ class NoAclCorrelationBehavior extends ModelBehavior
        return true;
    }

-    public function purgeCorrelations(Model $Model, $eventId = null): void
+    public function purgeCorrelations(Model $Model, $eventId = null)
    {
        if (!$eventId) {
            $Model->query('TRUNCATE TABLE no_acl_correlations;');
@ -368,7 +368,7 @@ class NoAclCorrelationBehavior extends ModelBehavior
        }
    }

-    public function purgeByValue(Model $Model, string $value): void
+    public function purgeByValue(Model $Model, string $value)
    {
        $valueIds = $Model->CorrelationValue->find('column', [
            'recursive' => -1,
--- a/app/Model/Correlation.php
+++ b/app/Model/Correlation.php
@ -1,9 +1,15 @@
 <?php
 App::uses('AppModel', 'Model');
-App::uses('RandomTool', 'Tools');

 /**
 * @property Attribute $Attribute
+ * @property Event $Event
+ * @property CorrelationValue $CorrelationValue
+ * @method saveCorrelations(array $correlations)
+ * @method runBeforeSaveCorrelation
+ * @method fetchRelatedEventIds(array $user, int $eventId, array $sgids)
+ * @method getFieldRules
+ * @method getContainRules($filter = null)
 */
 class Correlation extends AppModel
 {
@ -44,36 +50,27 @@ class Correlation extends AppModel
    /** @var array */
    private $exclusions;

-    /**
-     * Use old schema with `date` and `info` fields.
-     * @var bool
-     */
-    private $oldSchema;
-
-    /** @var bool */
-    private $deadlockAvoidance;
-
    /** @var bool */
    private $advancedCorrelationEnabled;

    /** @var array */
    private $cidrListCache;

-    private $__correlationEngine = 'DefaultCorrelation';
-
-    protected $_config = [];
+    /** @var string */
+    private $__correlationEngine;

    private $__tempContainCache = [];

-    public $OverCorrelatingValue = null;
+    /** @var OverCorrelatingValue */
+    public $OverCorrelatingValue;

    public function __construct($id = false, $table = null, $ds = null)
    {
        parent::__construct($id, $table, $ds);
        $this->__correlationEngine = $this->getCorrelationModelName();
-        $this->deadlockAvoidance = Configure::check('MISP.deadlock_avoidance') ? Configure::read('MISP.deadlock_avoidance') : false;
+        $deadlockAvoidance = Configure::check('MISP.deadlock_avoidance') ? Configure::read('MISP.deadlock_avoidance') : false;
        // load the currently used correlation engine
-        $this->Behaviors->load($this->__correlationEngine . 'Correlation', ['deadlockAvoidance' => false]);
+        $this->Behaviors->load($this->__correlationEngine . 'Correlation', ['deadlockAvoidance' => $deadlockAvoidance]);
        // getTableName() needs to be implemented by the engine - this points us to the table to be used
        $this->useTable = $this->getTableName();
        $this->advancedCorrelationEnabled = (bool)Configure::read('MISP.enable_advanced_correlations');
@ -247,16 +244,21 @@ class Correlation extends AppModel
     */
    private function __saveCorrelations(array $correlations)
    {
-        return $this->saveCorrelations($correlations);
+        try {
+            return $this->saveCorrelations($correlations);
+        } catch (Exception $e) {
+            // Correlations may fail for different reasons, such as the correlation already existing. We don't care and don't want to break the process
+            return true;
+        }
    }

-    public function correlateAttribute(array $attribute): void
+    public function correlateAttribute(array $attribute)
    {
        $this->runBeforeSaveCorrelation($attribute);
        $this->afterSaveCorrelation($attribute);
    }

-    public function beforeSaveCorrelation(array $attribute): void
+    public function beforeSaveCorrelation(array $attribute)
    {
        $this->runBeforeSaveCorrelation($attribute);
    }
@ -285,6 +287,7 @@ class Correlation extends AppModel
     * @param bool $full
     * @param array|false $event
     * @return array|bool|bool[]|mixed
+     * @throws Exception
     */
    public function afterSaveCorrelation($a, $full = false, $event = false)
    {
@ -331,7 +334,7 @@ class Correlation extends AppModel
            return true;
        }
        $correlations = [];
-        foreach ($correlatingValues as $k => $cV) {
+        foreach ($correlatingValues as $cV) {
            if ($cV === null) {
                continue;
            }
@ -342,6 +345,7 @@ class Correlation extends AppModel
                        'Attribute.value2' => $cV,
                        'NOT' => ['Attribute.type' => Attribute::PRIMARY_ONLY_CORRELATING_TYPES]
                    ],
+                    $extraConditions,
                ],
                'NOT' => [
                    'Attribute.event_id' => $a['Attribute']['event_id'],
@ -368,13 +372,17 @@ class Correlation extends AppModel
            $count = count($correlatingAttributes);
            if ($count > $correlationLimit) {
                // If we have more correlations for the value than the limit, set the block entry and stop the correlation process
-                $this->OverCorrelatingValue->block($cV, $count);
+                $this->OverCorrelatingValue->block($cV);
                return true;
            } else {
                // If we have fewer hits than the limit, proceed with the correlation, but first make sure we remove any existing blockers
                $this->OverCorrelatingValue->unblock($cV);
            }
            foreach ($correlatingAttributes as $b) {
+                // On a full correlation, only correlate with attributes that have a higher ID to avoid duplicate correlations
+                if ($full && $b['Attribute']['id'] < $b['Attribute']['id']) {
+                    continue;
+                }
                if (isset($b['Attribute']['value1'])) {
                    // TODO: Currently it is hard to check if value1 or value2 correlated, so we check value2 and if not, it is value1
                    $value = $cV === $b['Attribute']['value2'] ? $b['Attribute']['value2'] : $b['Attribute']['value1'];
@ -687,7 +695,7 @@ class Correlation extends AppModel
        return true;
    }

-    public function findTop(array $query): array
+    public function findTop(array $query)
    {
        try {
            $redis = $this->setupRedisWithException();
@ -742,7 +750,7 @@ class Correlation extends AppModel
     * Get list of all CIDR for correlation from database
     * @return array
     */
-    private function getCidrListFromDatabase(): array
+    private function getCidrListFromDatabase()
    {
        return $this->Attribute->find('column', [
            'conditions' => [
@ -760,7 +768,7 @@ class Correlation extends AppModel
    /**
     * @return array
     */
-    public function updateCidrList(): array
+    public function updateCidrList()
    {
        $redis = $this->setupRedisWithException();
        $cidrList = [];
@ -785,7 +793,7 @@ class Correlation extends AppModel
    /**
     * @return void
     */
-    public function clearCidrCache(): void
+    public function clearCidrCache()
    {
        $this->cidrListCache = null;
    }
@ -793,7 +801,7 @@ class Correlation extends AppModel
    /**
     * @return array
     */
-    public function getCidrList(): array
+    public function getCidrList()
    {
        if ($this->cidrListCache !== null) {
            return $this->cidrListCache;
@ -819,7 +827,7 @@ class Correlation extends AppModel
     * @param array $sgids List of sharing group IDs
     * @return array
     */
-    public function getAttributesRelatedToEvent(array $user, $eventIds, array $sgids): array
+    public function getAttributesRelatedToEvent(array $user, $eventIds, array $sgids)
    {
        return $this->runGetAttributesRelatedToEvent($user, $eventIds, $sgids);
    }
@ -832,7 +840,7 @@ class Correlation extends AppModel
     * @param bool $includeEventData Flag to include the event data in the response
     * @return array
     */
-    public function getRelatedAttributes($user, $sgids, $attribute, $fields=[], $includeEventData = false): array
+    public function getRelatedAttributes($user, $sgids, $attribute, $fields=[], $includeEventData = false)
    {
        if (in_array($attribute['type'], Attribute::NON_CORRELATING_TYPES)) {
            return [];
@ -842,11 +850,11 @@ class Correlation extends AppModel
    
    /**
     * @param array $user User array
-     * @param int $eventIds List of event IDs
+     * @param int $eventId List of event IDs
     * @param array $sgids List of sharing group IDs
     * @return array
     */
-    public function getRelatedEventIds(array $user, int $eventId, array $sgids): array
+    public function getRelatedEventIds(array $user, int $eventId, array $sgids)
    {
        $relatedEventIds = $this->fetchRelatedEventIds($user, $eventId, $sgids);
        if (empty($relatedEventIds)) {
@ -863,28 +871,55 @@ class Correlation extends AppModel
        return $data;
    }

-    public function setCorrelationExclusion($attribute)
+    /**
+     * @param array $attributes
+     * @return array
+     */
+    public function attachCorrelationExclusion(array $attributes)
    {
-        if (empty($this->__compositeTypes)) {
+        if (!isset($this->__compositeTypes)) {
            $this->__compositeTypes = $this->Attribute->getCompositeTypes();
        }
-        $values = [$attribute['value']];
-        if (in_array($attribute['type'], $this->__compositeTypes)) {
-            $values = explode('|', $attribute['value']);
+
+        $valuesToCheck = [];
+        foreach ($attributes as &$attribute) {
+            if (in_array($attribute['type'], $this->__compositeTypes, true)) {
+                $values = explode('|', $attribute['value']);
+                $valuesToCheck[$values[0]] = true;
+                $valuesToCheck[$values[1]] = true;
+            } else {
+                $values = [$attribute['value']];
+                $valuesToCheck[$values[0]] = true;
+            }
+
+            if ($this->__preventExcludedCorrelations($values[0])) {
+                $attribute['correlation_exclusion'] = true;
+            } elseif (!empty($values[1]) && $this->__preventExcludedCorrelations($values[1])) {
+                $attribute['correlation_exclusion'] = true;
+            }
        }
-        if ($this->__preventExcludedCorrelations($values[0])) {
-            $attribute['correlation_exclusion'] = true;
+
+        $overCorrelatingValues = array_flip($this->OverCorrelatingValue->find('column', [
+            'conditions' => ['value' => array_keys($valuesToCheck)],
+            'fields' => ['value'],
+        ]));
+        unset($valuesToCheck);
+
+        foreach ($attributes as &$attribute) {
+            if (in_array($attribute['type'], $this->__compositeTypes, true)) {
+                $values = explode('|', $attribute['value']);
+            } else {
+                $values = [$attribute['value']];
+            }
+
+            if (isset($overCorrelatingValues[$values[0]])) {
+                $attribute['over_correlation'] = true;
+            } elseif (!empty($values[1]) && isset($overCorrelatingValues[$values[1]])) {
+                $attribute['over_correlation'] = true;
+            }
        }
-        if (!empty($values[1]) && $this->__preventExcludedCorrelations($values[1])) {
-            $attribute['correlation_exclusion'] = true;
-        }
-        if ($this->OverCorrelatingValue->checkValue($values[0])) {
-            $attribute['over_correlation'] = true;
-        }
-        if (!empty($values[1]) && $this->OverCorrelatingValue->checkValue($values[1])) {
-            $attribute['over_correlation'] = true;
-        }
-        return $attribute;
+
+        return $attributes;
    }

    public function collectMetrics()
--- a/app/Model/CorrelationValue.php
+++ b/app/Model/CorrelationValue.php
@ -1,24 +1,21 @@
 <?php
 App::uses('AppModel', 'Model');
-App::uses('RandomTool', 'Tools');

 class CorrelationValue extends AppModel
 {
    public $recursive = -1;

-    public $actsAs = array(
-        'Containable'
-    );
-
-    public $validate = [
-    ];
-
+    /**
+     * @param string $value
+     * @return int
+     */
    public function getValueId($value)
    {
        // index is 191 long, missing the existing value lookup can lead to a duplicate entry
        $value = mb_substr($value, 0, 191);
        $existingValue = $this->find('first', [
            'recursive' => -1,
+            'fields' => ['id'],
            'conditions' => [
                'value' => $value
            ]
@ -31,22 +28,23 @@ class CorrelationValue extends AppModel
            } catch (Exception $e) {
                $existingValue = $this->find('first', [
                    'recursive' => -1,
+                    'fields' => ['id'],
                    'conditions' => [
                        'value' => $value
                    ]
                ]);
-                return $existingValue['ExistingValue']['id'];
+                return $existingValue['CorrelationValue']['id'];
            }
        } else {
            return $existingValue['CorrelationValue']['id'];
        }
-        return false;
    }

    public function getValue($id)
    {
        $existingValue = $this->find('first', [
            'recursive' => -1,
+            'fields' => ['value'],
            'conditions' => [
                'id' => $id
            ]
--- a/app/Model/Event.php
+++ b/app/Model/Event.php
@ -44,6 +44,8 @@ class Event extends AppModel

    public $mispVersion = '2.4.0';

+    private $__beforeSaveData = null;
+
    public $fieldDescriptions = array(
        'threat_level_id' => array('desc' => 'Risk levels: *low* means mass-malware, *medium* means APT malware, *high* means sophisticated APT malware or 0-day attack', 'formdesc' => 'Risk levels: low: mass-malware medium: APT malware high: sophisticated APT malware or 0-day attack'),
        'classification' => array('desc' => 'Set the Traffic Light Protocol classification. <ol><li><em>TLP:AMBER</em>- Share only within the organization on a need-to-know basis</li><li><em>TLP:GREEN:NeedToKnow</em>- Share within your constituency on the need-to-know basis.</li><li><em>TLP:GREEN</em>- Share within your constituency.</li></ol>'),
@ -431,6 +433,7 @@ class Event extends AppModel
        if (empty($this->data['Event']['uuid'])) {
            $this->data['Event']['uuid'] = CakeText::uuid();
        }
+        $this->__beforeSaveData = $this->data['Event'];
        return true;
    }

@ -438,8 +441,21 @@ class Event extends AppModel
    {
        $event = $this->data['Event'];
        if (!Configure::read('MISP.completely_disable_correlation') && !$created) {
-            $this->Attribute->Correlation->updateContainedCorrelations($event, 'event');
+            if (
+                empty($this->__beforeSaveData) ||
+                (
+                    isset($this->__beforeSaveData['distribution']) &&
+                    $event['distribution'] != $this->__beforeSaveData['distribution']
+                ) ||
+                (
+                    isset($this->__beforeSaveData['sharing_group_id']) &&
+                    $event['sharing_group_id'] != $this->__beforeSaveData['sharing_group_id']
+                )
+            ) {
+                $this->Attribute->Correlation->updateContainedCorrelations($event, 'event');
+            }
        }
+        $this->__beforeSaveData = null;
        if (empty($event['unpublishAction']) && empty($event['skip_zmq']) && $this->pubToZmq('event')) {
            $pubSubTool = $this->getPubSubTool();
            $eventForZmq = $this->quickFetchEvent($event['id']);
@ -621,7 +637,7 @@ class Event extends AppModel
        return $events;
    }

-    public function getRelatedEventCount($user, $eventId, $sgids)
+    public function getRelatedEventCount(array $user, $eventId, $sgids)
    {
        if (!isset($sgids) || empty($sgids)) {
            $sgids = array(-1);
@ -2031,6 +2047,8 @@ class Event extends AppModel
                    $event['Attribute'] = $this->__attachSharingGroups($event['Attribute'], $sharingGroupData);
                }

+                $event['Attribute'] = $this->Attribute->Correlation->attachCorrelationExclusion($event['Attribute']);
+
                // move all object attributes to a temporary container
                $tempObjectAttributeContainer = array();
                foreach ($event['Attribute'] as $key => &$attribute) {
@ -2038,7 +2056,6 @@ class Event extends AppModel
                        unset($event['Attribute'][$key]);
                        continue;
                    }
-                    $attribute = $this->Attribute->Correlation->setCorrelationExclusion($attribute);
                    if ($attribute['category'] === 'Financial fraud') {
                        $attribute = $this->Attribute->attachValidationWarnings($attribute);
                    }
@ -2723,17 +2740,8 @@ class Event extends AppModel
    public function set_filter_value(&$params, $conditions, $options)
    {
        if (!empty($params['value'])) {
-            $params[$options['filter']] = $this->convert_filters($params[$options['filter']]);
-            $conditions = $this->generic_add_filter($conditions, $params[$options['filter']], ['Attribute.value1', 'Attribute.value2']);
-            // Allows searching for ['value1' => [full, part1], 'value2' => [full, part2]]
-            if (is_string($params['value']) && strpos('|', $params['value']) !== false) {
-                $valueParts = explode('|', $params['value'], 2);
-                $convertedFilterVal1 = $this->convert_filters($valueParts[0]);
-                $convertedFilterVal2 = $this->convert_filters($valueParts[1]);
-                $conditionVal1 = $this->generic_add_filter([], $convertedFilterVal1, ['Attribute.value1'])['AND'][0]['OR'];
-                $conditionVal2 = $this->generic_add_filter([], $convertedFilterVal2, ['Attribute.value2'])['AND'][0]['OR'];
-                $conditions['AND'][0]['OR']['OR']['AND'] = [$conditionVal1, $conditionVal2];
-            }
+            $params[$options['filter']] = $this->convert_filters($params['value']);
+            $conditions = $this->generic_add_filter($conditions, $params['value'], ['Attribute.value1', 'Attribute.value2']);
        }

        return $conditions;
@ -2995,8 +3003,13 @@ class Event extends AppModel
                'noEventReports' => true,
                'noSightings' => true,
                'metadata' => $metadataOnly,
-            ])[0];
-
+            ]);
+            if (empty($eventForUser)) {
+                $this->Job->saveProgress($jobId, null, $k / $userCount * 100);
+                $this->loadLog()->createLogEntry($senderUser, 'alert', 'User', $user['id'], __('Something went wrong with alerting user #%s about event #%s. Sending was blocked due to insufficient access to the given event.'));
+                continue;
+            }
+            $eventForUser = $eventForUser[0];
            if ($this->User->UserSetting->checkPublishFilter($user, $eventForUser)) {
                $body = $this->prepareAlertEmail($eventForUser, $user, $oldpublish);
                $this->User->sendEmail(['User' => $user], $body, false, null);
@ -3343,16 +3356,16 @@ class Event extends AppModel
        if (empty($this->eventBlockRule)) {
            return true;
        }
-        if (!empty($rules['tags'])) {
-            if (!is_array($rules['tags'])) {
-                $rules['tags'] = [$rules['tags']];
+        if (!empty($this->eventBlockRule['tags'])) {
+            if (!is_array($this->eventBlockRule['tags'])) {
+                $this->eventBlockRule['tags'] = [$this->eventBlockRule['tags']];
            }
            $eventTags = Hash::extract($event, 'Event.Tag.{n}.name');
            if (empty($eventTags)) {
                $eventTags = Hash::extract($event, 'Event.EventTag.{n}.Tag.name');
            }
            if (!empty($eventTags)) {
-                foreach ($rules['tags'] as $blockTag) {
+                foreach ($this->eventBlockRule['tags'] as $blockTag) {
                    if (in_array($blockTag, $eventTags)) {
                        return false;
                    }
@ -7550,15 +7563,21 @@ class Event extends AppModel
    {
        if ($fullEvent) {
            if (empty(Configure::read('Plugin.ZeroMQ_include_attachments'))) {
-                foreach ($fullEvent[0]['Attribute'] as $k => $attribute) {
-                    if (isset($attribute['data'])) {
-                        unset($fullEvent[0]['Attribute'][$k]['data']);
+                if (!empty($fullEvent[0]['Attribute'])) {
+                    foreach ($fullEvent[0]['Attribute'] as $k => $attribute) {
+                        if (isset($attribute['data'])) {
+                            unset($fullEvent[0]['Attribute'][$k]['data']);
+                        }
                    }
                }
-                foreach ($fullEvent[0]['Object'] as $k => $object) {
-                    foreach ($object['Attribute'] as $k2 => $attribute) {
-                        if (isset($attribute['data'])) {
-                            unset($fullEvent[0]['Object'][$k]['Attribute'][$k2]['data']);
+                if (!empty($fullEvent[0]['Object'])) {
+                    foreach ($fullEvent[0]['Object'] as $k => $object) {
+                        if (!empty($object['Attribute'])) {
+                            foreach ($object['Attribute'] as $k2 => $attribute) {
+                                if (isset($attribute['data'])) {
+                                    unset($fullEvent[0]['Object'][$k]['Attribute'][$k2]['data']);
+                                }
+                            }
                        }
                    }
                }
--- a/app/Model/Feed.php
+++ b/app/Model/Feed.php
@ -1005,6 +1005,16 @@ class Feed extends AppModel
            }
        }
        if ($feed['Feed']['tag_id']) {
+            if (empty($feed['Tag']['name'])) {
+                $feed_tag = $this->Tag->find('first', [
+                    'conditions' => [
+                        'Tag.id' => $feed['Feed']['tag_id']
+                    ],
+                    'recursive' => -1,
+                    'fields' => ['Tag.name', 'Tag.colour', 'Tag.id']
+                ]);
+                $feed['Tag'] = $feed_tag['Tag'];
+            }
            if (!isset($event['Event']['Tag'])) {
                $event['Event']['Tag'] = array();
            }
--- a/app/Model/Log.php
+++ b/app/Model/Log.php
@ -402,6 +402,8 @@ class Log extends AppModel
            }
            if (!empty($data['Log']['description'])) {
                $entry .= " -- {$data['Log']['description']}";
+            } else if (!empty($data['Log']['change'])) {
+                $entry .= " -- " . json_encode($data['Log']['change']);
            }
            $this->syslog->write($action, $entry);
        }
--- a/app/Model/Module.php
+++ b/app/Model/Module.php
@ -213,7 +213,10 @@ class Module extends AppModel
            'action' => 'execute_workflow',
            'id' => 0,
        ];
-        if (empty($triggerData) && $this->Workflow->isTriggerCallable($trigger_id) && !empty($postData['attribute_uuid'])) {
+        if (!$this->Workflow->isTriggerCallable($trigger_id)) {
+            return true;
+        }
+        if (empty($triggerData) && !empty($postData['attribute_uuid'])) {
            $this->User = ClassRegistry::init('User');
            $this->Attribute = ClassRegistry::init('Attribute');
            $user = $this->User->getAuthUser(Configure::read('CurrentUserId'), true);
--- a/app/Model/OverCorrelatingValue.php
+++ b/app/Model/OverCorrelatingValue.php
@ -1,6 +1,5 @@
 <?php
 App::uses('AppModel', 'Model');
-App::uses('RandomTool', 'Tools');

 class OverCorrelatingValue extends AppModel
 {
@ -10,10 +9,14 @@ class OverCorrelatingValue extends AppModel
        'Containable'
    );

-    public $validate = [
-    ];

-    public function block($value, $count)
+    /**
+     * @param string $value
+     * @param int $count
+     * @return void
+     * @throws Exception
+     */
+    public function block($value, $count = 0)
    {
        $this->unblock($value);
        $this->create();
@ -25,15 +28,23 @@ class OverCorrelatingValue extends AppModel
        );
    }

+    /**
+     * @param string $value
+     * @return void
+     */
    public function unBlock($value)
    {
        $this->deleteAll(
            [
                'OverCorrelatingValue.value' => $value
-            ]
+            ],
+            false
        );
    }

+    /**
+     * @return int
+     */
    public function getLimit()
    {
        return Configure::check('MISP.correlation_limit') ? Configure::read('MISP.correlation_limit') : 20;
@ -55,14 +66,57 @@ class OverCorrelatingValue extends AppModel

    public function checkValue($value)
    {
-        $hit = $this->find('first', [
-            'recursive' => -1,
-            'conditions' => ['value' => $value],
-            'fields' => ['id']
-        ]);
-        if (empty($hit)) {
-            return false;
+        return $this->hasAny(['value' => $value]);
+    }
+
+    public function generateOccurrencesRouter()
+    {
+        if (Configure::read('MISP.background_jobs')) {
+            /** @var Job $job */
+            $job = ClassRegistry::init('Job');
+            $jobId = $job->createJob(
+                'SYSTEM',
+                Job::WORKER_DEFAULT,
+                'generateOccurrences',
+                '',
+                'Starting populating the occurrences field for the over correlating values.'
+            );
+
+            $this->getBackgroundJobsTool()->enqueue(
+                BackgroundJobsTool::DEFAULT_QUEUE,
+                BackgroundJobsTool::CMD_ADMIN,
+                [
+                    'jobGenerateOccurrences',
+                    $jobId
+                ],
+                true,
+                $jobId
+            );
+
+            return $jobId;
+        } else {
+            return $this->generateOccurrences();
        }
-        return true;
+    }
+
+    public function generateOccurrences()
+    {
+        $overCorrelations = $this->find('all', [
+            'recursive' => -1
+        ]);
+        $this->Attribute = ClassRegistry::init('Attribute');
+        foreach ($overCorrelations as &$overCorrelation) {
+            $count = $this->Attribute->find('count', [
+                'recursive' => -1,
+                'conditions' => [
+                    'OR' => [
+                        'Attribute.value1' => $overCorrelation['OverCorrelatingValue']['value'],
+                        'Attribute.value2' => $overCorrelation['OverCorrelatingValue']['value']
+                    ]
+                ]
+            ]);
+            $overCorrelation['OverCorrelatingValue']['occurrence'] = $count;
+        }
+        $this->saveMany($overCorrelations);
    }
 }
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@ -3213,7 +3213,7 @@ class Server extends AppModel
    private function getDatabaseIndexes($database, $table)
    {
        $sqlTableIndex = sprintf(
-            "SELECT DISTINCT TABLE_NAME, COLUMN_NAME, NON_UNIQUE FROM information_schema.statistics WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s';",
+            "SELECT DISTINCT TABLE_NAME, COLUMN_NAME, NON_UNIQUE FROM information_schema.statistics WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s' ORDER BY COLUMN_NAME;",
            $database,
            $table
        );
@ -5442,6 +5442,13 @@ class Server extends AppModel
                    'type' => 'string',
                    'null' => true,
                ),
+                'store_api_access_time' => array(
+                    'level' => 1,
+                    'description' => __('If enabled, MISP will capture the last API access time following a successful authentication using API keys, stored against a user under the last_api_access field.'),
+                    'value' => false,
+                    'test' => 'testBool',
+                    'type' => 'boolean',
+                ),
                'log_auth' => array(
                    'level' => 1,
                    'description' => __('If enabled, MISP will log all successful authentications using API keys. The requested URLs are also logged.'),
--- a/app/Model/User.php
+++ b/app/Model/User.php
@ -1404,6 +1404,22 @@ class User extends AppModel
        return $this->save($user, true, array('id', 'last_login', 'current_login'));
    }

+    /**
+     * Updates `last_api_access` time in database.
+     *
+     * @param array $user
+     * @return array|bool
+     * @throws Exception
+     */
+    public function updateAPIAccessTime(array $user)
+    {
+        if (!isset($user['id'])) {
+            throw new InvalidArgumentException("Invalid user object provided.");
+        }
+        $user['last_api_access'] = time();
+        return $this->save($user, true, array('id', 'last_api_access'));
+    }
+
    /**
     * Update field in user model and also set `date_modified`
     *
--- a/app/View/Correlations/over_correlations.ctp
+++ b/app/View/Correlations/over_correlations.ctp
@ -23,6 +23,11 @@
                                'url' => $baseurl . '/correlations/overCorrelations/scope:not_over_correlating',
                                'text' => __('Not over-correlating')
                            ],
+                            [
+                                'type' => 'simple',
+                                'url' => $baseurl . '/correlations/generateOccurrences',
+                                'text' => __('Regenerate occurrence counts')
+                            ]
                        ]
                    ]
                ]
--- a/app/View/Users/admin_index.ctp
+++ b/app/View/Users/admin_index.ctp
@ -199,6 +199,14 @@
                        'class' => 'short',
                        'data_path' => 'User.date_created'
                    ),
+                    array(
+                        'name' => __('Last API Access'),
+                        'sort' => 'User.last_api_access',
+                        'element' => 'datetime',
+                        'class' => 'short',
+                        'data_path' => 'User.last_api_access',
+                        'requirement' => !empty(Configure::read('MISP.store_api_access_time')) && Configure::read('MISP.store_api_access_time', false)
+                    ),
                    array(
                        'name' => (Configure::read('Plugin.CustomAuth_name') ? Configure::read('Plugin.CustomAuth_name') : __('External Auth')),
                        'sort' => 'User.external_auth_required',
--- a/app/files/misp-galaxy
+++ b/app/files/misp-galaxy
@ -1 +1 @@
-Subproject commit bfda561f5f29a8cca573e22789fb58252ad36c93
+Subproject commit b0ffb843b0bb69ea94d3ce9318f5123612b4ccc9
--- a/app/files/scripts/stix2/stix2misp.py
+++ b/app/files/scripts/stix2/stix2misp.py
@ -1325,12 +1325,19 @@ class ExternalStixParser(StixParser):
        else:
            misp_object = self.create_misp_object(attack_pattern)
            if hasattr(attack_pattern, 'external_references'):
+                references = defaultdict(set)
                for reference in attack_pattern.external_references:
-                    source_name = reference['source_name']
-                    value = reference['external_id'].split('-')[1] if source_name == 'capec' else reference['url']
-                    attribute = deepcopy(stix2misp_mapping.attack_pattern_references_mapping[source_name]) if source_name in stix2misp_mapping.attack_pattern_references_mapping else stix2misp_mapping.references_attribute_mapping
-                    attribute['value'] = value
-                    misp_object.add_attribute(**attribute)
+                    if hasattr(reference, 'url'):
+                        references['references'].add(reference.url)
+                    if hasattr(reference, 'external_id'):
+                        external_id = reference.external_id
+                        references['id'].add(external_id.split('-')[1] if external_id.startswith('CAPEC-') else external_id)
+                if references:
+                    for feature, values in references.items():
+                        for value in values:
+                            attribute = {'value': value}
+                            attribute.update(getattr(stix2misp_mapping, f'attack_pattern_{feature}_attribute'))
+                            misp_object.add_attribute(**attribute)
            self.fill_misp_object(misp_object, attack_pattern, 'attack_pattern_mapping')
            self.misp_event.add_object(**misp_object)

--- a/app/files/scripts/stix2/stix2misp_mapping.py
+++ b/app/files/scripts/stix2/stix2misp_mapping.py
@ -193,6 +193,8 @@ single_attribute_fields = ('type', 'value', 'to_ids')

 address_family_attribute_mapping = {'type': 'text','object_relation': 'address-family'}
 as_number_attribute_mapping = {'type': 'AS', 'object_relation': 'asn'}
+attack_pattern_id_attribute = {'type': 'text', 'object_relation': 'id'}
+attack_pattern_references_attribute = {'type': 'link', 'object_relation': 'references'}
 description_attribute_mapping = {'type': 'text', 'object_relation': 'description'}
 asn_subnet_attribute_mapping = {'type': 'ip-src', 'object_relation': 'subnet-announced'}
 cc_attribute_mapping = {'type': 'email-dst', 'object_relation': 'cc'}
--- a/app/files/warninglists
+++ b/app/files/warninglists
@ -1 +1 @@
-Subproject commit fc5599114f92926838b519f322b81a061039fe0e
+Subproject commit faee7c9dff59b61942f53aebee44ee0c59d8ca4f
--- a/app/webroot/doc/openapi.yaml
+++ b/app/webroot/doc/openapi.yaml
@ -2822,6 +2822,10 @@ components:
          $ref: "#/components/schemas/LimitSearchFilter"
        value:
          $ref: "#/components/schemas/AttributeValue"
+        value1:
+          $ref: "#/components/schemas/AttributeValue"
+        value2:
+          $ref: "#/components/schemas/AttributeValue"
        type:
          $ref: "#/components/schemas/AttributeType"
        category:
--- a/db_schema.json
+++ b/db_schema.json
--- a/docs/correlations.rework.md
+++ b/docs/correlations.rework.md
@ -0,0 +1,123 @@
+# MISP 2.4.160 - Correlation engine rework
+
+With the latest release of MISP, we have completely redone how we do correlations. Why we did all of this, how you can switch to the new engines as well as what sort of functionalities will be at your disposal are the topics that this blog post is meant to discuss, so grab a summer drink of your choice, kick back and let's dive into it!
+
+# Why did we have to retire the old engine?
+
+Whilst the original correlation engine has worked for us well for over a decade, its design hinged on **correlations being a rarity**. Back when we started with MISP, we were dealing with tiny, governmental and military communities only, purely sharing targeted attack information. This meant that **correlations were rare** and an **immediate prompt for further investigation**.
+
+Nowadays though, with the wealth of information we have access to, this phenomenon has shifted drastically. We're seeing different communities come together, share information that is relevant for different use-cases as well as often a natural duplication and overlap of analyses. 
+
+This recently lead to a great number of instances see their correlations explosively grow, reaching **several hundreds of Gigabytes in size on disk**, making instances unusable.
+
+This lead us to rethink and to rework the way we do correlations.
+
+# The new correlation engines
+
+To resolve the above issue, we ended up reimplementing the engine - and realised that a lot of the processing burden when it comes to correlations, is a result of the access control checks governing MISP. This is crucial for any sharing community to be in place, in order to avoid information leakage through correlations.
+
+With that said, a great many MISP instances are used internally within organisations, connected to sharing community instances. These instances normally only have a single organisation or team as user and therefore any data pulled down by the instance is visible to all of their users. In these cases, it would make sense to have an engine that avoids storing ACL information as well  as utilising them for filtering when fetching correlations.
+
+To accomodate both use-cases, we now have two correlation engines:
+
+- The `Default correlation engine` - for sharing communities or for any instance with more than one organisation, where access control is crucial
+- The `No ACL correlation engine` - for internal, single organisation or "endpoint" MISP instances.
+
+![](https://raw.githubusercontent.com/MISP/MISP/2.4/docs/img/2.4.160/new_engine.png)
+
+
+# The upgrade process
+
+Upgrading to MISP 2.4.160 will automatically take care of everything required to get going with the new engines. The old correlation engine's data store is automatically purged (the table is truncated) and once the new table structures are created, a recorrelation job is started. Depending on the amount of data you have in your instance and your system performance, this might take quite a long time. For our largest operational instance it took 40 hours to recorrelate the data, so don't worry if you are not seeing the correlations immediately. 
+
+The `Default correlation engine` will be used by default, but feel free switch to the No ACL correlation engine if that fits your use-case more. You can reach the new correlation control center via Administration -> Server settings -> Correlations.
+
+Despite the re-correlation potentially taking a long time, your instance will still be usable as usual during this time.
+
+### Some precautions you can take to ease the process:
+
+- Make sure that your mysql is able to perform well, it is especially important that the innodb_buffer_size is not using the rather restrictive default value
+- Potentially disalbe query caching as this is one of the situations where it can be quite detrimental due to the alternating high frequency reads/writes to the same table (you can do this by issuing the `SET GLOBAL query_cache_size = 0;` command via your MySQL CLI client)
+- Run multiple `default` background workers, since the correlation will keep one of the workers monitoring that queue busy for a prolonged period. (Add more via Administration->server settings->workers)
+
+# How is the new engine different than the old one?
+
+### A list of the main differences:
+
+- Reworked data model
+  - Correlation tables are purely made up of tinyint and int values - no more strings
+  - All correlating values are spun off into a new table (`correlation_values`)
+  - Correlation tables are now object ACL aware (this remedies a bug discovered in the old engine)
+  - Correlation tables now contain the ACL data of both correlating entities, making them bi-directional
+- The bi-directional nature of the new correlation data model means that we only store each correlation ones rather than storing separate A->B and B->A entries
+- New over-correlation feature protects your instance from values that are generating overwhelmingly noisy correlation
+- Reworked logic for managing the correlations
+- In the case of the `No ACL correlation engine`, the datamodel is reduced to storing just the two correlating attribute and event IDs along with a reference to the correlating value.
+- Casualties of the new engines:
+    - Proposal correlations have been sunset. They were of little use and were confusing due to their implementation
+- Rely on on-demand lookups for over-correlations
+
+### Some of the expected outcomes
+- Massively increased performance
+- Reduced size on disk (size on disk reduced to 1-25%, depending on the data-sets used, based on our various community instances)
+- Some functionalities that were unusable prior to the release are suddenly low-cost, such as the correlation count on the index
+
+### Issues to still resolve
+- Over-correlation table shows incorrect values for the number of correlating values (defaulting to the limit +1).
+- Some queries could still be tuned for some quick gains in performance
+
+# Switching engines
+
+The following proedure is to be used to switch between the `Default correlation engine` and the `No ACL engine`:
+- Navigate to Administration -> server settings -> correlations
+- Click on `Activate` under the chosen engine's header
+- Once the desired engine becomes active, truncate the table of the previously used engine to regain the disk space
+- Click on Re-correlate to start the process of correlating the data using the new engine
+
+![](https://raw.githubusercontent.com/MISP/MISP/2.4/docs/img/2.4.160/switching_engines.png)
+
+# Correlation filter management
+
+With the automatic and manual correlation filtering in place, we have two new systems to manage:
+
+### Over-correlation protection
+
+The new system will automatically restrict any correlations from being entered that would cross a given threshold. The data is stored in the database, via the `over_correlating_values` table and any correlation process will automatically add values to it.
+
+By default, the threshold allows for 20 correlating attributes with the same value, before it adds the value to the over correlation table and stops further correlations from being captured.
+
+To view the values that are overcorrelating head over to administration -> over-correlating values.
+
+![](https://raw.githubusercontent.com/MISP/MISP/2.4/docs/img/2.4.160/over_correlations.png)
+
+By clicking any of the values above, you will be redirected to the attribute search's results for the value, giving you a live result set.
+
+In addition to the values being blocked based on the threshold the state of this table will also show whether a value has an exclusion entry in the Correlation exclusions system.
+
+![](https://raw.githubusercontent.com/MISP/MISP/2.4/docs/img/2.4.160/over_correlations2.png)
+
+You will be able to see attrbutes as having too many correlations when viewing events, clicking on the magnifying glass will bring up the attribute search for the attribute value.
+
+### The Correlation exclusions
+
+This is the manual system that has existed for a few versions now in MISP. This system has been further improved and integrated into the event View.
+
+You can add new entries via the Over-correlation and the top correlations (Administration->top correlations) interfaces, as well as via the correlation exclusion index directly. You can also add an optional comment why you have excluded that value.
+
+![](https://raw.githubusercontent.com/MISP/MISP/2.4/docs/img/2.4.160/correlation_exclusions.png)
+
+Add, update or remove entries directly in the correlation exclusion index (Accessible via the top correlations index - Administration -> Top Correlations and then clicking on correlation exclusions).
+
+Any changes made are **NOT** actioned upon retroactively, until you run a "Clean up correlations" action.
+
+![](https://raw.githubusercontent.com/MISP/MISP/2.4/docs/img/2.4.160/correlation_exclusions2.png)
+
+Events will show excluded correlations on the attributes directly, similarly to over-correlations. The main difference is the message, rather than `Too many correlations`, correlations will show up as `Excluded`.
+
+# Engine development
+
+The new engine system is built with modularity in mind, we expect to develop new engines in the future as well as see the community give life to custom engines.
+
+Currently all generic correlation code (fetching data when correlating attributes, initiating processes) is handled by the Correlation model (`/app/Model/Correlation.php`) with custom engine implementations being available in the Behaviour directory (`/app/Model/Behavior/[correlation_name]CorrelationBehavior.php`).
+
+For developers: All functions of the existing Correlation Behaviors that are public are **REQUIRED** by the system to be implemented in the engine.
--- a/docs/img/2.4.160/correlation_exclusions.png
+++ b/docs/img/2.4.160/correlation_exclusions.png
--- a/docs/img/2.4.160/correlation_exclusions2.png
+++ b/docs/img/2.4.160/correlation_exclusions2.png
--- a/docs/img/2.4.160/new_engine.png
+++ b/docs/img/2.4.160/new_engine.png
--- a/docs/img/2.4.160/over_correlations.png
+++ b/docs/img/2.4.160/over_correlations.png
--- a/docs/img/2.4.160/over_correlations2.png
+++ b/docs/img/2.4.160/over_correlations2.png
--- a/docs/img/2.4.160/switching_engines.png
+++ b/docs/img/2.4.160/switching_engines.png
--- a/tests/testlive_comprehensive_local.py
+++ b/tests/testlive_comprehensive_local.py
@ -5,6 +5,7 @@ import uuid
 import subprocess
 import unittest
 import requests
+import time
 from xml.etree import ElementTree as ET
 from io import BytesIO
 import urllib3  # type: ignore
@ -23,13 +24,15 @@ key = os.environ["AUTH"]
 urllib3.disable_warnings()


-def create_simple_event():
+def create_simple_event() -> MISPEvent:
+    event_uuid = str(uuid.uuid4())
    event = MISPEvent()
-    event.info = 'This is a super simple test'
+    event.uuid = event_uuid
+    event.info = 'This is a super simple test ({})'.format(event_uuid.split('-')[0])
    event.distribution = Distribution.your_organisation_only
    event.threat_level_id = ThreatLevel.low
    event.analysis = Analysis.completed
-    event.add_attribute('text', str(uuid.uuid4()))
+    event.add_attribute('text', event_uuid)
    return event


@ -511,6 +514,51 @@ class TestComprehensive(unittest.TestCase):
            for event in (first, second, third, four):
                check_response(self.admin_misp_connector.delete_event(event))

+    def test_correlations(self):
+        first = create_simple_event()
+        first.add_attribute("ip-src", "10.0.0.1")
+        first = check_response(self.admin_misp_connector.add_event(first))
+
+        second = create_simple_event()
+        second.add_attribute("ip-src", "10.0.0.1")
+        second = check_response(self.admin_misp_connector.add_event(second))
+
+        # Reload to get event data with related events
+        first = check_response(self.admin_misp_connector.get_event(first))
+
+        try:
+            self.assertEqual(1, len(first.RelatedEvent), first.RelatedEvent)
+            self.assertEqual(1, len(second.RelatedEvent), second.RelatedEvent)
+        except:
+            raise
+        finally:
+            # Delete events
+            for event in (first, second):
+                check_response(self.admin_misp_connector.delete_event(event))
+
+    def test_advanced_correlations(self):
+        with MISPSetting(self.admin_misp_connector, {"MISP.enable_advanced_correlations": True}):
+            first = create_simple_event()
+            first.add_attribute("ip-src", "10.0.0.0/8")
+            first = check_response(self.admin_misp_connector.add_event(first))
+
+            second = create_simple_event()
+            second.add_attribute("ip-src", "10.0.0.1")
+            second = check_response(self.admin_misp_connector.add_event(second))
+
+            # Reload to get event data with related events
+            first = check_response(self.admin_misp_connector.get_event(first))
+
+            try:
+                self.assertEqual(1, len(first.RelatedEvent), first.RelatedEvent)
+                self.assertEqual(1, len(second.RelatedEvent), second.RelatedEvent)
+            except:
+                raise
+            finally:
+                # Delete events
+                for event in (first, second):
+                    check_response(self.admin_misp_connector.delete_event(event))
+
    def test_remove_orphaned_correlations(self):
        result = self.admin_misp_connector._check_json_response(self.admin_misp_connector._prepare_request('GET', 'servers/removeOrphanedCorrelations'))
        check_response(result)
@ -804,7 +852,7 @@ class TestComprehensive(unittest.TestCase):
        check_response(event)

        self.admin_misp_connector.publish(event, alert=False)
-
+        time.sleep(6)
        snort = self._search({'returnFormat': 'snort', 'eventid': event.id})
        self.assertIsInstance(snort, str)
        self.assertIn('8.8.8.8', snort)
@ -815,12 +863,49 @@ class TestComprehensive(unittest.TestCase):

        self.admin_misp_connector.delete_event(event)

+    def test_restsearch_composite_attribute(self):
+        event = create_simple_event()
+        attribute_1 = event.add_attribute('ip-src|port', '10.0.0.1|8080')
+        attribute_2 = event.add_attribute('ip-src|port', '10.0.0.2|8080')
+        event = self.user_misp_connector.add_event(event)
+        check_response(event)
+
+        search_result = self._search_attribute({'value': '10.0.0.1', 'eventid': event.id})
+        self.assertEqual(search_result['Attribute'][0]['uuid'], attribute_1.uuid)
+        self.assertEqual(len(search_result['Attribute']), 1)
+
+        search_result = self._search_attribute({'value': '8080', 'eventid': event.id})
+        self.assertEqual(len(search_result['Attribute']), 2)
+
+        search_result = self._search_attribute({'value1': '10.0.0.1', 'eventid': event.id})
+        self.assertEqual(len(search_result['Attribute']), 1)
+        self.assertEqual(search_result['Attribute'][0]['uuid'], attribute_1.uuid)
+
+        search_result = self._search_attribute({'value1': '10.0.0.2', 'eventid': event.id})
+        self.assertEqual(len(search_result['Attribute']), 1)
+        self.assertEqual(search_result['Attribute'][0]['uuid'], attribute_2.uuid)
+
+        search_result = self._search_attribute({'value2': '8080', 'eventid': event.id})
+        self.assertEqual(len(search_result['Attribute']), 2)
+
+        search_result = self._search_attribute({'value1': '10.0.0.1', 'value2': '8080', 'eventid': event.id})
+        self.assertEqual(len(search_result['Attribute']), 1)
+        self.assertEqual(search_result['Attribute'][0]['uuid'], attribute_1.uuid)
+
+        self.admin_misp_connector.delete_event(event)
+
+
    def _search(self, query: dict):
        response = self.admin_misp_connector._prepare_request('POST', 'events/restSearch', data=query)
        response = self.admin_misp_connector._check_response(response)
        check_response(response)
        return response

+    def _search_attribute(self, query: dict):
+        response = self.admin_misp_connector._prepare_request('POST', 'attributes/restSearch', data=query)
+        response = self.admin_misp_connector._check_response(response)
+        check_response(response)
+        return response

 if __name__ == '__main__':
    unittest.main()