Merge branch 'develop' into 2.4

VERSION.json

@@ -1 +1 @@
-{"major":2, "minor":4, "hotfix":162}
+{"major":2, "minor":4, "hotfix":163}

app/Console/Command/AdminShell.php

@@ -22,8 +22,6 @@ class AdminShell extends AppShell
         'Role', 'Feed', 'SharingGroupBlueprint', 'Correlation', 'OverCorrelatingValue'
     ];
 
-    public $tasks = ['ConfigLoad'];
-    
     public function getOptionParser()
     {
         $parser = parent::getOptionParser();
@@ -105,7 +103,6 @@ class AdminShell extends AppShell
 
     public function jobGenerateCorrelation()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0])) {
             die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Generate correlation'] . PHP_EOL);
         }
@@ -116,7 +113,6 @@ class AdminShell extends AppShell
 
     public function jobGenerateOccurrences()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0])) {
             die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Generate over-correlation occurrences'] . PHP_EOL);
         }
@@ -138,7 +134,6 @@ class AdminShell extends AppShell
 
     public function jobGenerateShadowAttributeCorrelation()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0])) {
             die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Generate shadow attribute correlation'] . PHP_EOL);
         }
@@ -152,14 +147,12 @@ class AdminShell extends AppShell
 
     public function updateMISP()
     {
-        $this->ConfigLoad->execute();
         $status = array('branch' => '2.4');
         echo $this->Server->update($status) . PHP_EOL;
     }
 
     public function updateAfterPull()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
             die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Update after pull'] . PHP_EOL);
         }
@@ -186,7 +179,6 @@ class AdminShell extends AppShell
             $this->error('This method does nothing when SimpleBackgroundJobs are enabled.');
         }
 
-        $this->ConfigLoad->execute();
         $this->Server->restartWorkers();
         echo PHP_EOL . 'Workers restarted.' . PHP_EOL;
     }
@@ -197,7 +189,6 @@ class AdminShell extends AppShell
             $this->error('This method does nothing when SimpleBackgroundJobs are enabled.');
         }
 
-        $this->ConfigLoad->execute();
         if (empty($this->args[0]) || !is_numeric($this->args[0])) {
             die('Usage: ' . $this->Server->command_line_functions['worker_management_tasks']['data']['Restart a worker'] . PHP_EOL);
         }
@@ -223,7 +214,6 @@ class AdminShell extends AppShell
             $this->error('This method does nothing when SimpleBackgroundJobs are enabled.');
         }
 
-        $this->ConfigLoad->execute();
         if (empty($this->args[0]) || !is_numeric($this->args[0])) {
             die('Usage: ' . $this->Server->command_line_functions['worker_management_tasks']['data']['Kill a worker'] . PHP_EOL);
         }
@@ -244,7 +234,6 @@ class AdminShell extends AppShell
             $this->error('This method does nothing when SimpleBackgroundJobs are enabled.');
         }
 
-        $this->ConfigLoad->execute();
         if (empty($this->args[0])) {
             die('Usage: ' . $this->Server->command_line_functions['worker_management_tasks']['data']['Start a worker'] . PHP_EOL);
         }
@@ -276,7 +265,6 @@ class AdminShell extends AppShell
 
     public function updateGalaxies()
     {
-        $this->ConfigLoad->execute();
         // The following is 7.x upwards only
         //$value = $this->args[0] ?? $this->args[0] ?? 0;
         $value = empty($this->args[0])  ? null : $this->args[0];
@@ -347,7 +335,6 @@ class AdminShell extends AppShell
 
     public function updateNoticeLists()
     {
-        $this->ConfigLoad->execute();
         $result = $this->Noticelist->update();
         if ($result) {
             echo 'Notice lists updated' . PHP_EOL;
@@ -359,7 +346,6 @@ class AdminShell extends AppShell
     # FIXME: Fails to pass userId/orgId properly, global update works.
     public function updateObjectTemplates()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0])) {
             die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Update object templates'] . PHP_EOL);
         } else {
@@ -392,7 +378,6 @@ class AdminShell extends AppShell
 
     public function jobUpgrade24()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Job upgrade'] . PHP_EOL);
         }
@@ -410,7 +395,6 @@ class AdminShell extends AppShell
 
     public function prune_update_logs()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Prune update logs'] . PHP_EOL);
         }
@@ -429,7 +413,6 @@ class AdminShell extends AppShell
 
     public function getWorkers()
     {
-        $this->ConfigLoad->execute();
         $result = $this->Server->workerDiagnostics($workerIssueCount);
         $query = 'all';
         if (!empty($this->args[0])) {
@@ -501,7 +484,6 @@ class AdminShell extends AppShell
 
     public function setDatabaseVersion()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0])) {
             die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Set database version'] . PHP_EOL);
         } else {
@@ -580,7 +562,6 @@ class AdminShell extends AppShell
 
     public function setDefaultRole()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0]) || !is_numeric($this->args[0])) {
             $roles = $this->Role->find('list', array(
                 'fields' => array('id', 'name')
@@ -615,7 +596,6 @@ class AdminShell extends AppShell
      */
     public function change_authkey()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0])) {
             echo 'MISP apikey command line tool' . PHP_EOL . 'To assign a new random API key for a user: ' . APP . 'Console/cake Admin change_authkey [user_email]' . PHP_EOL . 'To assign a fixed API key: ' . APP . 'Console/cake Admin change_authkey [user_email] [authkey]' . PHP_EOL;
             die();
@@ -646,7 +626,6 @@ class AdminShell extends AppShell
 
     public function recoverSinceLastSuccessfulUpdate()
     {
-        $this->ConfigLoad->execute();
         $this->loadModel('Log');
         $logs = $this->Log->find('all', array(
             'conditions' => array(
@@ -685,7 +664,6 @@ class AdminShell extends AppShell
 
     public function cleanCaches()
     {
-        $this->ConfigLoad->execute();
         echo 'Cleaning caches...' . PHP_EOL;
         $this->Server->cleanCacheFiles();
         echo '...caches lost in time, like tears in rain.' . PHP_EOL;
@@ -693,7 +671,6 @@ class AdminShell extends AppShell
 
     public function resetSyncAuthkeys()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0])) {
             echo sprintf(
                 __("MISP mass sync authkey reset command line tool" . PHP_EOL . "Usage: %sConsole/cake Admin resetSyncAuthkeys [user_id]" . PHP_EOL), APP
@@ -719,7 +696,6 @@ class AdminShell extends AppShell
 
     public function purgeFeedEvents()
     {
-        $this->ConfigLoad->execute();
         if (
             (empty($this->args[0]) || !is_numeric($this->args[0])) ||
             (empty($this->args[1]) || !is_numeric($this->args[1]))
@@ -759,7 +735,6 @@ class AdminShell extends AppShell
      */
     public function UserIP()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0])) {
             die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get IPs for user ID'] . PHP_EOL);
         }
@@ -787,7 +762,6 @@ class AdminShell extends AppShell
      */
     public function IPUser()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0])) {
             die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get user ID for user IP'] . PHP_EOL);
         }
@@ -1207,7 +1181,6 @@ class AdminShell extends AppShell
 
     public function truncateTable()
     {
-        $this->ConfigLoad->execute();
         if (!isset($this->args[0])) {
             die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Truncate table correlation'] . PHP_EOL);
         }

app/Console/Command/AppShell.php

@@ -32,9 +32,10 @@ class AppShell extends Shell
 
     public function initialize()
     {
-        parent::initialize();
         $this->ConfigLoad = $this->Tasks->load('ConfigLoad');
         $this->ConfigLoad->execute();
+
+        parent::initialize();
     }
 
     public function perform()

app/Console/Command/AuthkeyShell.php

@@ -10,13 +10,10 @@ class AuthkeyShell extends AppShell {
 
     public $uses = array('User', 'Log');
 
-    public $tasks = array('ConfigLoad');
-
     public function main()
     {
         $this->err('This method is deprecated. Next time please use `cake user change_authkey [user] [authkey]` command.');
 
-        $this->ConfigLoad->execute();
         if (!isset($this->args[0]) || empty($this->args[0])) echo 'MISP authkey reset command line tool.' . PHP_EOL . 'To assign a new authkey for a user:' . PHP_EOL . APP . 'Console/cake Authkey [email] [auth_key | optional]' . PHP_EOL;
         else {
             // get the users that need their password hashed

app/Console/Command/BaseurlShell.php

@@ -13,7 +13,6 @@ class BaseurlShell extends AppShell {
     {
         $this->err('This method is deprecated. Next time please use `cake admin setSetting MISP.baseurl [baseurl]` command.');
 
-        $this->ConfigLoad->execute();
         $baseurl = $this->args[0];
         $result = $this->Server->testBaseURL($baseurl);
         if (true !== $result) {

app/Console/Command/EventShell.php

@@ -15,7 +15,6 @@ require_once 'AppShell.php';
 class EventShell extends AppShell
 {
     public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Allowedlist', 'Server', 'Organisation', 'Correlation', 'Tag');
-    public $tasks = array('ConfigLoad');
 
     public function getOptionParser()
     {
@@ -121,7 +120,6 @@ class EventShell extends AppShell
 
     public function doPublish()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0])) {
             die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Do publish'] . PHP_EOL);
         }
@@ -157,7 +155,6 @@ class EventShell extends AppShell
 
     public function correlateValue()
     {
-        $this->ConfigLoad->execute();
         $value = $this->args[0];
 
         if (!empty($this->args[1])) {
@@ -182,7 +179,6 @@ class EventShell extends AppShell
 
     public function cache()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
             die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Cache event'] . PHP_EOL);
         }
@@ -223,7 +219,6 @@ class EventShell extends AppShell
 
     private function __runCaching($user, $typeData, $id, $export_type, $subType = '')
     {
-        $this->ConfigLoad->execute();
         $export_type = strtolower($typeData['type']);
         $final = $this->{$typeData['scope']}->restSearch($user, $typeData['params']['returnFormat'], $typeData['params'], false, $id);
         $dir = new Folder(APP . 'tmp/cached_exports/' . $export_type, true, 0750);
@@ -240,7 +235,6 @@ class EventShell extends AppShell
 
     public function cachebro()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Cache bro'] . PHP_EOL);
         }
@@ -281,7 +275,6 @@ class EventShell extends AppShell
 
     public function alertemail()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
             die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Alert email'] . PHP_EOL);
         }
@@ -314,7 +307,6 @@ class EventShell extends AppShell
 
     public function postsemail()
     {
-        $this->ConfigLoad->execute();
         if (
             empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) ||
             empty($this->args[3]) || empty($this->args[4]) || empty($this->args[5])
@@ -348,7 +340,6 @@ class EventShell extends AppShell
 
     public function enqueueCaching()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0])) {
             die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Enqueue caching'] . PHP_EOL);
         }
@@ -405,7 +396,6 @@ class EventShell extends AppShell
 
     public function publish()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0]) || empty($this->args[2]) || empty($this->args[3])) {
             die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Publish event'] . PHP_EOL);
         }
@@ -465,7 +455,6 @@ class EventShell extends AppShell
 
     public function publish_galaxy_clusters()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || !array_key_exists(3, $this->args)) {
             die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Publish Galaxy clusters'] . PHP_EOL);
         }
@@ -492,7 +481,6 @@ class EventShell extends AppShell
 
     public function enrichment()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
             die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Run enrichment'] . PHP_EOL);
         }
@@ -588,7 +576,6 @@ class EventShell extends AppShell
 
     public function recoverEvent()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Recover event'] . PHP_EOL);
         }

app/Console/Command/LiveShell.php

@@ -10,7 +10,6 @@ class LiveShell extends AppShell {
 
     public function main()
     {
-        $this->ConfigLoad->execute();
         $live = $this->args[0];
         if ($live != 0 && $live != 1) {
             echo 'Invalid parameters. Usage: /var/www/MISP/app/Console/cake Live [0|1]';

app/Console/Command/PasswordShell.php

@@ -14,7 +14,6 @@ class PasswordShell extends AppShell {
     {
         $this->err('This method is deprecated. Next time please use `cake user change_pw [user] [password]` command.');
 
-        $this->ConfigLoad->execute();
         if (!isset($this->args[0]) || empty($this->args[0]) || !isset($this->args[1]) || empty($this->args[1])) echo 'MISP password reset command line tool.' . PHP_EOL . 'To assign a new password for a user:' . PHP_EOL . APP . 'Console/cake Password [email] [password]' . PHP_EOL;
         else {
             // get the users that need their password hashed
@@ -42,7 +41,6 @@ class PasswordShell extends AppShell {
 
     public function getOptionParser()
     {
-        $this->ConfigLoad->execute();
         $parser = parent::getOptionParser();
         $parser->addOption('override_password_change', array(
             'short' => 'o',

app/Console/Command/ServerShell.php

@@ -14,6 +14,20 @@ class ServerShell extends AppShell
 {
     public $uses = array('Server', 'Task', 'Job', 'User', 'Feed');
 
+    public function getOptionParser()
+    {
+        $parser = parent::getOptionParser();
+        $parser->addSubcommand('fetchIndex', [
+            'help' => __('Fetch remote instance event index.'),
+            'parser' => array(
+                'arguments' => array(
+                    'server_id' => ['help' => __('Remote server ID.'), 'required' => true],
+                ),
+            )
+        ]);
+        return $parser;
+    }
+
     public function list()
     {
         $servers = $this->Server->find('all', [
@@ -55,9 +69,17 @@ class ServerShell extends AppShell
         echo $this->json($res) . PHP_EOL;
     }
 
+    public function fetchIndex()
+    {
+        $serverId = intval($this->args[0]);
+        $server = $this->getServer($serverId);
+        $serverSync = new ServerSyncTool($server, $this->Server->setupSyncRequest($server));
+        $index = $this->Server->getEventIndexFromServer($serverSync);
+        echo $this->json($index) . PHP_EOL;
+    }
+
     public function pullAll()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['PullAll'] . PHP_EOL);
         }
@@ -367,7 +389,6 @@ class ServerShell extends AppShell
 
     public function enqueuePull()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Enqueue pull'] . PHP_EOL);
         }
@@ -430,7 +451,6 @@ class ServerShell extends AppShell
 
     public function enqueueFeedFetch()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Enqueue feed fetch'] . PHP_EOL);
         }
@@ -480,7 +500,6 @@ class ServerShell extends AppShell
 
     public function enqueueFeedCache()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Enqueue feed cache'] . PHP_EOL);
         }
@@ -537,7 +556,6 @@ class ServerShell extends AppShell
 
     public function enqueuePush()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Enqueue push'] . PHP_EOL);
         }
@@ -580,7 +598,6 @@ class ServerShell extends AppShell
 
     public function sendPeriodicSummaryToUsers()
     {
-        $this->ConfigLoad->execute();
         $periods = $this->__getPeriodsForToday();
         $start_time = time();
         echo __n('Started periodic summary generation for the %s period', 'Started periodic summary generation for periods: %s', count($periods), implode(', ', $periods)) . PHP_EOL;
@@ -593,7 +610,7 @@ class ServerShell extends AppShell
                 $this->User->sendEmail($user, $emailTemplate, false, null);
             }
         }
-        echo __('All reports sent. Task took %s secondes', time() -  $start_time) . PHP_EOL;
+        echo __('All reports sent. Task took %s seconds', time() -  $start_time) . PHP_EOL;
     }
 
     private function __getPeriodsForToday(): array

app/Console/Command/WorkflowShell.php

@@ -6,11 +6,9 @@ require_once 'AppShell.php';
 class WorkflowShell extends AppShell {
 
     public $uses = ['Job', 'Workflow'];
-    public $tasks = ['ConfigLoad'];
 
     public function executeWorkflowForTrigger()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || empty($this->args[3])) {
             die(__('Invalid number of arguments.'));
         }
@@ -39,7 +37,6 @@ class WorkflowShell extends AppShell {
 
     public function walkGraph()
     {
-        $this->ConfigLoad->execute();
         if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || empty($this->args[3])) {
             die(__('Invalid number of arguments.'));
         }

app/Controller/AppController.php

@@ -49,14 +49,6 @@ class AppController extends Controller
 
     public $restResponsePayload = null;
 
-    // Used for _isAutomation(), a check that returns true if the controller & action combo matches an action that is a non-xml and non-json automation method
-    // This is used to allow authentication via headers for methods not covered by _isRest() - as that only checks for JSON and XML formats
-    public $automationArray = array(
-        'events' => array('csv', 'nids', 'hids', 'xml', 'restSearch', 'stix', 'updateGraph', 'downloadOpenIOCEvent'),
-        'attributes' => array('text', 'downloadAttachment', 'returnAttributes', 'restSearch', 'rpz', 'bro'),
-        'objects' => array('restSearch')
-    );
-
     protected $_legacyParams = array();
     /** @var array */
     public $userRole;

app/Controller/AttributesController.php

@@ -71,7 +71,8 @@ class AttributesController extends AppController
 
     public function index()
     {
-        $this->paginate['conditions']['AND'][] = $this->Attribute->buildConditions($this->Auth->user());
+        $user = $this->Auth->user();
+        $this->paginate['conditions']['AND'][] = $this->Attribute->buildConditions($user);
         $attributes = $this->paginate();
 
         if ($this->_isRest()) {
@@ -84,13 +85,13 @@ class AttributesController extends AppController
             'fields' => ['Orgc.id', 'Orgc.name', 'Orgc.uuid'],
         ]);
         $orgTable = Hash::combine($orgTable, '{n}.Orgc.id', '{n}.Orgc');
-        $sgids = $this->Attribute->SharingGroup->authorizedIds($this->Auth->user());
+        $sgids = $this->Attribute->SharingGroup->authorizedIds($user);
         foreach ($attributes as &$attribute) {
             if (isset($orgTable[$attribute['Event']['orgc_id']])) {
                 $attribute['Event']['Orgc'] = $orgTable[$attribute['Event']['orgc_id']];
             }
             $temp = $this->Attribute->Correlation->getRelatedAttributes(
-                $this->Auth->user(),
+                $user,
                 $sgids,
                 $attribute['Attribute'],
                 [],
@@ -104,7 +105,7 @@ class AttributesController extends AppController
             $attribute['Event']['RelatedAttribute'][$attribute['Attribute']['id']] = $temp;
         }
 
-        list($attributes, $sightingsData) = $this->__searchUI($attributes);
+        list($attributes, $sightingsData) = $this->__searchUI($attributes, $user);
         $this->set('isSearch', 0);
         $this->set('sightingsData', $sightingsData);
         $this->set('orgTable', array_column($orgTable, 'name', 'id'));
@@ -1540,6 +1541,7 @@ class AttributesController extends AppController
 
     public function search($continue = false)
     {
+        $user = $this->Auth->user();
         $exception = null;
         $filters = $this->__getSearchFilters($exception);
         if ($this->request->is('post') || !empty($this->request->params['named']['tags'])) {
@@ -1571,7 +1573,7 @@ class AttributesController extends AppController
         }
         if (!empty($filters)) {
             $filters['includeCorrelations'] = 1;
-            $params = $this->Attribute->restSearch($this->Auth->user(), 'json', $filters, true);
+            $params = $this->Attribute->restSearch($user, 'json', $filters, true);
             if (!isset($params['conditions']['Attribute.deleted'])) {
                 $params['conditions']['Attribute.deleted'] = 0;
             }
@@ -1589,7 +1591,7 @@ class AttributesController extends AppController
                 'fields' => ['Orgc.id', 'Orgc.name', 'Orgc.uuid'],
             ]);
             $orgTable = array_column(array_column($orgTable, 'Orgc'), null, 'id');
-            $sgids = $this->Attribute->SharingGroup->authorizedIds($this->Auth->user());
+            $sgids = $this->Attribute->SharingGroup->authorizedIds($user);
             foreach ($attributes as &$attribute) {
                 if (isset($orgTable[$attribute['Event']['orgc_id']])) {
                     $attribute['Event']['Orgc'] = $orgTable[$attribute['Event']['orgc_id']];
@@ -1599,7 +1601,7 @@ class AttributesController extends AppController
                 }
                 if (isset($filters['includeCorrelations'])) {
                     $temp = $this->Attribute->Correlation->getRelatedAttributes(
-                        $this->Auth->user(),
+                        $user,
                         $sgids,
                         $attribute['Attribute'],
                         [],
@@ -1617,7 +1619,7 @@ class AttributesController extends AppController
                 return $this->RestResponse->viewData($attributes, $this->response->type());
             }
 
-            list($attributes, $sightingsData) = $this->__searchUI($attributes);
+            list($attributes, $sightingsData) = $this->__searchUI($attributes, $user);
             $this->set('sightingsData', $sightingsData);
 
             if (isset($filters['tags']) && !empty($filters['tags'])) {
@@ -1653,7 +1655,12 @@ class AttributesController extends AppController
         }
     }
 
-    private function __searchUI(array $attributes)
+    /**
+     * @param array $attributes
+     * @param array $user
+     * @return array|array[]
+     */
+    private function __searchUI(array $attributes, array $user)
     {
         if (empty($attributes)) {
             return [[], []];
@@ -1663,11 +1670,8 @@ class AttributesController extends AppController
 
         $this->loadModel('Sighting');
         $this->loadModel('AttachmentScan');
-        $user = $this->Auth->user();
-        $attributeIds = [];
         $galaxyTags = [];
         foreach ($attributes as &$attribute) {
-            $attributeIds[] = $attribute['Attribute']['id'];
             if ($this->Attribute->isImage($attribute['Attribute'])) {
                 if (extension_loaded('gd')) {
                     // if extension is loaded, the data is not passed to the view because it is asynchronously fetched
@@ -1687,8 +1691,8 @@ class AttributesController extends AppController
 
             $attribute['Attribute']['AttributeTag'] = $attribute['AttributeTag'];
             foreach ($attribute['Attribute']['AttributeTag'] as $at) {
-                if (substr($at['Tag']['name'], 0, 12) === 'misp-galaxy:') {
+                if ($at['Tag']['is_galaxy']) {
-                    $galaxyTags[] = $at['Tag']['name'];
+                    $galaxyTags[$at['Tag']['id']] = $at['Tag']['name'];
                 }
             }
             unset($attribute['AttributeTag']);
@@ -1698,15 +1702,12 @@ class AttributesController extends AppController
         // Fetch galaxy clusters in one query
         if (!empty($galaxyTags)) {
             $this->loadModel('GalaxyCluster');
-            $clusters = $this->GalaxyCluster->getClusters($galaxyTags, $user, true, false);
+            $clusters = $this->GalaxyCluster->getClustersByTags($galaxyTags, $user, true, false);
             $clusters = array_column(array_column($clusters, 'GalaxyCluster'), null, 'tag_id');
         } else {
             $clusters = [];
         }
 
-        // Fetch correlations in one query
-        $correlations = $this->Attribute->Event->getRelatedAttributes($user, $attributeIds, false, 'attribute');
-
         // `attachFeedCorrelations` method expects different attribute format, so we need to transform that, then process
         // and then take information back to original attribute structure.
         $fakeEventArray = [];
@@ -1721,7 +1722,7 @@ class AttributesController extends AppController
                 }
                 $cluster = $clusters[$attributeTag['Tag']['id']];
                 $galaxyId = $cluster['Galaxy']['id'];
-                $cluster['local'] = isset($attributeTag['local']) ? $attributeTag['local'] : false;
+                $cluster['local'] = $attributeTag['local'] ?? false;
                 if (isset($attribute['Attribute']['Galaxy'][$galaxyId])) {
                     unset($cluster['Galaxy']);
                     $galaxies[$galaxyId]['GalaxyCluster'][] = $cluster;
@@ -1737,12 +1738,9 @@ class AttributesController extends AppController
             if (isset($attributesWithFeedCorrelations[$k]['Feed'])) {
                 $attributes[$k]['Attribute']['Feed'] = $attributesWithFeedCorrelations[$k]['Feed'];
             }
-            if (isset($correlations[$attribute['Attribute']['id']])) {
-                $attributes[$k]['Attribute']['RelatedAttribute'] = $correlations[$attribute['Attribute']['id']];
-            }
         }
         $sightingsData = $this->Sighting->attributesStatistics($attributes, $user);
-        return array($attributes, $sightingsData);
+        return [$attributes, $sightingsData];
     }
 
     public function checkComposites()

app/Controller/Component/IndexFilterComponent.php

@@ -10,6 +10,14 @@ class IndexFilterComponent extends Component
     public $Controller;
     public $isRest = null;
 
+    // Used for isApiFunction(), a check that returns true if the controller & action combo matches an action that is a non-xml and non-json automation method
+    // This is used to allow authentication via headers for methods not covered by _isRest() - as that only checks for JSON and XML formats
+    const AUTOMATION_ARRAY = array(
+        'events' => array('csv', 'nids', 'hids', 'xml', 'restSearch', 'stix', 'updateGraph', 'downloadOpenIOCEvent'),
+        'attributes' => array('text', 'downloadAttachment', 'returnAttributes', 'restSearch', 'rpz', 'bro'),
+        'objects' => array('restSearch'),
+    );
+
     public function initialize(Controller $controller)
     {
         $this->Controller = $controller;
@@ -121,6 +129,6 @@ class IndexFilterComponent extends Component
      */
     public function isApiFunction($controller, $action)
     {
-        return isset($this->Controller->automationArray[$controller]) && in_array($action, $this->Controller->automationArray[$controller], true);
+        return isset(self::AUTOMATION_ARRAY[$controller]) && in_array($action, self::AUTOMATION_ARRAY[$controller], true);
     }
 }

app/Controller/CorrelationsController.php

@@ -83,34 +83,39 @@ class CorrelationsController extends AppController
             'page' => 1,
             'order' => 'occurrence desc'
         ];
-        foreach (array_keys($query) as $custom_param) {
+        foreach ($query as $customParam => $foo) {
-            if (isset($this->params['named'][$custom_param])) {
+            if (isset($this->request->params['named'][$customParam])) {
-                $query[$custom_param] = $this->params['named'][$custom_param];
+                $query[$customParam] = $this->request->params['named'][$customParam];
             }
         }
-        if (isset($this->params['named']['scope'])) {
+        if (isset($this->request->params['named']['scope'])) {
             $limit = $this->Correlation->OverCorrelatingValue->getLimit();
-            if ($this->params['named']['scope'] === 'over_correlating') {
+            if ($this->request->params['named']['scope'] === 'over_correlating') {
+                $scope = 'over_correlating';
                 $query['conditions'][] = ['occurrence >=' => $limit];
-            } else if ($this->params['named']['scope'] === 'not_over_correlating') {
+            } else if ($this->request->params['named']['scope'] === 'not_over_correlating') {
                 $query['conditions'][] = ['occurrence <' => $limit];
+                $scope = 'not_over_correlating';
             }
+        } else {
+            $scope = 'all';
         }
         $data = $this->Correlation->OverCorrelatingValue->getOverCorrelations($query);
         $data = $this->Correlation->attachExclusionsToOverCorrelations($data);
 
         if ($this->_isRest()) {
             return $this->RestResponse->viewData($data, 'json');
-        } else {
+        }
+
         $this->__setPagingParams($query['page'], $query['limit'], count($data), 'named');
         $this->set('data', $data);
+        $this->set('scope', $scope);
         $this->set('title_for_layout', __('Index of over correlating values'));
         $this->set('menuData', [
             'menuList' => 'correlationExclusions',
             'menuItem' => 'over'
         ]);
     }
-    }
 
     public function switchEngine(string $engine)
     {
@@ -205,17 +210,15 @@ class CorrelationsController extends AppController
     {
         $this->loadModel('OverCorrelatingValue');
         $this->OverCorrelatingValue->generateOccurrencesRouter();
-        $message = __('Job queued.');
         if (Configure::read('MISP.background_jobs')) {
             $message = __('Job queued.');
         } else {
             $message = __('Over-correlations counted successfully.');
         }
-        if (!$this->_isRest()) {
+        if ($this->_isRest()) {
-            $this->Flash->info($message);
-            $this->redirect(['controller' => 'correlations', 'action' => 'overCorrelations']);
-        } else {
             return $this->RestResponse->saveSuccessResponse('Correlations', 'generateOccurrences', false, $this->response->type(), $message);
         }
+        $this->Flash->info($message);
+        $this->redirect(['controller' => 'correlations', 'action' => 'overCorrelations']);
     }
 }

app/Controller/EventsController.php

@@ -999,13 +999,15 @@ class EventsController extends AppController
         }
 
         $possibleColumns[] = 'attribute_count';
-        $possibleColumns[] = 'timestamp';
-        $possibleColumns[] = 'publish_timestamp';
 
         if (Configure::read('MISP.showCorrelationsOnIndex')) {
             $possibleColumns[] = 'correlations';
         }
 
+        if (Configure::read('MISP.showEventReportCountOnIndex')) {
+            $possibleColumns[] = 'report_count';
+        }
+
         if (Configure::read('MISP.showSightingsCountOnIndex')) {
             $possibleColumns[] = 'sightings';
         }
@@ -1014,10 +1016,6 @@ class EventsController extends AppController
             $possibleColumns[] = 'proposals';
         }
 
-        if (Configure::read('MISP.showEventReportCountOnIndex')) {
-            $possibleColumns[] = 'report_count';
-        }
-
         if (Configure::read('MISP.showDiscussionsCountOnIndex')) {
             $possibleColumns[] = 'discussion';
         }
@@ -1026,6 +1024,9 @@ class EventsController extends AppController
             $possibleColumns[] = 'creator_user';
         }
 
+        $possibleColumns[] = 'timestamp';
+        $possibleColumns[] = 'publish_timestamp';
+
         $userDisabledColumns = $this->User->UserSetting->getValueForUser($this->Auth->user()['id'], 'event_index_hide_columns');
         if ($userDisabledColumns === null) {
             $userDisabledColumns = self::DEFAULT_HIDDEN_INDEX_COLUMNS;
@@ -4238,29 +4239,28 @@ class EventsController extends AppController
             throw new NotFoundException(__('Event not found or you are not authorised to view it.'));
         }
         $id = $event['Event']['id'];
-        // #TODO i18n
         $exports = array(
-            'xml' => array(
-                'url' => $this->baseurl . '/events/restSearch/xml/eventid:' . $id . '.xml',
-                'text' => 'MISP XML (metadata + all attributes)',
-                'requiresPublished' => false,
-                'checkbox' => true,
-                'checkbox_text' => 'Encode Attachments',
-                'checkbox_set' => $this->baseurl . '/events/restSearch/xml/eventid:' . $id . '/withAttachments:1.xml',
-                'checkbox_default' => true
-            ),
             'json' => array(
                 'url' => $this->baseurl . '/events/restSearch/json/eventid:' . $id . '.json',
-                'text' => 'MISP JSON (metadata + all attributes)',
+                'text' => __('MISP JSON (metadata + all attributes)'),
                 'requiresPublished' => false,
                 'checkbox' => true,
-                'checkbox_text' => 'Encode Attachments',
+                'checkbox_text' => __('Encode Attachments'),
                 'checkbox_set' => $this->baseurl . '/events/restSearch/json/withAttachments:1/eventid:' . $id . '.json',
-                'checkbox_default' => true
+                'checkbox_default' => true,
+            ),
+            'xml' => array(
+                'url' => $this->baseurl . '/events/restSearch/xml/eventid:' . $id . '.xml',
+                'text' => __('MISP XML (metadata + all attributes)'),
+                'requiresPublished' => false,
+                'checkbox' => true,
+                'checkbox_text' => __('Encode Attachments'),
+                'checkbox_set' => $this->baseurl . '/events/restSearch/xml/eventid:' . $id . '/withAttachments:1.xml',
+                'checkbox_default' => true,
             ),
             'openIOC' => array(
                 'url' => $this->baseurl . '/events/restSearch/openioc/to_ids:1/published:1/eventid:' . $id . '.json',
-                'text' => 'OpenIOC (all indicators marked to IDS)',
+                'text' => __('OpenIOC (all indicators marked to IDS)'),
                 'requiresPublished' => false,
                 'checkbox' => false,
             ),
@@ -4269,73 +4269,73 @@ class EventsController extends AppController
                 'text' => 'CSV',
                 'requiresPublished' => false,
                 'checkbox' => true,
-                'checkbox_text' => 'Include non-IDS marked attributes',
+                'checkbox_text' => __('Include non-IDS marked attributes'),
-                'checkbox_set' => $this->baseurl . '/events/restSearch/returnFormat:csv/to_ids:1||0/published:1||0/includeContext:0/eventid:' . $id
+                'checkbox_set' => $this->baseurl . '/events/restSearch/returnFormat:csv/to_ids:1||0/published:1||0/includeContext:0/eventid:' . $id,
             ),
             'csv_with_context' => array(
                 'url' => $this->baseurl . '/events/restSearch/returnFormat:csv/to_ids:1/published:1/includeContext:1/eventid:' . $id,
-                'text' => 'CSV with additional context',
+                'text' => __('CSV with additional context'),
                 'requiresPublished' => false,
                 'checkbox' => true,
-                'checkbox_text' => 'Include non-IDS marked attributes',
+                'checkbox_text' => __('Include non-IDS marked attributes'),
-                'checkbox_set' => $this->baseurl . '/events/restSearch/returnFormat:csv/to_ids:1||0/published:1||0/includeContext:1/eventid:' . $id
+                'checkbox_set' => $this->baseurl . '/events/restSearch/returnFormat:csv/to_ids:1||0/published:1||0/includeContext:1/eventid:' . $id,
             ),
             'stix_xml' => array(
                 'url' => $this->baseurl . '/events/restSearch/stix/eventid:' . $id,
-                'text' => 'STIX 1 XML (metadata + all attributes)',
+                'text' => __('STIX 1 XML (metadata + all attributes)'),
                 'requiresPublished' => false,
                 'checkbox' => true,
-                'checkbox_text' => 'Encode Attachments',
+                'checkbox_text' => __('Encode Attachments'),
-                'checkbox_set' => $this->baseurl . '/events/restSearch/stix/eventid:' . $id . '/withAttachments:1'
+                'checkbox_set' => $this->baseurl . '/events/restSearch/stix/eventid:' . $id . '/withAttachments:1',
             ),
             'stix_json' => array(
                 'url' => $this->baseurl . '/events/restSearch/stix-json/eventid:' . $id,
-                'text' => 'STIX 1 JSON (metadata + all attributes)',
+                'text' => __('STIX 1 JSON (metadata + all attributes)'),
                 'requiresPublished' => false,
                 'checkbox' => true,
-                'checkbox_text' => 'Encode Attachments',
+                'checkbox_text' => __('Encode Attachments'),
-                'checkbox_set' => $this->baseurl . '/events/restSearch/stix-json/withAttachments:1/eventid:' . $id
+                'checkbox_set' => $this->baseurl . '/events/restSearch/stix-json/withAttachments:1/eventid:' . $id,
             ),
             'stix2_json' => array(
                 'url' => $this->baseurl . '/events/restSearch/stix2/eventid:' . $id,
                 'text' => 'STIX 2',
                 'requiresPublished' => false,
                 'checkbox' => true,
-                'checkbox_text' => 'Encode Attachments',
+                'checkbox_text' => __('Encode Attachments'),
-                'checkbox_set' => $this->baseurl . '/events/restSearch/stix2/eventid:' . $id . '/withAttachments:1'
+                'checkbox_set' => $this->baseurl . '/events/restSearch/stix2/eventid:' . $id . '/withAttachments:1',
             ),
             'rpz' => array(
                 'url' => $this->baseurl . '/attributes/restSearch/returnFormat:rpz/published:1||0/eventid:' . $id,
-                'text' => 'RPZ Zone file',
+                'text' => __('RPZ Zone file'),
                 'requiresPublished' => false,
                 'checkbox' => false,
             ),
             'suricata' => array(
                 'url' => $this->baseurl . '/events/restSearch/returnFormat:suricata/published:1||0/eventid:' . $id,
-                'text' => 'Download Suricata rules',
+                'text' => __('Suricata rules'),
                 'requiresPublished' => false,
                 'checkbox' => false,
             ),
             'snort' => array(
                 'url' => $this->baseurl . '/events/restSearch/returnFormat:snort/published:1||0/eventid:' . $id,
-                'text' => 'Download Snort rules',
+                'text' => __('Snort rules'),
                 'requiresPublished' => false,
                 'checkbox' => false,
             ),
             'bro' => array(
                 'url' => $this->baseurl . '/attributes/bro/download/all/false/' . $id,
                 // 'url' => '/attributes/restSearch/returnFormat:bro/published:1||0/eventid:' . $id,
-                'text' => 'Download Bro rules',
+                'text' => __('Bro rules'),
                 'requiresPublished' => false,
-                'checkbox' => false
+                'checkbox' => false,
             ),
             'text' => array(
-                'text' => 'Export all attribute values as a text file',
+                'text' => __('Export all attribute values as a text file'),
                 'url' => $this->baseurl . '/attributes/restSearch/returnFormat:text/published:1||0/eventid:' . $id,
                 'requiresPublished' => false,
                 'checkbox' => true,
-                'checkbox_text' => 'Include non-IDS marked attributes',
+                'checkbox_text' => __('Include non-IDS marked attributes'),
-                'checkbox_set' => $this->baseurl . '/attributes/restSearch/returnFormat:text/published:1||0/to_ids:1||0/eventid:' . $id
+                'checkbox_set' => $this->baseurl . '/attributes/restSearch/returnFormat:text/published:1||0/to_ids:1||0/eventid:' . $id,
             ),
         );
         if ($event['Event']['published'] == 0) {
@@ -4346,9 +4346,9 @@ class EventsController extends AppController
             }
             $exports['csv'] = array(
                 'url' => $this->baseurl . '/events/restSearch/returnFormat:csv/includeContext:0/eventid:' . $id,
-                'text' => 'CSV (event not published, IDS flag ignored)',
+                'text' => __('CSV (event not published, IDS flag ignored)'),
                 'requiresPublished' => false,
-                'checkbox' => false
+                'checkbox' => false,
             );
         }
         $this->loadModel('Module');
@@ -4370,7 +4370,7 @@ class EventsController extends AppController
 
     public function importChoice($id = false, $scope = 'event')
     {
-        if ($scope == 'event') {
+        if ($scope === 'event') {
             $event = $this->Event->fetchSimpleEvent($this->Auth->user(), $id);
             if (empty($event)) {
                 throw new NotFoundException(__('Event not found or you are not authorised to view it.'));
@@ -4416,7 +4416,7 @@ class EventsController extends AppController
                     $imports[$module['name']] = array(
                         'url' => $this->baseurl . '/events/importModule/' . $module['name'] . '/' . $id,
                         'text' => Inflector::humanize($module['name']),
-                            'ajax' => false
+                        'ajax' => false,
                     );
                 }
             }
@@ -4426,7 +4426,7 @@ class EventsController extends AppController
                     'url' => $this->baseurl . '/events/add_misp_export',
                     'text' => __('MISP standard (recommended exchange format - lossless)'),
                     'ajax' => false,
-                        'bold' => true
+                    'bold' => true,
                 ),
                 'STIX' => array(
                     'url' => $this->baseurl . '/events/upload_stix',

app/Controller/GalaxyClustersController.php

@@ -165,7 +165,6 @@ class GalaxyClustersController extends AppController
      */
     public function view($id)
     {
-        $id = $this->Toolbox->findIdByUuid($this->GalaxyCluster, $id);
         $cluster = $this->GalaxyCluster->fetchIfAuthorized($this->Auth->user(), $id, 'view', $throwErrors=true, $full=true);
         $tag = $this->GalaxyCluster->Tag->find('first', array(
             'conditions' => array(
@@ -181,12 +180,13 @@ class GalaxyClustersController extends AppController
         }
         if ($this->_isRest()) {
             return $this->RestResponse->viewData($cluster, $this->response->type());
-        } else {
+        }
+
         $clusters = [$cluster];
         $this->GalaxyCluster->attachExtendByInfo($this->Auth->user(), $clusters);
         $cluster = $clusters[0];
         $cluster = $this->GalaxyCluster->attachExtendFromInfo($this->Auth->user(), $cluster);
-            $this->set('id', $id);
+        $this->set('id', $cluster['GalaxyCluster']['id']);
         $this->set('galaxy', ['Galaxy' => $cluster['GalaxyCluster']['Galaxy']]);
         $this->set('galaxy_id', $cluster['GalaxyCluster']['galaxy_id']);
         $this->set('cluster', $cluster);
@@ -203,7 +203,7 @@ class GalaxyClustersController extends AppController
         if (!$cluster['GalaxyCluster']['default'] && !$cluster['GalaxyCluster']['published'] && $cluster['GalaxyCluster']['orgc_id'] == $this->Auth->user()['org_id']) {
             $this->Flash->warning(__('This cluster is not published. Users will not be able to use it'));
         }
-        }
+        $this->set('title_for_layout', __('Galaxy cluster %s', $cluster['GalaxyCluster']['value']));
     }
 
     /**

app/Controller/ObjectReferencesController.php

@@ -23,7 +23,7 @@ class ObjectReferencesController extends AppController
             }
         }
         if (empty($objectId)) {
-            throw new MethodNotAllowedException('No object defined.');
+            throw new NotFoundException('No object defined.');
         }
         if (Validation::uuid($objectId)) {
             $conditions = ['Object.uuid' => $objectId];
@@ -91,7 +91,8 @@ class ObjectReferencesController extends AppController
         } else {
             if ($this->_isRest()) {
                 return $this->RestResponse->describe('ObjectReferences', 'add', false, $this->response->type());
-            } else {
+            }
+
             $events = $this->ObjectReference->Object->Event->find('all', array(
                 'conditions' => array(
                     'OR' => array(
@@ -119,9 +120,7 @@ class ObjectReferencesController extends AppController
                     )
                 )
             ));
-                if (!empty($events)) {
             $event = $events[0];
-                }
             for ($i = 1; $i < count($events); $i++) {
                 $event['Attribute'] = array_merge($event['Attribute'], $events[$i]['Attribute']);
                 $event['Object'] = array_merge($event['Object'], $events[$i]['Object']);
@@ -151,7 +150,6 @@ class ObjectReferencesController extends AppController
             $this->render('ajax/add');
         }
     }
-    }
 
     public function delete($id, $hard = false)
     {

app/Controller/ObjectsController.php

@@ -356,19 +356,20 @@ class ObjectsController extends AppController
 
     public function edit($id, $update_template_available=false, $onlyAddNewAttribute=false)
     {
-        $object = $this->MispObject->fetchObjects($this->Auth->user(), array(
+        $user = $this->Auth->user();
+        $object = $this->MispObject->fetchObjects($user, array(
             'conditions' => $this->__objectIdToConditions($id),
         ));
         if (empty($object)) {
             throw new NotFoundException(__('Invalid object.'));
         }
         $object = $object[0];
-        $event = $this->MispObject->Event->fetchSimpleEvent($this->Auth->user(), $object['Event']['id']);
+        $event = $this->MispObject->Event->fetchSimpleEvent($user, $object['Event']['id']);
         if (!$this->__canModifyEvent($event)) {
             throw new ForbiddenException(__('Insufficient permissions to edit this object.'));
         }
         if (!$this->_isRest()) {
-            $this->MispObject->Event->insertLock($this->Auth->user(), $object['Event']['id']);
+            $this->MispObject->Event->insertLock($user, $object['Event']['id']);
         }
         if (!empty($object['Object']['template_uuid']) && !empty($object['Object']['template_version'])) {
             $template = $this->MispObject->ObjectTemplate->find('first', array(
@@ -425,7 +426,7 @@ class ObjectsController extends AppController
                 unset($this->request->data['Object']);
             }
             $objectToSave = $this->MispObject->attributeCleanup($this->request->data);
-            $objectToSave = $this->MispObject->deltaMerge($object, $objectToSave, $onlyAddNewAttribute, $this->Auth->user());
+            $objectToSave = $this->MispObject->deltaMerge($object, $objectToSave, $onlyAddNewAttribute, $user);
             $error_message = __('Object could not be saved.');
             $savedObject = array();
             if (!is_numeric($objectToSave)) {
@@ -435,10 +436,10 @@ class ObjectsController extends AppController
                 }
                 $error_message = __('Object could not be saved.') . PHP_EOL . implode(PHP_EOL, $object_validation_errors);
             } else {
-                $savedObject = $this->MispObject->fetchObjects($this->Auth->user(), array('conditions' => array('Object.id' => $object['Object']['id'])));
+                $savedObject = $this->MispObject->fetchObjects($user, array('conditions' => array('Object.id' => $object['Object']['id'])));
                 if (isset($this->request->data['deleted']) && $this->request->data['deleted']) {
                     $this->MispObject->deleteObject($savedObject[0], $hard=false, $unpublish=false);
-                    $savedObject = $this->MispObject->fetchObjects($this->Auth->user(), array('conditions' => array('Object.id' => $object['Object']['id']))); // make sure the object is deleted
+                    $savedObject = $this->MispObject->fetchObjects($user, array('conditions' => array('Object.id' => $object['Object']['id']))); // make sure the object is deleted
                 }
             }
             // we pre-validate the attributes before we create an object at this point
@@ -480,15 +481,15 @@ class ObjectsController extends AppController
             $enabledRows = array();
             $this->request->data['Object'] = $object['Object'];
             foreach ($template['ObjectTemplateElement'] as $k => $element) {
-                foreach ($object['Attribute'] as $k2 => $attribute) {
+                foreach ($object['Attribute'] as $attribute) {
-                    if ($attribute['object_relation'] == $element['object_relation']) {
+                    if ($attribute['object_relation'] === $element['object_relation']) {
                         $enabledRows[] = $k;
                         $this->request->data['Attribute'][$k] = $attribute;
                         if (!empty($element['values_list'])) {
                             $this->request->data['Attribute'][$k]['value_select'] = $attribute['value'];
                         } else {
                             if (!empty($element['sane_default'])) {
-                                if (in_array($attribute['value'], $element['sane_default'])) {
+                                if (in_array($attribute['value'], $element['sane_default'], true)) {
                                     $this->request->data['Attribute'][$k]['value_select'] = $attribute['value'];
                                 } else {
                                     $this->request->data['Attribute'][$k]['value_select'] = 'Enter value manually';
@@ -500,7 +501,7 @@ class ObjectsController extends AppController
             }
         }
         $this->set('enabledRows', $enabledRows);
-        $distributionData = $this->MispObject->Event->Attribute->fetchDistributionData($this->Auth->user());
+        $distributionData = $this->MispObject->Event->Attribute->fetchDistributionData($user);
         $this->set('distributionData', $distributionData);
         $this->set('event', $event);
         $this->set('ajax', false);
@@ -646,10 +647,9 @@ class ObjectsController extends AppController
     // Construct a template with valid object attributes to add to an object
     public function quickFetchTemplateWithValidObjectAttributes($id)
     {
-        $fields = array('template_uuid', 'template_version', 'id');
         $params = array(
             'conditions' => array('Object.id' => $id),
-            'fields' => $fields,
+            'fields' => array('template_uuid', 'template_version', 'id'),
             'flatten' => 1,
         );
         // fetchObjects restrict access based on user
@@ -664,11 +664,10 @@ class ObjectsController extends AppController
             $object = $object[0];
         }
         // get object attributes already set
-        $objectRelation = array();
+        $existsObjectRelation = array();
         foreach ($object['Attribute'] as $attr) {
-            $objectRelation[$attr['object_relation']] = 1;
+            $existsObjectRelation[$attr['object_relation']] = true;
         }
-        $objectRelation = array_keys($objectRelation);
         // get object attribute defined in the object's template
         $template = $this->MispObject->ObjectTemplate->find('first', array(
             'conditions' => array(
@@ -688,7 +687,7 @@ class ObjectsController extends AppController
         }
         // unset object invalid object attribute
         foreach ($template['ObjectTemplateElement'] as $i => $objAttr) {
-            if (in_array($objAttr['object_relation'], $objectRelation) && !$objAttr['multiple']) {
+            if (isset($existsObjectRelation[$objAttr['object_relation']]) && !$objAttr['multiple']) {
                 unset($template['ObjectTemplateElement'][$i]);
             }
         }
@@ -762,13 +761,13 @@ class ObjectsController extends AppController
                 throw new NotFoundException(__('Invalid template'));
             }
             if (empty($template['ObjectTemplateElement'])) {
-                throw new NotFoundException(__('Invalid fields') . ' `' . h($fieldName) . '`');
+                throw new NotFoundException(__('Invalid field `%s`', h($fieldName)));
             }
 
             // check if fields can be added
             foreach ($object['Attribute'] as $objAttr) {
                 $objectAttrFromTemplate = $template['ObjectTemplateElement'][0];
-                if ($objAttr['object_relation'] == $fieldName && !$objectAttrFromTemplate['multiple']) {
+                if ($objAttr['object_relation'] === $fieldName && !$objectAttrFromTemplate['multiple']) {
                     throw new NotFoundException(__('Invalid field'));
                 }
             }
@@ -1130,16 +1129,17 @@ class ObjectsController extends AppController
         $this->set('unmapped', $unmappedAttributes);
     }
 
-    function proposeObjectsFromAttributes($event_id, $selected_attributes='[]')
+    public function proposeObjectsFromAttributes($eventId, $selectedAttributes='[]')
     {
         if (!$this->request->is('ajax')) {
             throw new MethodNotAllowedException(__('This action can only be reached via AJAX.'));
         }
-        $selected_attributes = json_decode($selected_attributes, true);
+
-        $res = $this->MispObject->validObjectsFromAttributeTypes($this->Auth->user(), $event_id, $selected_attributes);
+        $selectedAttributes = $this->_jsonDecode($selectedAttributes);
-        $potential_templates = $res['templates'];
+        $res = $this->MispObject->validObjectsFromAttributeTypes($this->Auth->user(), $eventId, $selectedAttributes);
-        $attribute_types = $res['types'];
+        $potentialTemplates = $res['templates'];
-        usort($potential_templates, function($a, $b) {
+        $attributeTypes = $res['types'];
+        usort($potentialTemplates, function($a, $b) {
             if ($a['ObjectTemplate']['id'] == $b['ObjectTemplate']['id']) {
                 return 0;
             } else if (is_array($a['ObjectTemplate']['compatibility']) && is_array($b['ObjectTemplate']['compatibility'])) {
@@ -1152,13 +1152,17 @@ class ObjectsController extends AppController
                 return count($a['ObjectTemplate']['invalidTypes']) > count($b['ObjectTemplate']['invalidTypes']) ? 1 : -1;
             }
         });
-        $this->set('potential_templates', $potential_templates);
+        $this->set('potential_templates', $potentialTemplates);
-        $this->set('selected_types', $attribute_types);
+        $this->set('selected_types', $attributeTypes);
-        $this->set('event_id', $event_id);
+        $this->set('event_id', $eventId);
     }
 
     public function groupAttributesIntoObject($event_id, $selected_template, $selected_attribute_ids='[]')
     {
+        if (!$this->request->is('ajax')) {
+            throw new MethodNotAllowedException(__('This action can only be reached via AJAX.'));
+        }
+
         $event = $this->MispObject->Event->find('first', array(
             'recursive' => -1,
             'fields' => array('Event.id', 'Event.uuid', 'Event.orgc_id', 'Event.user_id', 'Event.publish_timestamp'),
@@ -1171,9 +1175,6 @@ class ObjectsController extends AppController
             throw new ForbiddenException(__('You do not have permission to do that.'));
         }
         $hard_delete_attribute = $event['Event']['publish_timestamp'] == 0;
-        if (!$this->request->is('ajax')) {
-            throw new MethodNotAllowedException(__('This action can only be reached via AJAX.'));
-        }
         if ($this->request->is('post')) {
             $template = $this->MispObject->ObjectTemplate->find('first', array(
                 'recursive' => -1,
@@ -1183,22 +1184,22 @@ class ObjectsController extends AppController
                 throw new NotFoundException(__('Invalid template.'));
             }
             $distribution = $this->request->data['Object']['distribution'];
-            $sharing_group_id = $this->request->data['Object']['sharing_group_id'];
+            $sharingGroupId = $this->request->data['Object']['sharing_group_id'] ?? 0;
             $comment = $this->request->data['Object']['comment'];
-            $selected_attribute_ids = json_decode($this->request->data['Object']['selectedAttributeIds'], true);
+            $selected_attribute_ids = $this->_jsonDecode($this->request->data['Object']['selectedAttributeIds']);
-            $selected_object_relation_mapping = json_decode($this->request->data['Object']['selectedObjectRelationMapping'], true);
+            $selected_object_relation_mapping = $this->_jsonDecode($this->request->data['Object']['selectedObjectRelationMapping']);
             if ($distribution == 4) {
-                $sg = $this->MispObject->SharingGroup->fetchSG($sharing_group_id, $this->Auth->user());
+                $sg = $this->MispObject->SharingGroup->fetchSG($sharingGroupId, $this->Auth->user());
                 if (empty($sg)) {
                     throw new NotFoundException(__('Invalid sharing group.'));
                 }
             } else {
-                $sharing_group_id = 0;
+                $sharingGroupId = 0;
             }
             $object = array(
                 'Object' => array(
                     'distribution' => $distribution,
-                    'sharing_group_id' => $sharing_group_id,
+                    'sharing_group_id' => $sharingGroupId,
                     'comment' => $comment,
                 ),
                 'Attribute' => array()
@@ -1212,7 +1213,7 @@ class ObjectsController extends AppController
                 return $this->RestResponse->saveFailResponse('Objects', 'Created from Attributes', false, $error, $this->response->type());
             }
         } else {
-            $selected_attribute_ids = json_decode($selected_attribute_ids, true);
+            $selected_attribute_ids = $this->_jsonDecode($selected_attribute_ids);
             $selected_attributes = $this->MispObject->Attribute->fetchAttributes($this->Auth->user(), array('conditions' => array(
                 'Attribute.id' => $selected_attribute_ids,
                 'Attribute.event_id' => $event_id,
@@ -1232,7 +1233,7 @@ class ObjectsController extends AppController
             $conformity_result = $this->MispObject->ObjectTemplate->checkTemplateConformityBasedOnTypes($template, $selected_attributes);
             $skipped_attributes = 0;
             foreach ($selected_attributes as $i => $attribute) {
-                if (in_array($attribute['Attribute']['type'], $conformity_result['invalidTypes'])) {
+                if (in_array($attribute['Attribute']['type'], $conformity_result['invalidTypes'], true)) {
                     unset($selected_attributes[$i]);
                     $array_position = array_search($attribute['Attribute']['id'], $selected_attribute_ids);
                     unset($selected_attribute_ids[$array_position]);

app/Controller/UserSettingsController.php

@@ -213,7 +213,8 @@ class UserSettingsController extends AppController
         if ($this->_isRest()) {
             // GET request via the API should describe the endpoint
             return $this->RestResponse->describe('UserSettings', 'setSetting', false, $this->response->type());
-        } else {
+        }
+
         // load the valid settings from the model
         if ($this->_isSiteAdmin()) {
             $users = $this->UserSetting->User->find('list', array(
@@ -233,7 +234,7 @@ class UserSettingsController extends AppController
         $this->set('setting', $setting);
         $this->set('users', $users);
         $this->set('validSettings', $this->UserSetting->settingPlaceholders($this->Auth->user()));
-        }
+        $this->set('title_for_layout', __('Set User Setting'));
     }
 
     public function getSetting($userId = null, $setting = null)

app/Controller/UsersController.php

@@ -2766,21 +2766,12 @@ class UsersController extends AppController
 
     public function notificationSettings()
     {
-        $user_id = $this->Auth->user('id');
+        $user = $this->Auth->user();
-        $user = $this->User->find('first', [
-            'recursive' => -1,
-            'conditions' => ['User.id' => $user_id],
-            'contain' => [
-                'UserSetting',
-            ]
-        ]);
-        if (empty($user)) {
-            throw new NotFoundException(__('Invalid user'));
-        }
 
         if ($this->request->is('post') || $this->request->is('put')) {
-            $success = $this->User->saveNotificationSettings($user_id, $this->request->data);
+            $success = $this->User->saveNotificationSettings($user['id'], $this->request->data);
-            if (!empty($success)) {
+            if ($success) {
+                $this->_refreshAuth();
                 $message = __('Notification settings saved');
                 $this->Flash->success($message);
                 $this->redirect(['action' => 'view', 'me']);
@@ -2789,32 +2780,38 @@ class UsersController extends AppController
                 $this->Flash->error($message);
             }
         }
-        $user['periodic_settings'] = $this->User->extractPeriodicSettingForUser($user);
+
-        $this->request->data = $user;
+        $this->request->data = [
-        $this->set('user', $user);
+            'User' => $user,
+            'periodic_settings' => $this->User->fetchPeriodicSettingForUser($user['id']),
+        ];
         $this->loadModel('Attribute');
-        $distributionData = $this->Attribute->fetchDistributionData($this->Auth->user());
+        $distributionData = $this->Attribute->fetchDistributionData($user);
         unset($distributionData['levels'][5]);
         $this->set('sharingGroups', $distributionData['sgs']);
         $this->set('distributionLevels', $distributionData['levels']);
-        $this->loadModel('Organisation');
+
-        $orgs = $this->Organisation->find('list', [
+        $conditions = $this->User->Organisation->createConditions($user);
-            'conditions' => ['local' => 1],
+        $conditions['local'] = true;
+        $orgs = $this->User->Organisation->find('list', [
+            'conditions' => $conditions,
             'fields' => ['id', 'name'],
             'order' => 'name',
         ]);
         $this->set('orgs', $orgs);
         $this->set('user', $user);
+        $this->set('title_for_layout', __('Notification settings'));
     }
 
     public function viewPeriodicSummary(string $period)
     {
-        $summary = $this->User->generatePeriodicSummary($this->Auth->user('id'), $period);
+        $userId = $this->Auth->user('id');
-        $periodic_settings = $this->User->extractPeriodicSettingForUser($this->Auth->user('id'));
+        $summary = $this->User->generatePeriodicSummary($userId, $period);
-        $notification_settings = $this->User->getUsablePeriodicSettingForUser($periodic_settings, $period);
+        $periodicSettings = $this->User->fetchPeriodicSettingForUser($userId);
-        $this->set('periodic_settings', $periodic_settings);
+        $this->set('periodic_settings', $periodicSettings);
         $this->set('summary', $summary);
         $this->set('period', $period);
+        $this->set('title_for_layout', __('Periodic summary'));
     }
 
     private function __canChangePassword()

app/Lib/Export/AttackExport.php

@@ -127,7 +127,7 @@ class AttackExport
         $result['galaxyId'] = $this->__galaxy_id;
         $matrixGalaxies = $this->__GalaxyModel->getAllowedMatrixGalaxies();
         $result['matrixGalaxies'] = $matrixGalaxies;
-        return json_encode($result);
+        return JsonTool::encode($result);
     }
 
     public function separator()

app/Lib/Export/ContextExport.php

@@ -1,8 +1,6 @@
 <?php
 class ContextExport
 {
-    private $__attack_export_tool = null;
-
     public $additional_params = [
         'flatten' => 1,
         'includeEventTags' => 1,
@@ -12,21 +10,43 @@ class ContextExport
         'noShadowAttributes' => true,
         'sgReferenceOnly' => true,
         'includeEventCorrelations' => false,
+        'fetchFullClusters' => false,
     ];
+
     private $__eventTags = [];
+    /** @var array Tag name => Galaxy */
     private $__eventGalaxies = [];
 
     private $__aggregatedTags = [];
     private $__aggregatedClusters = [];
 
     private $__taxonomyFetched = [];
-    private $__galaxyFetched = [];
 
     private $__passedOptions = [];
 
     public $non_restrictive_export = true;
     public $renderView = 'context_view';
 
+    /** @var AttackExport */
+    private $AttackExport;
+
+    /** @var Taxonomy */
+    private $Taxonomy;
+
+    /** @var Galaxy */
+    private $Galaxy;
+
+    public function header($options = array())
+    {
+        $this->Taxonomy = ClassRegistry::init('Taxonomy');
+        $this->Galaxy = ClassRegistry::init('Galaxy');
+        App::uses('AttackExport', 'Export');
+        $this->AttackExport = new AttackExport();
+        $this->__passedOptions = $options;
+
+        return '';
+    }
+
     public function handler($data, $options = array())
     {
         $this->__aggregate($data, Hash::extract($data, 'EventTag.{n}.Tag'));
@@ -35,32 +55,20 @@ class ContextExport
                 $this->__aggregate($attribute, Hash::extract($attribute, 'AttributeTag.{n}.Tag'));
             }
         }
-        $this->__attack_export_tool->handler($data, $options);
+        $this->AttackExport->handler($data, $options);
-        return '';
-    }
-
-    public function header($options = array())
-    {
-        $this->__TaxonomyModel = ClassRegistry::init('Taxonomy');
-        $this->__GalaxyModel = ClassRegistry::init('Galaxy');
-        App::uses('AttackExport', 'Export');
-        $this->__attack_export_tool = new AttackExport();
-        $this->__attack_export_tool->handler($options);
-        $this->__passedOptions = $options;
-
         return '';
     }
 
     public function footer()
     {
-        $attackFinal = $this->__attack_export_tool->footer();
+        $attackFinal = $this->AttackExport->footer();
         $this->__aggregateTagsPerTaxonomy();
         $this->__aggregateClustersPerGalaxy();
-        $attackData = json_decode($attackFinal, true);
+        $attackData = $attackFinal === '' ? [] : JsonTool::decode($attackFinal);
         if (!empty($this->__passedOptions['filters']['staticHtml'])) {
             $attackData['static'] = true;
         }
-        return json_encode([
+        return JsonTool::encode([
             'attackData' => $attackData,
             'tags' => $this->__aggregatedTags,
             'clusters' => $this->__aggregatedClusters,
@@ -69,17 +77,15 @@ class ContextExport
 
     public function separator()
     {
-        $this->__attack_export_tool->separator();
         return '';
     }
 
-    private function __aggregate($entity, $tags)
+    private function __aggregate(array $entity, array $tags)
     {
         if (!empty($entity['Galaxy'])) {
             foreach ($entity['Galaxy'] as $galaxy) {
                 foreach ($galaxy['GalaxyCluster'] as $galaxyCluster) {
                     $this->__eventGalaxies[$galaxyCluster['tag_name']] = $galaxyCluster;
-                    $this->fetchGalaxyForTag($galaxyCluster['tag_name']);
                 }
             }
         }
@@ -94,42 +100,50 @@ class ContextExport
         }
     }
 
-    private function fetchTaxonomyForTag($tagname)
+    /**
+     * @param string $tagName
+     * @return void
+     * @throws RedisException
+     */
+    private function fetchTaxonomyForTag($tagName)
     {
-        $splits = $this->__TaxonomyModel->splitTagToComponents($tagname);
+        $splits = $this->Taxonomy->splitTagToComponents($tagName);
+        if ($splits === null) {
+            return; // tag is not taxonomy tag
+        }
         if (!isset($this->__taxonomyFetched[$splits['namespace']])) {
-            $fetchedTaxonomy = $this->__TaxonomyModel->getTaxonomyForTag($tagname, false, true);
+            $fetchedTaxonomy = $this->Taxonomy->getTaxonomyForTag($tagName, false, true);
             if (!empty($fetchedTaxonomy)) {
-                $this->__taxonomyFetched[$splits['namespace']]['Taxonomy'] = $fetchedTaxonomy['Taxonomy'];
+                $fetched = [
-                $this->__taxonomyFetched[$splits['namespace']]['TaxonomyPredicate'] = [];
+                    'Taxonomy' => $fetchedTaxonomy['Taxonomy'],
+                    'TaxonomyPredicate' => [],
+                ];
                 foreach ($fetchedTaxonomy['TaxonomyPredicate'] as $predicate) {
-                    $this->__taxonomyFetched[$splits['namespace']]['TaxonomyPredicate'][$predicate['value']] = $predicate;
+                    $fetched['TaxonomyPredicate'][$predicate['value']] = $predicate;
                     if (!empty($predicate['TaxonomyEntry'])) {
-                        $this->__taxonomyFetched[$splits['namespace']]['TaxonomyPredicate'][$predicate['value']]['TaxonomyEntry'] = [];
+                        $fetched['TaxonomyPredicate'][$predicate['value']]['TaxonomyEntry'] = [];
                         foreach ($predicate['TaxonomyEntry'] as $entry) {
-                            $this->__taxonomyFetched[$splits['namespace']]['TaxonomyPredicate'][$predicate['value']]['TaxonomyEntry'][$entry['value']] = $entry;
+                            $fetched['TaxonomyPredicate'][$predicate['value']]['TaxonomyEntry'][$entry['value']] = $entry;
                         }
                     }
                 }
+                $this->__taxonomyFetched[$splits['namespace']] = $fetched;
+            } else {
+                // Do not try to fetch non existing taxonomy again
+                $this->__taxonomyFetched[$splits['namespace']] = false;
             }
         }
     }
 
-    private function fetchGalaxyForTag($tagname)
-    {
-        $splits = $this->__TaxonomyModel->splitTagToComponents($tagname);
-        $galaxy = $this->__GalaxyModel->find('first', array(
-            'recursive' => -1,
-            'conditions' => array('Galaxy.type' => $splits['predicate'])
-        ));
-        $this->__galaxyFetched[$splits['predicate']] = $galaxy;
-    }
-
     private function __aggregateTagsPerTaxonomy()
     {
         ksort($this->__eventTags);
         foreach ($this->__eventTags as $tagname => $tagData) {
-            $splits = $this->__TaxonomyModel->splitTagToComponents($tagname);
+            $splits = $this->Taxonomy->splitTagToComponents($tagname);
+            if ($splits === null) {
+                $this->__aggregatedTags['Custom Tags'][]['Tag'] = $tagData;
+                continue;
+            }
             $taxonomy = [];
             if (!empty($this->__taxonomyFetched[$splits['namespace']])) {
                 $taxonomy = $this->__taxonomyFetched[$splits['namespace']];
@@ -137,7 +151,7 @@ class ContextExport
             if (!empty($taxonomy['TaxonomyPredicate'][$splits['predicate']])) {
                 $predicate = $taxonomy['TaxonomyPredicate'][$splits['predicate']];
                 $entry = null;
-                if (!empty($splits['value'])) {
+                if (!empty($splits['value']) && isset($predicate['TaxonomyEntry'][$splits['value']])) {
                     $entry = $predicate['TaxonomyEntry'][$splits['value']];
                 }
                 unset($predicate['TaxonomyEntry']);
@@ -155,12 +169,24 @@ class ContextExport
 
     private function __aggregateClustersPerGalaxy()
     {
+        $galaxyTypes = [];
+        foreach ($this->__eventGalaxies as $tagName => $foo) {
+            $splits = $this->Taxonomy->splitTagToComponents($tagName);
+            $galaxyTypes[$splits['predicate']] = true;
+        }
+
+        $fetchedGalaxies = $this->Galaxy->find('all', [
+            'recursive' => -1,
+            'conditions' => array('Galaxy.type' => array_keys($galaxyTypes)),
+        ]);
+        $fetchedGalaxies = array_column(array_column($fetchedGalaxies, 'Galaxy'), null, 'type');
+
         ksort($this->__eventGalaxies);
-        foreach ($this->__eventGalaxies as $tagname => $cluster) {
+        foreach ($this->__eventGalaxies as $tagName => $cluster) {
-            $splits = $this->__TaxonomyModel->splitTagToComponents($tagname);
+            $splits = $this->Taxonomy->splitTagToComponents($tagName);
-            $galaxy = $this->__galaxyFetched[$splits['predicate']];
+            $galaxy = $fetchedGalaxies[$splits['predicate']];
             $this->__aggregatedClusters[$splits['predicate']][] = [
-                'Galaxy' => $galaxy['Galaxy'],
+                'Galaxy' => $galaxy,
                 'GalaxyCluster' => $cluster,
             ];
         }

app/Lib/Export/NidsExport.php

@@ -1,6 +1,6 @@
 <?php
 
-class NidsExport
+abstract class NidsExport
 {
     public $rules = array();
 
@@ -858,15 +858,16 @@ class NidsExport
         }
     }
 
+    /**
+     * @param array $attribute
+     * @return array|string[]
+     */
     public static function getIpPort($attribute)
     {
-        $ipport = array();
         if (strpos($attribute['type'], 'port') !== false) {
-            $ipport = explode('|', $attribute['value']);
+            return explode('|', $attribute['value']);
         } else {
-            $ipport[0] = $attribute['value'];
+            return [$attribute['value'], 'any'];
-            $ipport[1] = 'any';
+        }
-        }
-        return $ipport;
     }
 }

app/Lib/Tools/AttributeValidationTool.php

@@ -635,11 +635,14 @@ class AttributeValidationTool
     }
 
     /**
-     * @param $value
+     * @param string $value
      * @return bool
      */
     private static function isSsdeep($value)
     {
+        if (strpos($value, "\n") !== false) {
+            return false;
+        }
         $parts = explode(':', $value);
         if (count($parts) !== 3) {
             return false;

app/Lib/Tools/ColourPaletteTool.php

@@ -1,8 +1,10 @@
 <?php
 class ColourPaletteTool
 {
-
+    /**
-    // pass the number of distinct colours to receive an array of colours
+     * @param int $count Pass the number of distinct colours to receive an array of colours
+     * @return array
+     */
     public function createColourPalette($count)
     {
         $interval = 1 / $count;
@@ -13,6 +15,10 @@ class ColourPaletteTool
         return $colours;
     }
 
+    /**
+     * @param array $hsv
+     * @return string
+     */
     public function HSVtoRGB(array $hsv)
     {
         list($H, $S, $V) = $hsv;
@@ -50,12 +56,16 @@ class ColourPaletteTool
         return $this->convertToHex(array($R, $G, $B));
     }
 
+    /**
+     * @param array $channels
+     * @return string
+     */
     public function convertToHex($channels)
     {
         $colour = '#';
         foreach ($channels as $channel) {
-            $channel = strval(dechex(round($channel*255)));
+            $channel = dechex(round($channel*255));
-            if (strlen($channel) == 1) {
+            if (strlen($channel) === 1) {
                 $channel = '0' . $channel;
             }
             $colour .= $channel;

app/Lib/Tools/HttpSocketExtended.php

@@ -65,6 +65,14 @@ class HttpSocketJsonException extends Exception
 
 class HttpSocketResponseExtended extends HttpSocketResponse
 {
+    /**
+     * @return bool
+     */
+    public function isNotModified()
+    {
+        return $this->code == 304;
+    }
+
     /**
      * @param string $message
      * @throws SocketException

app/Lib/Tools/RedisTool.php

@@ -0,0 +1,84 @@
+<?php
+class RedisTool
+{
+    /** @var Redis|null */
+    private static $connection;
+
+    /** @var string */
+    private static $serializer;
+
+    /**
+     * @return Redis
+     * @throws Exception
+     */
+    public static function init()
+    {
+        if (self::$connection) {
+            return self::$connection;
+        }
+
+        if (!class_exists('Redis')) {
+            throw new Exception("Class Redis doesn't exists. Please install redis extension for PHP.");
+        }
+
+        $host = Configure::read('MISP.redis_host') ?: '127.0.0.1';
+        $port = Configure::read('MISP.redis_port') ?: 6379;
+        $database = Configure::read('MISP.redis_database') ?: 13;
+        $pass = Configure::read('MISP.redis_password');
+
+        $redis = new Redis();
+        if (!$redis->connect($host, (int) $port)) {
+            throw new Exception("Could not connect to Redis: {$redis->getLastError()}");
+        }
+        if (!empty($pass)) {
+            if (!$redis->auth($pass)) {
+                throw new Exception("Could not authenticate to Redis: {$redis->getLastError()}");
+            }
+        }
+        if (!$redis->select($database)) {
+            throw new Exception("Could not select Redis database $database: {$redis->getLastError()}");
+        }
+        self::$connection = $redis;
+        return $redis;
+    }
+
+    /**
+     * @param mixed $data
+     * @return string
+     * @throws JsonException
+     */
+    public static function serialize($data)
+    {
+        if (self::$serializer === null) {
+            self::$serializer = Configure::read('MISP.redis_serializer') ?: false;
+        }
+
+        if (self::$serializer === 'igbinary') {
+            return igbinary_serialize($data);
+        } else {
+            return JsonTool::encode($data);
+        }
+    }
+
+    /**
+     * @param string $string
+     * @return mixed
+     * @throws JsonException
+     */
+    public static function deserialize($string)
+    {
+        if ($string === false) {
+            return false;
+        }
+
+        if (self::$serializer === null) {
+            self::$serializer = Configure::read('MISP.redis_serializer') ?: false;
+        }
+
+        if (self::$serializer === 'igbinary') {
+            return igbinary_unserialize($string);
+        } else {
+            return JsonTool::decode($string);
+        }
+    }
+}

app/Lib/Tools/SendEmailTemplate.php

@@ -24,7 +24,7 @@ class SendEmailTemplate
     /**
      * This value will be used for grouping emails in mail client.
      * @param string|null $referenceId
-     * @return string
+     * @return string|void
      */
     public function referenceId($referenceId = null)
     {
@@ -49,7 +49,7 @@ class SendEmailTemplate
     /**
      * Get subject from template. Must be called after render method.
      * @param string|null $subject
-     * @return string
+     * @return string|void
      */
     public function subject($subject = null)
     {
@@ -84,7 +84,6 @@ class SendEmailTemplate
         $View->set($this->viewVars);
         $View->set('hideDetails', $hideDetails);
 
-        $View->viewPath = $View->layoutPath = 'Emails' . DS . 'html';
         try {
             $View->viewPath = $View->layoutPath = 'Emails' . DS . 'html' . DS . 'Custom';
             $html = $View->render($this->viewName); // Attempt to load a custom template if it exists
@@ -93,7 +92,7 @@ class SendEmailTemplate
             try {
                 $html = $View->render($this->viewName);
             } catch (MissingViewException $e) {
-                $html = null; // HTMl template is optional
+                $html = null; // HTML template is optional
             }
         }
 

app/Lib/Tools/ServerSyncTool.php

@@ -72,12 +72,14 @@ class ServerSyncTool
 
     /**
      * @param array $params
+     * @param string|null $etag
+     * @return HttpSocketResponseExtended
      * @throws HttpSocketHttpException
      * @throws HttpSocketJsonException
      */
-    public function eventIndex($params = [])
+    public function eventIndex($params = [], $etag = null)
     {
-        return $this->post('/events/index', $params);
+        return $this->post('/events/index', $params, null, $etag);
     }
 
     /**
@@ -420,11 +422,13 @@ class ServerSyncTool
      * @param string $url Relative URL
      * @param mixed $data
      * @param string|null $logMessage
+     * @param string|null $etag
      * @return HttpSocketResponseExtended
      * @throws HttpSocketHttpException
      * @throws HttpSocketJsonException
+     * @throws JsonException
      */
-    private function post($url, $data, $logMessage = null)
+    private function post($url, $data, $logMessage = null, $etag = null)
     {
         $protectedMode = !empty($data['Event']['protected']);
         $data = JsonTool::encode($data);
@@ -445,6 +449,16 @@ class ServerSyncTool
             $request['header']['x-pgp-signature'] = $this->signEvent($data);
         }
 
+        if ($etag) {
+            // Remove compression marks that adds Apache for compressed content
+            $etagWithoutQuotes = trim($etag, '"');
+            $dashPos = strrpos($etagWithoutQuotes, '-');
+            if ($dashPos && in_array(substr($etagWithoutQuotes, $dashPos + 1), ['br', 'gzip'], true)) {
+                $etag = '"' . substr($etagWithoutQuotes, 0, $dashPos) . '"';
+            }
+            $request['header']['If-None-Match'] = $etag;
+        }
+
         if (strlen($data) > 1024) { // do not compress small body
             if ($this->isSupported(self::FEATURE_BR) && function_exists('brotli_compress')) {
                 $request['header']['Content-Encoding'] = 'br';
@@ -458,6 +472,9 @@ class ServerSyncTool
         $start = microtime(true);
         $response = $this->socket->post($url, $data, $request);
         $this->log($start, 'POST', $url, $response);
+        if ($etag && $response->isNotModified()) {
+            return $response; // if etag was provided and response code is 304, it is valid response
+        }
         if (!$response->isOk()) {
             throw new HttpSocketHttpException($response, $url);
         }
@@ -515,7 +532,7 @@ class ServerSyncTool
     }
 
     /**
-     * @param float $start
+     * @param float $start Microtime when request was send
      * @param string $method HTTP method
      * @param string $url
      * @param HttpSocketResponse $response
@@ -525,7 +542,7 @@ class ServerSyncTool
         $duration = round(microtime(true) - $start, 3);
         $responseSize = strlen($response->body);
         $ce = $response->getHeader('Content-Encoding');
-        $logEntry = '[' . date("Y-m-d H:i:s") . "] \"$method $url\" {$response->code} $responseSize $duration $ce\n";
+        $logEntry = '[' . date('Y-m-d H:i:s', intval($start)) . "] \"$method $url\" {$response->code} $responseSize $duration $ce\n";
         file_put_contents(APP . 'tmp/logs/server-sync.log', $logEntry, FILE_APPEND | LOCK_EX);
     }
 }

app/Lib/Tools/TrendingTool.php

@@ -44,7 +44,7 @@ class TrendingTool
                 }
                 $allTags[$tag] = true;
                 $trendAnalysis[$timestamp][$tag] = [
-                    'occurence' => round($amount / $eventNumberPerRollingWindow[$timestamp], 2),
+                    'occurrence' => round($amount / $eventNumberPerRollingWindow[$timestamp], 2),
                     'raw_change' => $rawChange,
                     'percent_change' => $percentChange,
                     'change_sign' => $rawChange > 0 ? 1 : ($rawChange < 0 ? -1 : 0),
@@ -54,9 +54,9 @@ class TrendingTool
                 foreach (array_keys($trendAnalysis[$timestamp]) as $tag) {
                     if (empty($trendAnalysis[$previousTimestamp][$tag])) {
                         $trendAnalysis[$previousTimestamp][$tag] = [
-                            'occurence' => 0,
+                            'occurrence' => 0,
                             'raw_change' => -$amount,
-                            'percent_change' => 100 * (-$amount / $amount),
+                            'percent_change' => round(100 * (-$amount / $amount), 2),
                             'change_sign' => -$amount > 0 ? 1 : (-$amount < 0 ? -1 : 0),
                         ];
                     }

app/Model/AppModel.php

@@ -25,6 +25,7 @@ App::uses('LogableBehavior', 'Assets.models/behaviors');
 App::uses('RandomTool', 'Tools');
 App::uses('FileAccessTool', 'Tools');
 App::uses('JsonTool', 'Tools');
+App::uses('RedisTool', 'Tools');
 App::uses('BetterCakeEventManager', 'Tools');
 
 class AppModel extends Model
@@ -38,9 +39,6 @@ class AppModel extends Model
     /** @var BackgroundJobsTool */
     private static $loadedBackgroundJobsTool;
 
-    /** @var null|Redis */
-    private static $__redisConnection;
-
     private $__profiler = array();
 
     public $elasticSearchClient;
@@ -48,6 +46,9 @@ class AppModel extends Model
     /** @var AttachmentTool|null */
     private $attachmentTool;
 
+    /** @var Workflow|null */
+    private $Workflow;
+
     // deprecated, use $db_changes
     // major -> minor -> hotfix -> requires_logout
     const OLD_DB_CHANGES = array(
@@ -83,7 +84,7 @@ class AppModel extends Model
         75 => false, 76 => true, 77 => false, 78 => false, 79 => false, 80 => false,
         81 => false, 82 => false, 83 => false, 84 => false, 85 => false, 86 => false,
         87 => false, 88 => false, 89 => false, 90 => false, 91 => false, 92 => false,
-        93 => false, 94 => false, 95 => true, 96 => false, 97 => true,
+        93 => false, 94 => false, 95 => true, 96 => false, 97 => true, 98 => false,
     );
 
     const ADVANCED_UPDATES_DESCRIPTION = array(
@@ -1873,6 +1874,9 @@ class AppModel extends Model
                     ADD COLUMN `notification_monthly`   tinyint(1) NOT NULL DEFAULT 0
                 ;";
                 break;
+            case 98:
+                $this->__addIndex('object_template_elements', 'object_template_id');
+                break;
             case 'fixNonEmptySharingGroupID':
                 $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
                 $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
@@ -2621,7 +2625,7 @@ class AppModel extends Model
         return $remainingTime > 0 || $failThresholdReached;
     }
 
-    public function getUpdateFailNumber()
+    private function getUpdateFailNumber()
     {
         $this->AdminSetting = ClassRegistry::init('AdminSetting');
         $updateFailNumber = $this->AdminSetting->getSetting('update_fail_number');
@@ -2634,7 +2638,7 @@ class AppModel extends Model
         $this->AdminSetting->changeSetting('update_fail_number', 0);
     }
 
-    public function __increaseUpdateFailNumber()
+    private function __increaseUpdateFailNumber()
     {
         $this->AdminSetting = ClassRegistry::init('AdminSetting');
         $updateFailNumber = $this->AdminSetting->getSetting('update_fail_number');
@@ -2736,7 +2740,7 @@ class AppModel extends Model
         return true;
     }
 
-    public function removeDuplicatedUUIDs()
+    private function removeDuplicatedUUIDs()
     {
         $removedResults = array(
             'Event' => $this->removeDuplicateEventUUIDs(),
@@ -2781,7 +2785,7 @@ class AppModel extends Model
         return $counter;
     }
 
-    public function removeDuplicateAttributeUUIDs()
+    private function removeDuplicateAttributeUUIDs()
     {
         $this->Attribute = ClassRegistry::init('Attribute');
         $this->Log = ClassRegistry::init('Log');
@@ -2835,7 +2839,7 @@ class AppModel extends Model
         return $counter;
     }
 
-    public function removeDuplicateEventUUIDs()
+    private function removeDuplicateEventUUIDs()
     {
         $this->Event = ClassRegistry::init('Event');
         $this->Log = ClassRegistry::init('Log');
@@ -2882,37 +2886,11 @@ class AppModel extends Model
      * Similar method as `setupRedis`, but this method throw exception if Redis cannot be reached.
      * @return Redis
      * @throws Exception
+     * @deprecated
      */
     public function setupRedisWithException()
     {
-        if (self::$__redisConnection) {
+        return RedisTool::init();
-            return self::$__redisConnection;
-        }
-
-        if (!class_exists('Redis')) {
-            throw new Exception("Class Redis doesn't exists. Please install redis extension for PHP.");
-        }
-
-        $host = Configure::read('MISP.redis_host') ?: '127.0.0.1';
-        $port = Configure::read('MISP.redis_port') ?: 6379;
-        $database = Configure::read('MISP.redis_database') ?: 13;
-        $pass = Configure::read('MISP.redis_password');
-
-        $redis = new Redis();
-        if (!$redis->connect($host, (int) $port)) {
-            throw new Exception("Could not connect to Redis: {$redis->getLastError()}");
-        }
-        if (!empty($pass)) {
-            if (!$redis->auth($pass)) {
-                throw new Exception("Could not authenticate to Redis: {$redis->getLastError()}");
-            }
-        }
-        if (!$redis->select($database)) {
-            throw new Exception("Could not select Redis database $database: {$redis->getLastError()}");
-        }
-
-        self::$__redisConnection = $redis;
-        return $redis;
     }
 
     /**
@@ -2924,7 +2902,7 @@ class AppModel extends Model
     public function setupRedis()
     {
         try {
-            return $this->setupRedisWithException();
+            return RedisTool::init();
         } catch (Exception $e) {
             return false;
         }
@@ -3282,7 +3260,7 @@ class AppModel extends Model
         return $filter;
     }
 
-    public function convert_to_memory_limit_to_mb($val)
+    protected function convert_to_memory_limit_to_mb($val)
     {
         $val = trim($val);
         if ($val == -1) {
@@ -3736,22 +3714,6 @@ class AppModel extends Model
         return $dataSourceName === 'Database/Mysql' || $dataSourceName === 'Database/MysqlObserver' || $dataSourceName === 'Database/MysqlExtended' || $dataSource instanceof Mysql;
     }
 
-    public function getCorrelationModelName()
-    {
-        if (!empty(Configure::read('MISP.correlation_engine'))) {
-            return Configure::read('MISP.correlation_engine');
-        }
-        return 'Default';
-    }
-
-    public function loadCorrelationModel()
-    {
-        if (!empty(Configure::read('MISP.correlation_engine'))) {
-            return ClassRegistry::init(Configure::read('MISP.correlation_engine'));
-        }
-        return ClassRegistry::init('Correlation');
-    }
-
     /**
      * executeTrigger
      *
@@ -3763,9 +3725,6 @@ class AppModel extends Model
      */
     public function executeTrigger($trigger_id, array $data=[], array &$blockingErrors=[], array $logging=[]): bool
     {
-        if ($this->Workflow === null) {
-            $this->Workflow = ClassRegistry::init('Workflow');
-        }
         if ($this->isTriggerCallable($trigger_id)) {
            $success = $this->Workflow->executeWorkflowForTriggerRouter($trigger_id, $data, $blockingErrors, $logging);
            if (!empty($logging) && empty($success)) {
@@ -3787,13 +3746,6 @@ class AppModel extends Model
             $this->Workflow->checkTriggerListenedTo($trigger_id);
     }
 
-    public function addPendingLogEntry($logEntry)
-    {
-        $logEntries = Configure::read('pendingLogEntries');
-        $logEntries[] = $logEntry;
-        Configure::write('pendingLogEntries', $logEntries);
-    }
-
     /**
      * Use different CakeEventManager to fix memory leak
      * @return CakeEventManager
@@ -3808,7 +3760,8 @@ class AppModel extends Model
         return $this->_eventManager;
     }
 
-    private function __retireOldCorrelationEngine($user = null) {
+    private function __retireOldCorrelationEngine($user = null)
+    {
         if ($user === null) {
             $user = [
                 'id' => 0,

app/Model/Attribute.php

@@ -107,18 +107,19 @@ class Attribute extends AppModel
     const UPLOAD_DEFINITIONS = ['attachment'];
 
     // skip Correlation for the following types
-    const NON_CORRELATING_TYPES = array(
+    const NON_CORRELATING_TYPES = [
         'comment',
         'http-method',
         'aba-rtn',
         'gender',
         'counter',
+        'float',
         'port',
         'nationality',
         'cortex',
         'boolean',
         'anonymised'
-    );
+    ];
 
     const PRIMARY_ONLY_CORRELATING_TYPES = array(
         'ip-src|port',
@@ -509,21 +510,26 @@ class Attribute extends AppModel
     public function beforeDelete($cascade = true)
     {
         // delete attachments from the disk
-        $this->read(); // first read the attribute from the db
+        $attribute = $this->find('first', [
-        if ($this->typeIsAttachment($this->data['Attribute']['type'])) {
+            'recursive' => -1,
-            $this->loadAttachmentTool()->delete($this->data['Attribute']['event_id'], $this->data['Attribute']['id']);
+            'conditions' => [
+                'id' => $this->id,
+            ]
+        ]);
+        if ($this->typeIsAttachment($attribute['Attribute']['type'])) {
+            $this->loadAttachmentTool()->delete($attribute['Attribute']['event_id'], $attribute['Attribute']['id']);
         }
         // update correlation..
-        $this->Correlation->beforeSaveCorrelation($this->data['Attribute']);
+        $this->Correlation->beforeSaveCorrelation($attribute['Attribute']);
-        if (!empty($this->data['Attribute']['id'])) {
+        if (!empty($attribute['Attribute']['id'])) {
             if ($this->pubToZmq('attribute')) {
                 $pubSubTool = $this->getPubSubTool();
-                $pubSubTool->attribute_save($this->data, 'delete');
+                $pubSubTool->attribute_save($attribute, 'delete');
             }
             $kafkaTopic = $this->kafkaTopic('attribute');
             if ($kafkaTopic) {
                 $kafkaPubTool = $this->getKafkaPubTool();
-                $kafkaPubTool->publishJson($kafkaTopic, $this->data, 'delete');
+                $kafkaPubTool->publishJson($kafkaTopic, $attribute, 'delete');
             }
         }
     }
@@ -902,7 +908,7 @@ class Attribute extends AppModel
         if ($maxWidth == $defaultMaxSize && $maxHeight == $defaultMaxSize) {
             $thumbnailInRedis = Configure::read('MISP.thumbnail_in_redis');
             if ($thumbnailInRedis) {
-                $redis = $this->setupRedisWithException();
+                $redis = RedisTool::init();
                 if ($data = $redis->get("misp:thumbnail:attribute:{$attribute['Attribute']['id']}:$outputFormat")) {
                     return $data;
                 }
@@ -924,7 +930,7 @@ class Attribute extends AppModel
         // Save just when requested default thumbnail size
         if ($maxWidth == $defaultMaxSize && $maxHeight == $defaultMaxSize) {
             if ($thumbnailInRedis) {
-                $redis->set("misp:thumbnail:attribute:{$attribute['Attribute']['id']}:$outputFormat", $imageData, 3600);
+                $redis->setex("misp:thumbnail:attribute:{$attribute['Attribute']['id']}:$outputFormat", 3600, $imageData);
             } else {
                 $this->loadAttachmentTool()->save($attribute['Attribute']['event_id'], $attribute['Attribute']['id'], $imageData, $suffix);
             }
@@ -1018,163 +1024,6 @@ class Attribute extends AppModel
         return $data;
     }
 
-    public function hids($user, $type, $tags = '', $from = false, $to = false, $last = false, $jobId = false, $enforceWarninglist = false)
-    {
-        if (empty($user)) {
-            throw new MethodNotAllowedException(__('Could not read user.'));
-        }
-        // check if it's a valid type
-        if ($type != 'md5' && $type != 'sha1' && $type != 'sha256') {
-            throw new UnauthorizedException(__('Invalid hash type.'));
-        }
-        $conditions = array();
-        $typeArray = array($type, 'filename|' . $type);
-        if ($type == 'md5') {
-            $typeArray[] = 'malware-sample';
-        }
-        $rules = array();
-        $eventIds = $this->Event->fetchEventIds($user, [
-            'from' => $from,
-            'to' => $to,
-            'last' => $last
-        ]);
-        if (!empty($tags)) {
-            $tag = ClassRegistry::init('Tag');
-            $args = $this->dissectArgs($tags);
-            $tagArray = $tag->fetchEventTagIds($args[0], $args[1]);
-            if (!empty($tagArray[0])) {
-                foreach ($eventIds as $k => $v) {
-                    if (!in_array($v['Event']['id'], $tagArray[0])) {
-                        unset($eventIds[$k]);
-                    }
-                }
-            }
-            if (!empty($tagArray[1])) {
-                foreach ($eventIds as $k => $v) {
-                    if (in_array($v['Event']['id'], $tagArray[1])) {
-                        unset($eventIds[$k]);
-                    }
-                }
-            }
-        }
-        App::uses('HidsExport', 'Export');
-        $continue = false;
-        $eventCount = count($eventIds);
-        if ($jobId) {
-            $this->Job = ClassRegistry::init('Job');
-            $this->Job->id = $jobId;
-            if (!$this->Job->exists()) {
-                $jobId = false;
-            }
-        }
-        foreach ($eventIds as $k => $event) {
-            $conditions['AND'] = array('Attribute.to_ids' => 1, 'Event.published' => 1, 'Attribute.type' => $typeArray, 'Attribute.event_id' => $event['Event']['id']);
-            $options = array(
-                    'conditions' => $conditions,
-                    'group' => array('Attribute.type', 'Attribute.value1'),
-                    'enforceWarninglist' => $enforceWarninglist,
-                    'flatten' => true
-            );
-            $items = $this->fetchAttributes($user, $options);
-            if (empty($items)) {
-                continue;
-            }
-            $export = new HidsExport();
-            $rules = array_merge($rules, $export->export($items, strtoupper($type), $continue));
-            $continue = true;
-            if ($jobId && ($k % 10 == 0)) {
-                $this->Job->saveField('progress', $k * 80 / $eventCount);
-            }
-        }
-        return $rules;
-    }
-
-
-    public function nids($user, $format, $id = false, $continue = false, $tags = false, $from = false, $to = false, $last = false, $type = false, $enforceWarninglist = false, $includeAllTags = false)
-    {
-        if (empty($user)) {
-            throw new MethodNotAllowedException(__('Could not read user.'));
-        }
-        $eventIds = $this->Event->fetchEventIds($user, [
-            'from' => $from,
-            'to' => $to,
-            'last' => $last
-        ]);
-
-        // If we sent any tags along, load the associated tag names for each attribute
-        if ($tags) {
-            $tag = ClassRegistry::init('Tag');
-            $args = $this->dissectArgs($tags);
-            $tagArray = $tag->fetchEventTagIds($args[0], $args[1]);
-            if (!empty($tagArray[0])) {
-                foreach ($eventIds as $k => $v) {
-                    if (!in_array($v['Event']['id'], $tagArray[0])) {
-                        unset($eventIds[$k]);
-                    }
-                }
-            }
-            if (!empty($tagArray[1])) {
-                foreach ($eventIds as $k => $v) {
-                    if (in_array($v['Event']['id'], $tagArray[1])) {
-                        unset($eventIds[$k]);
-                    }
-                }
-            }
-        }
-
-        if ($id) {
-            foreach ($eventIds as $k => $v) {
-                if ($v['Event']['id'] !== $id) {
-                    unset($eventIds[$k]);
-                }
-            }
-        }
-
-        if ($format == 'suricata') {
-            App::uses('NidsSuricataExport', 'Export');
-        } else {
-            App::uses('NidsSnortExport', 'Export');
-        }
-
-        $rules = array();
-        foreach ($eventIds as $event) {
-            $conditions['AND'] = array('Attribute.to_ids' => 1, "Event.published" => 1, 'Attribute.event_id' => $event['Event']['id']);
-            $valid_types = array('ip-dst', 'ip-src', 'ip-dst|port', 'ip-src|port', 'eppn', 'email', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'domain', 'domain|ip', 'hostname', 'url', 'user-agent', 'snort');
-            $conditions['AND']['Attribute.type'] = $valid_types;
-            if (!empty($type)) {
-                $conditions['AND'][] = array('Attribute.type' => $type);
-            }
-
-            $params = array(
-                    'conditions' => $conditions, // array of conditions
-                    'recursive' => -1, // int
-                    'fields' => array('Attribute.id', 'Attribute.event_id', 'Attribute.type', 'Attribute.value'),
-                    'contain' => array('Event'=> array('fields' => array('Event.id', 'Event.threat_level_id'))),
-                    'group' => array('Attribute.type', 'Attribute.value1'), // fields to GROUP BY
-                    'enforceWarninglist' => $enforceWarninglist,
-                    'includeAllTags' => $includeAllTags,
-                    'flatten' => true
-            );
-            $items = $this->fetchAttributes($user, $params);
-            if (empty($items)) {
-                continue;
-            }
-            // export depending on the requested type
-            switch ($format) {
-                case 'suricata':
-                    $export = new NidsSuricataExport();
-                    break;
-                case 'snort':
-                    $export = new NidsSnortExport();
-                    break;
-            }
-            $rules = array_merge($rules, $export->export($items, $user['nids_sid'], $format, $continue));
-            // Only prepend the comments once
-            $continue = true;
-        }
-        return $rules;
-    }
-
     public function set_filter_tags(&$params, $conditions, $options)
     {
         if (empty($params['tags']) && empty($params['event_tags'])) {
@@ -1309,212 +1158,6 @@ class Attribute extends AppModel
         return $conditions;
     }
 
-    public function text($user, $type, $tags = false, $eventId = false, $allowNonIDS = false, $from = false, $to = false, $last = false, $enforceWarninglist = false, $allowNotPublished = false)
-    {
-        //permissions are taken care of in fetchAttributes()
-        $conditions['AND'] = array();
-        if ($allowNonIDS === false) {
-            $conditions['AND']['Attribute.to_ids'] = 1;
-            if ($allowNotPublished === false) {
-                $conditions['AND']['Event.published'] = 1;
-            }
-        }
-        if (!is_array($type) && $type !== 'all') {
-            $conditions['AND']['Attribute.type'] = $type;
-        }
-        if ($from) {
-            $conditions['AND']['Event.date >='] = $from;
-        }
-        if ($to) {
-            $conditions['AND']['Event.date <='] = $to;
-        }
-        if ($last) {
-            $conditions['AND']['Event.publish_timestamp >='] = $last;
-        }
-
-        if ($eventId !== false) {
-            $conditions['AND'][] = array('Event.id' => $eventId);
-        } elseif ($tags !== false) {
-            $passed_param = array('tags' => $tags);
-            $conditions = $this->set_filter_tags($passed_param, $conditions, array('scope' => 'Attribute'));
-        }
-        $attributes = $this->fetchAttributes($user, array(
-                'conditions' => $conditions,
-                'order' => 'Attribute.value1 ASC',
-                'fields' => array('value'),
-                'contain' => array('Event' => array(
-                    'fields' => array('Event.id', 'Event.published', 'Event.date', 'Event.publish_timestamp'),
-                )),
-                'enforceWarninglist' => $enforceWarninglist,
-                'flatten' => 1
-        ));
-        return $attributes;
-    }
-
-    public function rpz($user, $tags = false, $eventId = false, $from = false, $to = false, $enforceWarninglist = false)
-    {
-        // we can group hostname and domain as well as ip-src and ip-dst in this case
-        $conditions['AND'] = array('Attribute.to_ids' => 1, 'Event.published' => 1);
-        $typesToFetch = array('ip' => array('ip-src', 'ip-dst'), 'domain' => array('domain'), 'hostname' => array('hostname'));
-        if ($from) {
-            $conditions['AND']['Event.date >='] = $from;
-        }
-        if ($to) {
-            $conditions['AND']['Event.date <='] = $to;
-        }
-        if ($eventId !== false) {
-            $conditions['AND'][] = array('Event.id' => $eventId);
-        }
-        if ($tags !== false) {
-            // If we sent any tags along, load the associated tag names for each attribute
-            $tag = ClassRegistry::init('Tag');
-            $args = $this->dissectArgs($tags);
-            $tagArray = $tag->fetchEventTagIds($args[0], $args[1]);
-            $temp = array();
-            foreach ($tagArray[0] as $accepted) {
-                $temp['OR'][] = array('Event.id' => $accepted);
-            }
-            $conditions['AND'][] = $temp;
-            $temp = array();
-            foreach ($tagArray[1] as $rejected) {
-                $temp['AND'][] = array('Event.id !=' => $rejected);
-            }
-            $conditions['AND'][] = $temp;
-        }
-        $values = array();
-        foreach ($typesToFetch as $k => $v) {
-            $tempConditions = $conditions;
-            $tempConditions['type'] = $v;
-            $temp = $this->fetchAttributes(
-                    $user,
-                    array(
-                        'conditions' => $tempConditions,
-                        'fields' => array('Attribute.value'), // array of field names
-                        'enforceWarninglist' => $enforceWarninglist,
-                        'flatten' => 1
-                    )
-            );
-            if (empty($temp)) {
-                continue;
-            }
-            if ($k == 'hostname') {
-                foreach ($temp as $value) {
-                    $found = false;
-                    if (isset($values['domain'])) {
-                        foreach ($values['domain'] as $domain) {
-                            if (strpos($value['Attribute']['value'], $domain) != 0) {
-                                $found = true;
-                            }
-                        }
-                    }
-                    if (!$found) {
-                        $values[$k][] = $value['Attribute']['value'];
-                    }
-                }
-            } else {
-                foreach ($temp as $value) {
-                    $values[$k][] = $value['Attribute']['value'];
-                }
-            }
-            unset($temp);
-        }
-        return $values;
-    }
-
-    public function bro($user, $type, $tags = false, $eventId = false, $from = false, $to = false, $last = false, $enforceWarninglist = false, $skipHeader = false)
-    {
-        App::uses('BroExport', 'Export');
-        $export = new BroExport();
-        if ($type == 'all') {
-            $types = array_keys($export->mispTypes);
-        } else {
-            $types = array($type);
-        }
-        $intel = array();
-        foreach ($types as $type) {
-            //restricting to non-private or same org if the user is not a site-admin.
-            $conditions['AND'] = array('Attribute.to_ids' => 1, 'Event.published' => 1);
-            if ($from) {
-                $conditions['AND']['Event.date >='] = $from;
-            }
-            if ($to) {
-                $conditions['AND']['Event.date <='] = $to;
-            }
-            if ($last) {
-                $conditions['AND']['Event.publish_timestamp >='] = $last;
-            }
-            if ($eventId !== false) {
-                $temp = array();
-                $args = $this->dissectArgs($eventId);
-                foreach ($args[0] as $accepted) {
-                    $temp['OR'][] = array('Event.id' => $accepted);
-                }
-                $conditions['AND'][] = $temp;
-                $temp = array();
-                foreach ($args[1] as $rejected) {
-                    $temp['AND'][] = array('Event.id !=' => $rejected);
-                }
-                $conditions['AND'][] = $temp;
-            }
-            if ($tags !== false) {
-                // If we sent any tags along, load the associated tag names for each attribute
-                $tag = ClassRegistry::init('Tag');
-                $args = $this->dissectArgs($tags);
-                $tagArray = $tag->fetchEventTagIds($args[0], $args[1]);
-                $temp = array();
-                foreach ($tagArray[0] as $accepted) {
-                    $temp['OR'][] = array('Event.id' => $accepted);
-                }
-                $conditions['AND'][] = $temp;
-                $temp = array();
-                foreach ($tagArray[1] as $rejected) {
-                    $temp['AND'][] = array('Event.id !=' => $rejected);
-                }
-                $conditions['AND'][] = $temp;
-            }
-            $this->Allowedlist = ClassRegistry::init('Allowedlist');
-            $this->allowedlist = $this->Allowedlist->getBlockedValues();
-            $instanceString = 'MISP';
-            if (Configure::read('MISP.host_org_id') && Configure::read('MISP.host_org_id') > 0) {
-                $this->Event->Orgc->id = Configure::read('MISP.host_org_id');
-                if ($this->Event->Orgc->exists()) {
-                    $instanceString = $this->Event->Orgc->field('name') . ' MISP';
-                }
-            }
-            $mispTypes = $export->getMispTypes($type);
-            foreach ($mispTypes as $mispType) {
-                $conditions['AND']['Attribute.type'] = $mispType[0];
-                $intel = array_merge($intel, $this->__bro($user, $conditions, $mispType[1], $export, $this->allowedlist, $instanceString, $enforceWarninglist));
-            }
-        }
-        natsort($intel);
-        $intel = array_unique($intel);
-        if (empty($skipHeader)) {
-            array_unshift($intel, $export->header);
-        }
-        return $intel;
-    }
-
-    private function __bro($user, $conditions, $valueField, $export, $allowedlist, $instanceString, $enforceWarninglist)
-    {
-        $attributes = $this->fetchAttributes(
-            $user,
-            array(
-                'conditions' => $conditions, // array of conditions
-                'order' => 'Attribute.value' . $valueField . ' ASC',
-                'recursive' => -1, // int
-                'fields' => array('Attribute.id', 'Attribute.event_id', 'Attribute.type', 'Attribute.category', 'Attribute.comment', 'Attribute.to_ids', 'Attribute.value', 'Attribute.value' . $valueField),
-                'contain' => array('Event' => array('fields' => array('Event.id', 'Event.threat_level_id', 'Event.orgc_id', 'Event.uuid'))),
-                'enforceWarninglist' => $enforceWarninglist,
-                'flatten' => 1
-            )
-        );
-        $orgs = $this->Event->Orgc->find('list', array(
-                'fields' => array('Orgc.id', 'Orgc.name')
-        ));
-        return $export->export($attributes, $orgs, $valueField, $allowedlist, $instanceString);
-    }
-
     /**
      * @param int|false $jobId
      * @param int|false $eventId
@@ -1561,7 +1204,8 @@ class Attribute extends AppModel
                 );
             }
         } else {
-            $attributeCount = $this->__iteratedCorrelation($jobId, $full, $attributeCount);
+            // Not sure why that line was added. If there are no events, there are no correlations to save
+            // $attributeCount = $this->__iteratedCorrelation($jobId, $full, $attributeCount);
         }
         if ($jobId) {
             $this->Job->saveStatus($jobId, true);
@@ -1581,7 +1225,7 @@ class Attribute extends AppModel
     {
         if ($jobId) {
             $message = $attributeId ? __('Correlating Attribute %s', $attributeId) : __('Correlating Event %s (%s MB used)', $eventId, intval(memory_get_usage() / 1024 / 1024));
-            $this->Job->saveProgress($jobId, $message, ($j / $eventCount) * 100);
+            $this->Job->saveProgress($jobId, $message, !empty($eventCount) ? ($j / $eventCount) * 100 : 0);
         }
         $attributeConditions = [
             'Attribute.deleted' => 0,
@@ -2149,13 +1793,13 @@ class Attribute extends AppModel
         if ($options['includeDecayScore']) {
             $options['includeEventTags'] = true;
         }
-        if (!$user['Role']['perm_sync'] || !isset($options['deleted']) || !$options['deleted']) {
+        if (isset($options['deleted'])) {
-            $params['conditions']['AND']['Attribute.deleted'] = 0;
-        } else {
             if ($options['deleted'] === "only") {
                 $options['deleted'] = 1;
             }
             $params['conditions']['AND']['(Attribute.deleted + 0)'] = $options['deleted'];
+        } elseif (!$user['Role']['perm_sync'] || !isset($options['deleted']) || !$options['deleted']) {
+            $params['conditions']['AND']['Attribute.deleted'] = 0;
         }
         if (isset($options['group'])) {
             $params['group'] = !empty($options['group']) ? $options['group'] : false;
@@ -2371,7 +2015,7 @@ class Attribute extends AppModel
 
         $tags = $this->AttributeTag->Tag->find('all', [
             'conditions' => $conditions,
-            'fields' => ['id', 'name', 'colour', 'numerical_value'],
+            'fields' => ['id', 'name', 'colour', 'numerical_value', 'is_galaxy'],
             'recursive' => -1,
         ]);
         $tags = array_column(array_column($tags, 'Tag'), null, 'id');

app/Model/Behavior/DefaultCorrelationBehavior.php

@@ -58,12 +58,18 @@ class DefaultCorrelationBehavior extends ModelBehavior
         return self::TABLE_NAME;
     }
 
+    /**
+     * @param Model $Model
+     * @param string $value
+     * @param array $a
+     * @param array $b
+     * @return array
+     */
     public function createCorrelationEntry(Model $Model, $value, $a, $b)
     {
-        $valueId = $this->Correlation->CorrelationValue->getValueId($value);
         if ($this->deadlockAvoidance) {
             return [
-                'value_id' => $valueId,
+                'value_id' => $value,
                 '1_event_id' => $a['Event']['id'],
                 '1_object_id' => $a['Attribute']['object_id'],
                 '1_attribute_id' => $a['Attribute']['id'],
@@ -87,7 +93,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
             ];
         } else {
             return [
-                (int) $valueId,
+                $value,
                 (int) $a['Event']['id'],
                 (int) $a['Attribute']['object_id'],
                 (int) $a['Attribute']['id'],
@@ -112,7 +118,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
         }
     }
 
-    public function saveCorrelations(Model $Model, $correlations)
+    public function saveCorrelations(Model $Model, array $correlations)
     {
         $fields = [
             'value_id',
@@ -138,14 +144,16 @@ class DefaultCorrelationBehavior extends ModelBehavior
             'object_sharing_group_id'
         ];
 
+        $this->Correlation->CorrelationValue->replaceValueWithId($correlations, $this->deadlockAvoidance ? 'value_id' : 0);
+
         if ($this->deadlockAvoidance) {
-          return $this->Correlation->saveMany($correlations, array(
+            return $this->Correlation->saveMany($correlations, [
                 'atomic' => false,
                 'callbacks' => false,
                 'deep' => false,
                 'validate' => false,
-                'fieldList' => $fields
+                'fieldList' => $fields,
-            ));
+            ]);
         } else {
             $db = $this->Correlation->getDataSource();
             // Split to chunks datasource is is enabled
@@ -196,7 +204,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
     /**
      * Fetch correlations for given event.
      * @param array $user
-     * @param int $eventId
+     * @param int|array $eventId
      * @param array $sgids
      * @param bool $primary
      * @return array
@@ -237,7 +245,6 @@ class DefaultCorrelationBehavior extends ModelBehavior
             'contain' => [
                 'CorrelationValue' => [
                     'fields' => [
-                        'CorrelationValue.id',
                         'CorrelationValue.value'
                     ]
                 ]
@@ -256,7 +263,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
     /**
      * @param Correlation $Model
      * @param array $user
-     * @param int $id Event ID
+     * @param int|array $id Event ID
      * @param array $sgids
      * @return array
      */
@@ -329,7 +336,6 @@ class DefaultCorrelationBehavior extends ModelBehavior
             [
                 '1_attribute_id',
                 '1_object_id',
-                '1_event_id',
                 '1_distribution',
                 '1_object_distribution',
                 '1_event_distribution',
@@ -337,12 +343,10 @@ class DefaultCorrelationBehavior extends ModelBehavior
                 '1_object_sharing_group_id',
                 '1_event_sharing_group_id',
                 '1_org_id',
-                'value_id'
             ],
             [
                 'attribute_id',
                 'object_id',
-                'event_id',
                 'distribution',
                 'object_distribution',
                 'event_distribution',
@@ -350,11 +354,10 @@ class DefaultCorrelationBehavior extends ModelBehavior
                 'object_sharing_group_id',
                 'event_sharing_group_id',
                 'org_id',
-                'value_id'
             ]
         ];
         $prefixes = ['1_', ''];
-        $correlated_attribute_ids = [];
+        $correlatedAttributeIds = [];
         foreach ($conditions as $k => $condition) {
             $temp_correlations = $Model->find('all', [
                 'recursive' => -1,
@@ -368,10 +371,15 @@ class DefaultCorrelationBehavior extends ModelBehavior
                             continue;
                         }
                     }
-                    $correlated_attribute_ids[] = $temp_correlation['Correlation'][$prefixes[$k] . 'attribute_id'];
+                    $correlatedAttributeIds[] = $temp_correlation['Correlation'][$prefixes[$k] . 'attribute_id'];
                 }
             }
         }
+
+        if (empty($correlatedAttributeIds)) {
+            return [];
+        }
+
         $contain = [];
         if (!empty($includeEventData)) {
             $contain['Event'] = [
@@ -394,7 +402,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
         $relatedAttributes = $Model->Attribute->find('all', [
             'recursive' => -1,
             'conditions' => [
-                'Attribute.id' => $correlated_attribute_ids
+                'Attribute.id' => $correlatedAttributeIds
             ],
             'fields' => $fields,
             'contain' => $contain

app/Model/Behavior/NoAclCorrelationBehavior.php

@@ -1,17 +1,14 @@
 <?php
-
 App::uses('AppModel', 'Model');
-App::uses('RandomTool', 'Tools');
 
 /**
  * ACL-less correlation behaviour for end-point instances
  */
 class NoAclCorrelationBehavior extends ModelBehavior
 {
+    const TABLE_NAME = 'no_acl_correlations';
 
-    private $__tableName = 'no_acl_correlations';
+    const CONFIG = [
-
-    private $__config = [
         'AttributeFetcher' => [
             'fields' =>  [
                 'Attribute.event_id',
@@ -31,26 +28,36 @@ class NoAclCorrelationBehavior extends ModelBehavior
         ]
     ];
 
-    public $Correlation = null;
+    /** @var Correlation */
+    public $Correlation;
 
+    /** @var bool */
     private $deadlockAvoidance = false;
 
-    public function setup(Model $Model, $settings = []) {
+    public function setup(Model $Model, $settings = [])
-        $Model->useTable = $this->__tableName;
+    {
+        $Model->useTable = self::TABLE_NAME;
         $this->Correlation = $Model;
         $this->deadlockAvoidance = $settings['deadlockAvoidance'];
     }
 
     public function getTableName(Model $Model)
     {
-        return $this->__tableName;
+        return self::TABLE_NAME;
     }
 
-    public function createCorrelationEntry(Model $Model, $value, $a, $b) {
+    /**
-        $value_id = $this->Correlation->CorrelationValue->getValueId($value);
+     * @param Model $Model
+     * @param string $value
+     * @param array $a
+     * @param array $b
+     * @return array|int[]
+     */
+    public function createCorrelationEntry(Model $Model, $value, $a, $b)
+    {
         if ($this->deadlockAvoidance) {
             return [
-                'value_id' => $value_id,
+                'value_id' => $value,
                 '1_event_id' => $a['Event']['id'],
                 '1_attribute_id' => $a['Attribute']['id'],
                 'event_id' => $b['Event']['id'],
@@ -58,7 +65,7 @@ class NoAclCorrelationBehavior extends ModelBehavior
             ];
         } else {
             return [
-                (int) $value_id,
+                $value,
                 (int) $a['Event']['id'],
                 (int) $a['Attribute']['id'],
                 (int) $b['Event']['id'],
@@ -67,7 +74,12 @@ class NoAclCorrelationBehavior extends ModelBehavior
         }
     }
 
-    public function saveCorrelations(Model $Model, $correlations)
+    /**
+     * @param Model $Model
+     * @param array $correlations
+     * @return bool
+     */
+    public function saveCorrelations(Model $Model, array $correlations)
     {
         $fields = [
             'value_id',
@@ -77,29 +89,32 @@ class NoAclCorrelationBehavior extends ModelBehavior
             'attribute_id'
         ];
 
+        $this->Correlation->CorrelationValue->replaceValueWithId($correlations, $this->deadlockAvoidance ? 'value_id' : 0);
+
         if ($this->deadlockAvoidance) {
-          return $this->Correlation->saveMany($correlations, array(
+            return $this->Correlation->saveMany($correlations, [
                 'atomic' => false,
                 'callbacks' => false,
                 'deep' => false,
                 'validate' => false,
-                'fieldList' => $fields
+                'fieldList' => $fields,
-            ));
+            ]);
         } else {
             $db = $this->Correlation->getDataSource();
             // Split to chunks datasource is is enabled
             if (count($correlations) > 100) {
                 foreach (array_chunk($correlations, 100) as $chunk) {
-                    $db->insertMulti('no_acl_correlations', $fields, $chunk);
+                    $db->insertMulti(self::TABLE_NAME, $fields, $chunk);
                 }
                 return true;
             } else {
-                return $db->insertMulti('no_acl_correlations', $fields, $correlations);
+                return $db->insertMulti(self::TABLE_NAME, $fields, $correlations);
             }
         }
     }
 
-    public function runBeforeSaveCorrelation(Model $Model, $attribute) {
+    public function runBeforeSaveCorrelation(Model $Model, $attribute)
+    {
         // (update-only) clean up the relation of the old value: remove the existing relations related to that attribute, we DO have a reference, the id
         // ==> DELETE FROM no_acl_correlations WHERE 1_attribute_id = $a_id OR attribute_id = $a_id; */
         // first check if it's an update
@@ -117,26 +132,37 @@ class NoAclCorrelationBehavior extends ModelBehavior
         }
     }
 
+    /**
+     * @param Model $Model
+     * @param string|null $filter
+     * @return false|mixed
+     */
     public function getContainRules(Model $Model, $filter = null)
     {
         if (empty($filter)) {
-            return $this->__config['AttributeFetcher']['contain'];
+            return self::CONFIG['AttributeFetcher']['contain'];
         } else {
-            return empty($this->__config['AttributeFetcher']['contain'][$filter]) ? false : $this->__config['AttributeFetcher']['contain'][$filter];
+            return self::CONFIG['AttributeFetcher']['contain'][$filter] ?? false;
         }
     }
 
     public function getFieldRules(Model $Model)
     {
-        return $this->__config['AttributeFetcher']['fields'];
+        return self::CONFIG['AttributeFetcher']['fields'];
     }
 
-    private function __collectCorrelations($user, $id, $primary)
+    /**
+     * @param array $user
+     * @param int $eventId
+     * @param bool $primary
+     * @return array
+     */
+    private function __collectCorrelations(array $user, $eventId, $primary)
     {
         $max_correlations = Configure::read('MISP.max_correlations_per_event') ?: 5000;
         $source = $primary ? '' : '1_';
         $prefix = $primary ? '1_' : '';
-        $correlations = $this->Correlation->find('all', [
+        return $this->Correlation->find('all', [
             'fields' => [
                 $source . 'attribute_id',
                 $prefix . 'attribute_id',
@@ -144,7 +170,7 @@ class NoAclCorrelationBehavior extends ModelBehavior
             ],
             'conditions' => [
                 'OR' => [
-                    $source . 'event_id' => $id
+                    $source . 'event_id' => $eventId
                 ],
                 'AND' => [
                     [
@@ -159,7 +185,6 @@ class NoAclCorrelationBehavior extends ModelBehavior
             'contain' => [
                 'CorrelationValue' => [
                     'fields' => [
-                        'CorrelationValue.id',
                         'CorrelationValue.value'
                     ]
                 ]
@@ -167,38 +192,45 @@ class NoAclCorrelationBehavior extends ModelBehavior
             'order' => false,
             'limit' => $max_correlations
         ]);
-        return $correlations;
     }
 
-    public function runGetAttributesRelatedToEvent(Model $Model, $user, $id)
+    /**
+     * @param Model $Model
+     * @param array $user
+     * @param int|array $id Event ID
+     * @return array
+     */
+    public function runGetAttributesRelatedToEvent(Model $Model, array $user, $id)
     {
-        $temp_correlations = $this->__collectCorrelations($user, $id, false);
-        $temp_correlations_1 = $this->__collectCorrelations($user, $id, true);
         $correlations = [];
-        $event_ids = [];
+        $eventIds = [];
+
+        $temp_correlations = $this->__collectCorrelations($user, $id, false);
         foreach ($temp_correlations as $temp_correlation) {
             $correlations[] = [
                 'id' => $temp_correlation['Correlation']['event_id'],
                 'attribute_id' => $temp_correlation['Correlation']['attribute_id'],
                 'parent_id' => $temp_correlation['Correlation']['1_attribute_id'],
-                'value' => $temp_correlation['CorrelationValue']['value']
+                'value' => $temp_correlation['CorrelationValue']['value'],
             ];
-            $event_ids[$temp_correlation['Correlation']['event_id']] = true;
+            $eventIds[$temp_correlation['Correlation']['event_id']] = true;
         }
-        foreach ($temp_correlations_1 as $temp_correlation) {
+
+        $temp_correlations = $this->__collectCorrelations($user, $id, true);
+        foreach ($temp_correlations as $temp_correlation) {
             $correlations[] = [
                 'id' => $temp_correlation['Correlation']['1_event_id'],
                 'attribute_id' => $temp_correlation['Correlation']['1_attribute_id'],
                 'parent_id' => $temp_correlation['Correlation']['attribute_id'],
-                'value' => $temp_correlation['CorrelationValue']['value']
+                'value' => $temp_correlation['CorrelationValue']['value'],
             ];
-            $event_ids[$temp_correlation['Correlation']['1_event_id']] = true;
+            $eventIds[$temp_correlation['Correlation']['1_event_id']] = true;
         }
         if (empty($correlations)) {
             return [];
         }
         $conditions = [
-            'Event.id' => array_keys($event_ids)
+            'Event.id' => array_keys($eventIds)
         ];
         $events = $Model->Event->find('all', array(
             'recursive' => -1,
@@ -214,9 +246,9 @@ class NoAclCorrelationBehavior extends ModelBehavior
                 continue;
             }
             $event = $events[$eventId];
-            $correlation['org_id'] = $events[$eventId]['orgc_id'];
+            $correlation['org_id'] = $event['orgc_id'];
-            $correlation['info'] = $events[$eventId]['info'];
+            $correlation['info'] = $event['info'];
-            $correlation['date'] = $events[$eventId]['date'];
+            $correlation['date'] = $event['date'];
             $parentId = $correlation['parent_id'];
             unset($correlation['parent_id']);
             $relatedAttributes[$parentId][] = $correlation;
@@ -224,47 +256,40 @@ class NoAclCorrelationBehavior extends ModelBehavior
         return $relatedAttributes;
     }
 
+    /**
+     * @param Correlation $Model
+     * @param array $user Not used
+     * @param array $sgids Not used
+     * @param array $attribute
+     * @param array $fields Attribute fields to fetch
+     * @param bool $includeEventData
+     * @return array
+     */
     public function runGetRelatedAttributes(Model $Model, $user, $sgids, $attribute, $fields = [], $includeEventData = false)
     {
-        // LATER getRelatedAttributes($attribute) this might become a performance bottleneck
+        $correlatedAttributeIds = $Model->find('column', [
-        // prepare the conditions
+            'conditions' => [
-        $conditions = [
-            [
                 'Correlation.1_event_id !=' => $attribute['event_id'],
-                'Correlation.attribute_id' => $attribute['id']
+                'Correlation.attribute_id' => $attribute['id'],
             ],
-            [
+            'fields' => ['1_attribute_id'],
-                'Correlation.event_id !=' => $attribute['event_id'],
-                'Correlation.1_attribute_id' => $attribute['id']
-            ]
-        ];
-        $corr_fields = [
-            [
-                '1_attribute_id',
-                '1_event_id',
-                'value_id'
-            ],
-            [
-                'attribute_id',
-                'event_id',
-                'value_id'
-            ]
-        ];
-        $prefixes = ['1_', ''];
-        $correlated_attribute_ids = [];
-        foreach ($conditions as $k => $condition) {
-            $temp_correlations = $Model->find('all', [
-                'recursive' => -1,
-                'conditions' => $condition,
-                'fields' => $corr_fields[$k]
         ]);
-            if (!empty($temp_correlations)) {
+
-                foreach ($temp_correlations as $temp_correlation) {
+        $correlatedAttributeIds2 = $Model->find('column', [
-                    $correlated_attribute_ids[] = $temp_correlation['Correlation'][$prefixes[$k] . 'attribute_id'];
+            'conditions' => [
+                'Correlation.event_id !=' => $attribute['event_id'],
+                'Correlation.1_attribute_id' => $attribute['id'],
+            ],
+            'fields' => ['attribute_id'],
+        ]);
+        foreach ($correlatedAttributeIds2 as $tempCorrelation) {
+            $correlatedAttributeIds[] = $tempCorrelation;
         }
+
+        if (empty($correlatedAttributeIds)) {
+            return [];
         }
-        }
+
-        $contain = [];
         if (!empty($includeEventData)) {
             $contain['Event'] = [
                 'fields' => [
@@ -282,18 +307,21 @@ class NoAclCorrelationBehavior extends ModelBehavior
                     'Event.org_id'
                 ]
             ];
+        } else {
+            $contain = [];
         }
+
         $relatedAttributes = $Model->Attribute->find('all', [
             'recursive' => -1,
             'conditions' => [
-                'Attribute.id' => $correlated_attribute_ids
+                'Attribute.id' => $correlatedAttributeIds
             ],
             'fields' => $fields,
             'contain' => $contain
         ]);
         if (!empty($includeEventData)) {
             $results = [];
-            foreach ($relatedAttributes as $k => $attribute) {
+            foreach ($relatedAttributes as $attribute) {
                 $temp = $attribute['Attribute'];
                 $temp['Event'] = $attribute['Event'];
                 $results[] = $temp;
@@ -304,30 +332,31 @@ class NoAclCorrelationBehavior extends ModelBehavior
         }
     }
 
+    /**
+     * @param Correlation $Model
+     * @param array $user Not used
+     * @param int $eventId
+     * @param array $sgids Not used
+     * @return array
+     */
     public function fetchRelatedEventIds(Model $Model, array $user, int $eventId, array $sgids)
     {
-        // search the correlation table for the event ids of the related events
+        $primaryEventIds = $this->__filterRelatedEvents($Model, $eventId, true);
-        // Rules:
+        $secondaryEventIds = $this->__filterRelatedEvents($Model, $eventId, false);
-        // 1. Event is owned by the user (org_id matches)
+        return array_unique(array_merge($primaryEventIds, $secondaryEventIds), SORT_REGULAR);
-        // 2. User is allowed to see both the event and the org:
-        //    a.  Event:
-        //        i. Event has a distribution between 1-3 (community only, connected communities, all orgs)
-        //        ii. Event has a sharing group that the user is accessible to view
-        //    b.  Attribute:
-        //        i. Attribute has a distribution of 5 (inheritance of the event, for this the event check has to pass anyway)
-        //        ii. Atttibute has a distribution between 1-3 (community only, connected communities, all orgs)
-        //        iii. Attribute has a sharing group that the user is accessible to view
-        $primaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, true);
-        $secondaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, false);
-        return array_unique(array_merge($primaryEventIds,$secondaryEventIds));
-
     }
 
-    private function __filterRelatedEvents(Model $Model, array $user, int $eventId, bool $primary)
+    /**
+     * @param Correlation $Model
+     * @param int $eventId
+     * @param bool $primary
+     * @return array
+     */
+    private function __filterRelatedEvents(Model $Model, int $eventId, bool $primary)
     {
         $current = $primary ? '' : '1_';
         $prefix = $primary ? '1_' : '';
-        $correlations = $Model->find('all', [
+        return $Model->find('column', [
             'recursive' => -1,
             'fields' => [
                 $prefix . 'event_id'
@@ -337,8 +366,6 @@ class NoAclCorrelationBehavior extends ModelBehavior
             ],
             'unique' => true,
         ]);
-        $eventIds = Hash::extract($correlations, '{n}.Correlation.' . $prefix . 'event_id');
-        return $eventIds;
     }
 
     public function updateContainedCorrelations(
@@ -368,6 +395,11 @@ class NoAclCorrelationBehavior extends ModelBehavior
         }
     }
 
+    /**
+     * @param Correlation $Model
+     * @param string $value
+     * @return void
+     */
     public function purgeByValue(Model $Model, string $value)
     {
         $valueIds = $Model->CorrelationValue->find('column', [

app/Model/Correlation.php

@@ -6,7 +6,8 @@ App::uses('AppModel', 'Model');
  * @property Event $Event
  * @property CorrelationValue $CorrelationValue
  * @method saveCorrelations(array $correlations)
- * @method runBeforeSaveCorrelation
+ * @method createCorrelationEntry(string $value, array $a, array $b)
+ * @method runBeforeSaveCorrelation(array $attribute)
  * @method fetchRelatedEventIds(array $user, int $eventId, array $sgids)
  * @method getFieldRules
  * @method getContainRules($filter = null)
@@ -16,8 +17,6 @@ class Correlation extends AppModel
     const CACHE_NAME = 'misp:top_correlations',
         CACHE_AGE = 'misp:top_correlations_age';
 
-    private $__compositeTypes = [];
-
     public $belongsTo = array(
         'Attribute' => [
             'className' => 'Attribute',
@@ -56,9 +55,6 @@ class Correlation extends AppModel
     /** @var array */
     private $cidrListCache;
 
-    /** @var string */
-    private $__correlationEngine;
-
     private $__tempContainCache = [];
 
     /** @var OverCorrelatingValue */
@@ -67,10 +63,10 @@ class Correlation extends AppModel
     public function __construct($id = false, $table = null, $ds = null)
     {
         parent::__construct($id, $table, $ds);
-        $this->__correlationEngine = $this->getCorrelationModelName();
+        $correlationEngine = $this->getCorrelationModelName();
-        $deadlockAvoidance = Configure::check('MISP.deadlock_avoidance') ? Configure::read('MISP.deadlock_avoidance') : false;
+        $deadlockAvoidance = Configure::read('MISP.deadlock_avoidance') ?: false;
         // load the currently used correlation engine
-        $this->Behaviors->load($this->__correlationEngine . 'Correlation', ['deadlockAvoidance' => $deadlockAvoidance]);
+        $this->Behaviors->load($correlationEngine . 'Correlation', ['deadlockAvoidance' => $deadlockAvoidance]);
         // getTableName() needs to be implemented by the engine - this points us to the table to be used
         $this->useTable = $this->getTableName();
         $this->advancedCorrelationEnabled = (bool)Configure::read('MISP.enable_advanced_correlations');
@@ -183,17 +179,6 @@ class Correlation extends AppModel
         return $correlatingAttributes;
     }
 
-    /**
-     * @param string $value
-     * @param array $a Attribute A
-     * @param array $b Attribute B
-     * @return array
-     */
-    private function __createCorrelationEntry($value, $a, $b)
-    {
-        return $this->createCorrelationEntry($value, $a, $b);
-    }
-
     public function correlateValue($value, $jobId = false)
     {
         $correlatingAttributes = $this->__getMatchingAttributes($value);
@@ -216,7 +201,7 @@ class Correlation extends AppModel
                 if ($correlatingAttribute['Attribute']['event_id'] === $correlatingAttribute2['Attribute']['event_id']) {
                     continue;
                 }
-                $correlations[] = $this->__createCorrelationEntry($value, $correlatingAttribute, $correlatingAttribute2);
+                $correlations[] = $this->createCorrelationEntry($value, $correlatingAttribute, $correlatingAttribute2);
             }
             $extraCorrelations = $this->__addAdvancedCorrelations($correlatingAttribute);
             if (!empty($extraCorrelations)) {
@@ -224,8 +209,8 @@ class Correlation extends AppModel
                     if ($correlatingAttribute['Attribute']['event_id'] === $extraCorrelation['Attribute']['event_id']) {
                         continue;
                     }
-                    $correlations[] = $this->__createCorrelationEntry($value, $correlatingAttribute, $extraCorrelation);
+                    $correlations[] = $this->createCorrelationEntry($value, $correlatingAttribute, $extraCorrelation);
-                    //$correlations = $this->__createCorrelationEntry($value, $extraCorrelation, $correlatingAttribute, $correlations);
+                    //$correlations = $this->createCorrelationEntry($value, $extraCorrelation, $correlatingAttribute, $correlations);
                 }
             }
             if ($jobId && $k % 100 === 0) {
@@ -252,17 +237,16 @@ class Correlation extends AppModel
         }
     }
 
-    public function correlateAttribute(array $attribute)
-    {
-        $this->runBeforeSaveCorrelation($attribute);
-        $this->afterSaveCorrelation($attribute);
-    }
-
     public function beforeSaveCorrelation(array $attribute)
     {
         $this->runBeforeSaveCorrelation($attribute);
     }
 
+    /**
+     * @param string $scope
+     * @param int $id
+     * @return false|array
+     */
     private function __cachedGetContainData($scope, $id)
     {
         if (!empty($this->getContainRules($scope))) {
@@ -374,13 +358,13 @@ class Correlation extends AppModel
                 // If we have more correlations for the value than the limit, set the block entry and stop the correlation process
                 $this->OverCorrelatingValue->block($cV);
                 return true;
-            } else {
+            } else if ($count !== 0) {
                 // If we have fewer hits than the limit, proceed with the correlation, but first make sure we remove any existing blockers
                 $this->OverCorrelatingValue->unblock($cV);
             }
             foreach ($correlatingAttributes as $b) {
                 // On a full correlation, only correlate with attributes that have a higher ID to avoid duplicate correlations
-                if ($full && $b['Attribute']['id'] < $b['Attribute']['id']) {
+                if ($full && $a['Attribute']['id'] < $b['Attribute']['id']) {
                     continue;
                 }
                 if (isset($b['Attribute']['value1'])) {
@@ -390,9 +374,9 @@ class Correlation extends AppModel
                     $value = $cV;
                 }
                 if ($a['Attribute']['id'] > $b['Attribute']['id']) {
-                    $correlations[] = $this->__createCorrelationEntry($value, $a, $b);
+                    $correlations[] = $this->createCorrelationEntry($value, $a, $b);
                 } else {
-                    $correlations[] = $this->__createCorrelationEntry($value, $b, $a);
+                    $correlations[] = $this->createCorrelationEntry($value, $b, $a);
                 }
             }
         }
@@ -842,7 +826,7 @@ class Correlation extends AppModel
      */
     public function getRelatedAttributes($user, $sgids, $attribute, $fields=[], $includeEventData = false)
     {
-        if (in_array($attribute['type'], Attribute::NON_CORRELATING_TYPES)) {
+        if (in_array($attribute['type'], Attribute::NON_CORRELATING_TYPES, true)) {
             return [];
         }
         return $this->runGetRelatedAttributes($user, $sgids, $attribute, $fields, $includeEventData);
@@ -850,7 +834,7 @@ class Correlation extends AppModel
     
     /**
      * @param array $user User array
-     * @param int $eventId List of event IDs
+     * @param int $eventId Event ID
      * @param array $sgids List of sharing group IDs
      * @return array
      */
@@ -877,16 +861,19 @@ class Correlation extends AppModel
      */
     public function attachCorrelationExclusion(array $attributes)
     {
-        if (!isset($this->__compositeTypes)) {
+        $compositeTypes = $this->Attribute->getCompositeTypes();
-            $this->__compositeTypes = $this->Attribute->getCompositeTypes();
-        }
-
         $valuesToCheck = [];
         foreach ($attributes as &$attribute) {
-            if (in_array($attribute['type'], $this->__compositeTypes, true)) {
+            if ($attribute['disable_correlation'] || in_array($attribute['type'],Attribute::NON_CORRELATING_TYPES, true)) {
+                continue;
+            }
+            $primaryOnly = in_array($attribute['type'], Attribute::PRIMARY_ONLY_CORRELATING_TYPES, true);
+            if (in_array($attribute['type'], $compositeTypes, true)) {
                 $values = explode('|', $attribute['value']);
                 $valuesToCheck[$values[0]] = true;
+                if (!$primaryOnly) {
                     $valuesToCheck[$values[1]] = true;
+                }
             } else {
                 $values = [$attribute['value']];
                 $valuesToCheck[$values[0]] = true;
@@ -894,7 +881,7 @@ class Correlation extends AppModel
 
             if ($this->__preventExcludedCorrelations($values[0])) {
                 $attribute['correlation_exclusion'] = true;
-            } elseif (!empty($values[1]) && $this->__preventExcludedCorrelations($values[1])) {
+            } elseif (!empty($values[1]) && !$primaryOnly && $this->__preventExcludedCorrelations($values[1])) {
                 $attribute['correlation_exclusion'] = true;
             }
         }
@@ -903,16 +890,20 @@ class Correlation extends AppModel
         unset($valuesToCheck);
 
         foreach ($attributes as &$attribute) {
-            if (in_array($attribute['type'], $this->__compositeTypes, true)) {
+            if ($attribute['disable_correlation'] || in_array($attribute['type'],Attribute::NON_CORRELATING_TYPES, true)) {
-                $values = explode('|', $attribute['value']);
+                continue;
-            } else {
+            }
-                $values = [$attribute['value']];
+            $primaryOnly = in_array($attribute['type'], Attribute::PRIMARY_ONLY_CORRELATING_TYPES, true);
+            if (in_array($attribute['type'], $compositeTypes, true)) {
+                $values = explode('|', $attribute['value']);
+                $values = OverCorrelatingValue::truncateValues($values);
+            } else {
+                $values = [OverCorrelatingValue::truncate($attribute['value'])];
             }
-            $values = $this->OverCorrelatingValue->truncateValues($values);
 
             if (isset($overCorrelatingValues[$values[0]])) {
                 $attribute['over_correlation'] = true;
-            } elseif (!empty($values[1]) && isset($overCorrelatingValues[$values[1]])) {
+            } elseif (!empty($values[1])  && !$primaryOnly && isset($overCorrelatingValues[$values[1]])) {
                 $attribute['over_correlation'] = true;
             }
         }
@@ -922,8 +913,9 @@ class Correlation extends AppModel
 
     public function collectMetrics()
     {
-        $results['engine'] = $this->getCorrelationModelName();
+        $results = [
-        $results['db'] = [
+            'engine' => $this->getCorrelationModelName(),
+            'db' => [
                 'Default' => [
                     'name' => __('Default correlation engine'),
                     'tables' => [
@@ -954,8 +946,9 @@ class Correlation extends AppModel
                         ]
                     ]
                 ]
+            ],
+            'over_correlations' => $this->OverCorrelatingValue->find('count'),
         ];
-        $results['over_correlations'] = $this->OverCorrelatingValue->find('count');
         $this->CorrelationExclusion = ClassRegistry::init('CorrelationExclusion');
         $results['excluded_correlations'] = $this->CorrelationExclusion->find('count');
         foreach ($results['db'] as &$result) {
@@ -996,4 +989,12 @@ class Correlation extends AppModel
         }
         return $result === true;
     }
+
+    /**
+     * @return string
+     */
+    private function getCorrelationModelName()
+    {
+        return Configure::read('MISP.correlation_engine') ?: 'Default';
+    }
 }

app/Model/CorrelationValue.php

@@ -5,6 +5,62 @@ class CorrelationValue extends AppModel
 {
     public $recursive = -1;
 
+    /**
+     * @param array $correlations
+     * @param string|int $valueIndex
+     * @return void
+     */
+    public function replaceValueWithId(array &$correlations, $valueIndex)
+    {
+        $values = array_column($correlations, $valueIndex);
+        $valueIds = $this->getIds($values);
+
+        foreach ($correlations as &$correlation) {
+            $value = mb_substr($correlation[$valueIndex], 0, 191);
+            $correlation[$valueIndex] = $valueIds[$value];
+        }
+    }
+
+    /**
+     * @param array $values
+     * @return array Value in key, value ID in value
+     */
+    private function getIds(array $values)
+    {
+        foreach ($values as &$value) {
+            $value = mb_substr($value, 0, 191);
+        }
+        $values = array_unique($values, SORT_REGULAR); // Remove duplicate values
+        $existingValues = $this->find('list', [
+            'recursive' => -1,
+            'callbacks' => false,
+            'fields' => ['value', 'id'],
+            'conditions' => [
+                'value' => $values,
+            ],
+        ]);
+
+        $notExistValues = array_diff($values, array_keys($existingValues));
+        if (!empty($notExistValues)) {
+            $this->getDataSource()->begin();
+            foreach ($notExistValues as $notExistValue) {
+                $this->create();
+                try {
+                    $this->save(['value' => $notExistValue], [
+                        'callbacks' => false,
+                        'validate' => false,
+                    ]);
+                    $existingValues[$notExistValue] = $this->id;
+                } catch (Exception $e) {
+                    $existingValues[$notExistValue] = $this->getValueId($notExistValue);
+                }
+            }
+            $this->getDataSource()->commit();
+        }
+
+        return $existingValues;
+    }
+
     /**
      * @param string $value
      * @return int

app/Model/CryptographicKey.php

@@ -89,13 +89,13 @@ class CryptographicKey extends AppModel
     }
 
     /**
-     * @return string Instance key fingerprint
+     * @return string|false Instance key fingerprint or false is no key is set
      * @throws Crypt_GPG_BadPassphraseException
      * @throws Crypt_GPG_Exception
      */
     public function ingestInstanceKey()
     {
-        // If instance just key stored just in GPG homedir, use that key.
+        // If instance key is stored just in GPG homedir, use that key.
         if (Configure::read('MISP.download_gpg_from_homedir')) {
             if (!$this->gpg) {
                 throw new Exception("Could not initiate GPG");
@@ -130,7 +130,7 @@ class CryptographicKey extends AppModel
             try {
                 $this->gpg->importKey($instanceKey);
             } catch (Crypt_GPG_NoDataException $e) {
-                throw new MethodNotAllowedException("Could not import the instance key..");
+                throw new MethodNotAllowedException("Could not import the instance key.");
             }
             $fingerprint = $this->gpg->getFingerprint(Configure::read('GnuPG.email'));
             if ($redis) {

app/Model/DecayingModel.php

@@ -19,6 +19,9 @@ class DecayingModel extends AppModel
     private $__registered_model_classes = array(); // Proxy for already instantiated classes
     public $allowed_overrides = array('threshold' => 1, 'lifetime' => 1, 'decay_speed' => 1);
 
+    /** @var array */
+    private $defaultModelsCache;
+
     public function afterFind($results, $primary = false) {
         foreach ($results as $k => $v) {
             if (!empty($v['DecayingModel']['parameters'])) {
@@ -207,7 +210,7 @@ class DecayingModel extends AppModel
         return $default_models;
     }
 
-    public function fetchAllAllowedModels($user, $full=true, $filters=array(), $additionnal_conditions=array())
+    public function fetchAllAllowedModels($user, $full=true, $filters=array(), $additionalConditions=array())
     {
         $conditions = array();
         if (!$user['Role']['perm_site_admin']) {
@@ -223,7 +226,7 @@ class DecayingModel extends AppModel
                 $conditions[] = array('not' => array('DecayingModel.uuid' => null));
             }
         }
-        $conditions[] = array('AND' => $additionnal_conditions);
+        $conditions[] = array('AND' => $additionalConditions);
         $decayingModels = $this->find('all', array(
             'conditions' => $conditions,
             'include' => $full ? 'DecayingModelMapping' : ''
@@ -638,17 +641,29 @@ class DecayingModel extends AppModel
         return $attribute;
     }
 
-    public function attachScoresToEvent($user, $event, $model_id=false, $model_overrides=array(), $include_full_model=0)
+    /**
+     * @param array $user
+     * @param array $event
+     * @param int|bool $modelId
+     * @param array $modelOverrides
+     * @param bool $includeFullModel
+     * @return array
+     */
+    public function attachScoresToEvent(array $user, array $event, $modelId = false, $modelOverrides = [], $includeFullModel = false)
     {
-        $models = [];
+        if ($modelId === false) { // fetch all allowed and associated models
-        if ($model_id === false) { // fetch all allowed and associated models
+            if (isset($this->defaultModelsCache[$user['id']])) {
-            $models = $this->fetchAllAllowedModels($user, false, [], ['DecayingModel.enabled' => true]);
+                $models = $this->defaultModelsCache[$user['id']];
             } else {
-            $models = $this->fetchModels($user, $model_id, false, array());
+                $models = $this->fetchAllAllowedModels($user, false, [], ['DecayingModel.enabled' => true]);
+                $this->defaultModelsCache[$user['id']] = $models;
+            }
+        } else {
+            $models = $this->fetchModels($user, $modelId, false, array());
         }
         foreach ($models as $model) {
-            if (!empty($model_overrides)) {
+            if (!empty($modelOverrides)) {
-                $model = $this->overrideModelParameters($model, $model_overrides);
+                $model = $this->overrideModelParameters($model, $modelOverrides);
             }
             $eventScore = $this->getScoreForEvent($event, $model);
             $decayed = $this->isEventDecayed($model, $eventScore['score']);
@@ -661,7 +676,7 @@ class DecayingModel extends AppModel
                     'name' => $model['DecayingModel']['name']
                 ]
             ];
-            if ($include_full_model) {
+            if ($includeFullModel) {
                 $to_attach['DecayingModel'] = $model['DecayingModel'];
             }
             $event['event_scores'][] = $to_attach;

app/Model/Event.php

@@ -689,26 +689,18 @@ class Event extends AppModel
     }
 
     /**
+     * Get related attributes for event
      * @param array $user
-     * @param int|array $id Event ID when $scope is 'event', Attribute ID when $scope is 'attribute'
+     * @param int|array $eventIds Event IDs
-     * @param bool $shadowAttribute
-     * @param string $scope 'event' or 'attribute'
      * @return array
      */
-    public function getRelatedAttributes(array $user, $id, $shadowAttribute = false, $scope = 'event')
+    public function getRelatedAttributes(array $user, $eventIds)
     {
-        if ($shadowAttribute) {
+        if (!isset($this->Correlation)) {
-            // no longer supported
+            $this->Correlation = ClassRegistry::init('Correlation');
-            return [];
-        } else {
-            $parentIdField = '1_attribute_id';
-            $correlationModelName = 'Correlation';
-        }
-        if (!isset($this->{$correlationModelName})) {
-            $this->{$correlationModelName} = ClassRegistry::init($correlationModelName);
         }
         $sgids = $this->SharingGroup->authorizedIds($user);
-        $relatedAttributes = $this->{$correlationModelName}->getAttributesRelatedToEvent($user, $id, $sgids);
+        $relatedAttributes = $this->Correlation->getAttributesRelatedToEvent($user, $eventIds, $sgids);
         return $relatedAttributes;
     }
 
@@ -2068,7 +2060,9 @@ class Event extends AppModel
                     $event['Attribute'] = $this->__attachSharingGroups($event['Attribute'], $sharingGroupData);
                 }
 
+                if (!empty($options['includeGranularCorrelations'])) {
                     $event['Attribute'] = $this->Attribute->Correlation->attachCorrelationExclusion($event['Attribute']);
+                }
 
                 // move all object attributes to a temporary container
                 $tempObjectAttributeContainer = array();
@@ -2182,7 +2176,7 @@ class Event extends AppModel
         }
 
         $this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
-        $clusters = $this->GalaxyCluster->getClusters($galaxyTags, $user, true, $fetchFullCluster);
+        $clusters = $this->GalaxyCluster->getClustersByTags($galaxyTags, $user, true, $fetchFullCluster);
 
         if (empty($clusters)) {
             return;
@@ -2200,7 +2194,7 @@ class Event extends AppModel
                 if (isset($clustersByTagIds[$tagId])) {
                     $cluster = $clustersByTagIds[$tagId];
                     $galaxyId = $cluster['Galaxy']['id'];
-                    $cluster['local'] = isset($eventTag['local']) ? $eventTag['local'] : false;
+                    $cluster['local'] = $eventTag['local'] ?? false;
                     if (isset($event['Galaxy'][$galaxyId])) {
                         unset($cluster['Galaxy']);
                         $event['Galaxy'][$galaxyId]['GalaxyCluster'][] = $cluster;
@@ -2224,7 +2218,7 @@ class Event extends AppModel
                         if (isset($clustersByTagIds[$tagId])) {
                             $cluster = $clustersByTagIds[$tagId];
                             $galaxyId = $cluster['Galaxy']['id'];
-                            $cluster['local'] = isset($attributeTag['local']) ? $attributeTag['local'] : false;
+                            $cluster['local'] = $attributeTag['local'] ?? false;
                             if (isset($attribute['Galaxy'][$galaxyId])) {
                                 unset($cluster['Galaxy']);
                                 $attribute['Galaxy'][$galaxyId]['GalaxyCluster'][] = $cluster;
@@ -4521,46 +4515,6 @@ class Event extends AppModel
         }
     }
 
-    public function generateLocked()
-    {
-        $this->User = ClassRegistry::init('User');
-        $this->User->recursive = -1;
-        $localOrgs = array();
-        $conditions = array();
-        $orgs = $this->User->find('all', array('fields' => array('DISTINCT org_id')));
-        foreach ($orgs as $k => $org) {
-            $orgs[$k]['User']['count'] = $this->User->getOrgMemberCount($orgs[$k]['User']['org_id']);
-            if ($orgs[$k]['User']['count'] > 1) {
-                $localOrgs[] = $orgs[$k]['User']['org_id'];
-                $conditions['AND'][] = array('orgc !=' => $orgs[$k]['User']['org_id']);
-            } elseif ($orgs[$k]['User']['count'] == 1) {
-                // If we only have a single user for an org, check if that user is a sync user. If not, then it is a valid local org and the events created by him/her should be unlocked.
-                $this->User->recursive = 1;
-                $user = ($this->User->find('first', array(
-                        'fields' => array('id', 'role_id'),
-                        'conditions' => array('org_id' => $org['User']['org_id']),
-                        'contain' => array('Role' => array(
-                                'fields' => array('id', 'perm_sync'),
-                        ))
-                )));
-                if (!$user['Role']['perm_sync']) {
-                    $conditions['AND'][] = array('orgc !=' => $orgs[$k]['User']['org_id']);
-                }
-            }
-        }
-        // Don't lock stuff that's already locked
-        $conditions['AND'][] = array('locked !=' => true);
-        $this->recursive = -1;
-        $toBeUpdated = $this->find('count', array(
-                'conditions' => $conditions
-        ));
-        $this->updateAll(
-                array('Event.locked' => 1),
-                $conditions
-        );
-        return $toBeUpdated;
-    }
-
     public function reportValidationIssuesEvents()
     {
         $this->Behaviors->detach('Regexp');
@@ -4583,20 +4537,6 @@ class Event extends AppModel
         return array($result, $k);
     }
 
-    public function generateThreatLevelFromRisk()
-    {
-        $risk = array('Undefined' => 4, 'Low' => 3, 'Medium' => 2, 'High' => 1);
-        $events = $this->find('all', array('recursive' => -1));
-        $k = 0;
-        foreach ($events as $k => $event) {
-            if ($event['Event']['threat_level_id'] == 0 && isset($event['Event']['risk'])) {
-                $event['Event']['threat_level_id'] = $risk[$event['Event']['risk']];
-                $this->save($event);
-            }
-        }
-        return $k;
-    }
-
     // check two version strings. If version 1 is older than 2, return -1, if they are the same return 0, if version 2 is older return 1
     public function compareVersions($version1, $version2)
     {
@@ -4925,7 +4865,9 @@ class Event extends AppModel
     ) {
         $object['category'] = $object['meta-category'];
 
-        $include = empty($filterType['attributeFilter']) || $filterType['attributeFilter'] === 'all' || $filterType['attributeFilter'] === 'object' || $object['meta-category'] === $filterType['attributeFilter'];
+        $include = empty($filterType['attributeFilter']) ||
+            in_array($filterType['attributeFilter'], array('all', 'object', 'correlation', 'proposal', 'warning')) ||
+            $object['meta-category'] === $filterType['attributeFilter'];
 
         if (!$include) {
             return null;
@@ -6891,7 +6833,7 @@ class Event extends AppModel
             }
         }
 
-        $notCachedTags = array_diff_key($tagIds, isset($this->assetCache['tags']) ? $this->assetCache['tags'] : []);
+        $notCachedTags = array_diff_key($tagIds, $this->assetCache['tags'] ?? []);
         if (empty($notCachedTags)) {
             return;
         }
@@ -7087,7 +7029,7 @@ class Event extends AppModel
      */
     private function __clusterEventIds($exportTool, $eventIds)
     {
-        $memory_in_mb = $this->Attribute->convert_to_memory_limit_to_mb(ini_get('memory_limit'));
+        $memory_in_mb = $this->convert_to_memory_limit_to_mb(ini_get('memory_limit'));
         $default_attribute_memory_coefficient = Configure::check('MISP.default_attribute_memory_coefficient') ? Configure::read('MISP.default_attribute_memory_coefficient') : 80;
         $default_event_memory_divisor = Configure::check('MISP.default_event_memory_multiplier') ? Configure::read('MISP.default_event_memory_divisor') : 3;
         $memory_scaling_factor = isset($exportTool->memory_scaling_factor) ? $exportTool->memory_scaling_factor : $default_attribute_memory_coefficient;
@@ -7656,6 +7598,10 @@ class Event extends AppModel
 
     public function getTrendsForTagsFromEvents(array $events, int $baseDayRange, int $rollingWindows=3, $tagFilterPrefixes=null): array
     {
+        $oldestTimestamp = $this->resolveTimeDelta($baseDayRange + $baseDayRange * $rollingWindows . 'd');
+        $events = array_filter($events, function($event) use ($oldestTimestamp) { // Filter out events having old modification compared to their publish_timestamp
+            return $event['Event']['timestamp'] >= $oldestTimestamp;
+        });
         App::uses('TrendingTool', 'Tools');
         $trendingTool = new TrendingTool($this);
         $trendAnalysis = $trendingTool->getTrendsForTags($events, $baseDayRange, $rollingWindows, $tagFilterPrefixes);

app/Model/EventLock.php

@@ -82,7 +82,7 @@ class EventLock extends AppModel
     public function deleteBackgroundJobLock($eventId, $jobId)
     {
         try {
-            $redis = $this->setupRedisWithException();
+            $redis = RedisTool::init();
         } catch (Exception $e) {
             return false;
         }
@@ -100,7 +100,7 @@ class EventLock extends AppModel
     public function checkLock(array $user, $eventId)
     {
         try {
-            $redis = $this->setupRedisWithException();
+            $redis = RedisTool::init();
         } catch (Exception $e) {
             return [];
         }
@@ -113,7 +113,7 @@ class EventLock extends AppModel
         $output = [];
         $now = time();
         foreach ($keys as $value) {
-            $value = $this->jsonDecode($value);
+            $value = RedisTool::deserialize($value);
             if ($value['timestamp'] + self::DEFAULT_TTL > $now) {
                 $output[] = $value;
             }
@@ -130,13 +130,13 @@ class EventLock extends AppModel
     private function insertLockToRedis($eventId, $lockId, array $data)
     {
         try {
-            $redis = $this->setupRedisWithException();
+            $redis = RedisTool::init();
         } catch (Exception $e) {
             return false;
         }
 
         $pipeline = $redis->pipeline();
-        $pipeline->hSet(self::PREFIX . $eventId, $lockId, json_encode($data));
+        $pipeline->hSet(self::PREFIX . $eventId, $lockId, RedisTool::serialize($data));
         $pipeline->expire(self::PREFIX . $eventId, self::DEFAULT_TTL); // prolong TTL
         return $pipeline->exec()[0] !== false;
     }

app/Model/GalaxyCluster.php

@@ -967,21 +967,17 @@ class GalaxyCluster extends AppModel
     }
 
     /**
-     * @param array $namesOrIds Cluster tag names or cluster IDs
+     * @param array $tagNames Cluster tag names with tag ID in key
      * @param array $user
      * @param bool $postProcess If true, self::postprocess method will be called.
      * @param bool $fetchFullCluster
      * @return array
      */
-    public function getClusters(array $namesOrIds, array $user, $postProcess = true, $fetchFullCluster = true)
+    public function getClustersByTags(array $tagNames, array $user, $postProcess = true, $fetchFullCluster = true)
     {
-        if (count(array_filter($namesOrIds, 'is_numeric')) === count($namesOrIds)) { // all elements are numeric
+        $options = [
-            $conditions = array('GalaxyCluster.id' => $namesOrIds);
+            'conditions' => ['GalaxyCluster.tag_name' => $tagNames],
-        } else {
+        ];
-            $conditions = array('GalaxyCluster.tag_name' => $namesOrIds);
-        }
-
-        $options = ['conditions' => $conditions];
         if (!$fetchFullCluster) {
             $options['contain'] = ['Galaxy', 'GalaxyElement'];
         }
@@ -989,17 +985,10 @@ class GalaxyCluster extends AppModel
         $clusters = $this->fetchGalaxyClusters($user, $options, $fetchFullCluster);
 
         if (!empty($clusters) && $postProcess) {
-            $tagNames = array_map('strtolower', array_column(array_column($clusters, 'GalaxyCluster'), 'tag_name'));
+            $tagIds = array_change_key_case(array_flip($tagNames));
-            $tagIds = $this->Tag->find('list', [
-                'conditions' => ['LOWER(Tag.name)' => $tagNames],
-                'recursive' => -1,
-                'fields' => array('Tag.name', 'Tag.id'),
-            ]);
-            $tagIds = array_change_key_case($tagIds);
-
             foreach ($clusters as $k => $cluster) {
                 $tagName = strtolower($cluster['GalaxyCluster']['tag_name']);
-                $clusters[$k] = $this->postprocess($cluster, isset($tagIds[$tagName]) ? $tagIds[$tagName] : null);
+                $clusters[$k] = $this->postprocess($cluster, $tagIds[$tagName] ?? null);
             }
         }
 
@@ -1142,7 +1131,7 @@ class GalaxyCluster extends AppModel
         }
 
         $this->Event = ClassRegistry::init('Event');
-        $sharingGroupData = $this->Event->__cacheSharingGroupData($user, false);
+        $sharingGroupData = $this->Event->__cacheSharingGroupData($user, true);
         foreach ($clusters as $i => $cluster) {
             if (!empty($cluster['GalaxyCluster']['sharing_group_id']) && isset($sharingGroupData[$cluster['GalaxyCluster']['sharing_group_id']])) {
                 $clusters[$i]['SharingGroup'] = $sharingGroupData[$cluster['GalaxyCluster']['sharing_group_id']];
@@ -1404,10 +1393,11 @@ class GalaxyCluster extends AppModel
     }
 
     /**
-     * fetchClusterById Simple ACL-aware method to fetch a cluster by Id or UUID
+     * Simple ACL-aware method to fetch a cluster by Id or UUID
      *
      * @param array $user
-     * @param  int|string $clusterId
+     * @param int|string $clusterId Cluster ID or UUID
+     * @param bool $throwErrors
      * @param bool $full
      * @return array
      */
@@ -1415,37 +1405,27 @@ class GalaxyCluster extends AppModel
     {
         $alias = $this->alias;
         if (Validation::uuid($clusterId)) {
-            $temp = $this->find('first', array(
+            $conditions = array("${alias}.uuid" => $clusterId);
-                'recursive' => -1,
+        } elseif (is_numeric($clusterId)) {
-                'fields' => array("${alias}.id", "${alias}.uuid"),
+            $conditions = array("${alias}.id" => $clusterId);
-                'conditions' => array("${alias}.uuid" => $clusterId)
+        } else{
-            ));
-            if (empty($temp)) {
             if ($throwErrors) {
                 throw new NotFoundException(__('Invalid galaxy cluster'));
             }
             return array();
         }
-            $clusterId = $temp[$alias]['id'];
+
-        } elseif (!is_numeric($clusterId)) {
+        return $this->fetchGalaxyClusters($user, ['conditions' => $conditions], $full=$full);
-            if ($throwErrors) {
-                throw new NotFoundException(__('Invalid galaxy cluster'));
-            }
-            return array();
-        }
-        $conditions = array('conditions' => array("${alias}.id" => $clusterId));
-        $cluster = $this->fetchGalaxyClusters($user, $conditions, $full=$full);
-        return $cluster;
     }
 
 
     /**
-     * fetchIfAuthorized Fetches a cluster and checks if the user has the authorization to perform the requested operation
+     * Fetches a cluster and checks if the user has the authorization to perform the requested operation
      *
      * @param  array $user
      * @param  int|string|array $cluster
      * @param  mixed $authorizations the requested actions to be performed on the cluster
-     * @param  bool  $throwErrors Should the function throws excpetion if users is not allowed to perform the action
+     * @param  bool  $throwErrors Should the function throws exception if users is not allowed to perform the action
      * @param  bool  $full
      * @return array The cluster or an error message
      */
@@ -1474,7 +1454,7 @@ class GalaxyCluster extends AppModel
             return $cluster;
         }
 
-        if (in_array('view', $authorizations) && count($authorizations) == 1) {
+        if (in_array('view', $authorizations) && count($authorizations) === 1) {
             return $cluster;
         } else {
             if (!$user['Role']['perm_galaxy_editor']) {

app/Model/MispObject.php

@@ -452,7 +452,7 @@ class MispObject extends AppModel
         return false;
     }
 
-    public function saveObject($object, $eventId, $template = false, $user, $errorBehaviour = 'drop', $breakOnDuplicate = false)
+    public function saveObject(array $object, $eventId, $template = false, $user, $errorBehaviour = 'drop', $breakOnDuplicate = false)
     {
         $templateFields = array(
             'name' => 'name',
@@ -482,7 +482,6 @@ class MispObject extends AppModel
             }
         }
         $this->create();
-        $result = false;
         if ($this->save($object)) {
             $result = $this->id;
             foreach ($object['Attribute'] as $k => $attribute) {
@@ -538,7 +537,6 @@ class MispObject extends AppModel
         $params = array(
             'conditions' => $this->buildConditions($user),
             'fields' => array(),
-            'recursive' => -1
         );
         if (isset($options['conditions'])) {
             $params['conditions']['AND'][] = $options['conditions'];
@@ -692,12 +690,14 @@ class MispObject extends AppModel
         return $results;
     }
 
-    /*
+    /**
      * Prepare the template form view's data, setting defaults, sorting elements
+     * @param array $template
+     * @param array $request
+     * @return array
      */
-    public function prepareTemplate($template, $request = array())
+    public function prepareTemplate(array $template, array $request = array())
     {
-        $temp = array();
         usort($template['ObjectTemplateElement'], function ($a, $b) {
             return $a['ui-priority'] < $b['ui-priority'];
         });
@@ -709,26 +709,28 @@ class MispObject extends AppModel
                 $request_rearranged[$attribute['object_relation']][] = $attribute;
             }
         }
-        foreach ($template_object_elements as $k => $v) {
+        $typeDefinitions = $this->Event->Attribute->typeDefinitions;
+        $categoryDefinitions = $this->Event->Attribute->categoryDefinitions;
+        foreach ($template_object_elements as $v) {
             if (empty($request_rearranged[$v['object_relation']])) {
-                if (isset($this->Event->Attribute->typeDefinitions[$v['type']])) {
+                if (isset($typeDefinitions[$v['type']])) {
-                    $v['default_category'] = $this->Event->Attribute->typeDefinitions[$v['type']]['default_category'];
+                    $v['default_category'] = $typeDefinitions[$v['type']]['default_category'];
-                    $v['to_ids'] = $this->Event->Attribute->typeDefinitions[$v['type']]['to_ids'];
+                    $v['to_ids'] = $typeDefinitions[$v['type']]['to_ids'];
                     if (empty($v['categories'])) {
                         $v['categories'] = array();
-                        foreach ($this->Event->Attribute->categoryDefinitions as $catk => $catv) {
+                        foreach ($categoryDefinitions as $catk => $catv) {
-                            if (in_array($v['type'], $catv['types'])) {
+                            if (in_array($v['type'], $catv['types'], true)) {
                                 $v['categories'][] = $catk;
                             }
                         }
                     }
                     $template['ObjectTemplateElement'][] = $v;
                 } else {
-                    $template['warnings'][] = 'Missing attribute type "' . $v['type'] . '" found. Omitted template element ("' . $template_object_elements[$k]['object_relation'] . '") that would not pass validation due to this.';
+                    $template['warnings'][] = 'Missing attribute type "' . $v['type'] . '" found. Omitted template element ("' . $v['object_relation'] . '") that would not pass validation due to this.';
                 }
             } else {
                 foreach ($request_rearranged[$v['object_relation']] as $request_item) {
-                    if (isset($this->Event->Attribute->typeDefinitions[$v['type']])) {
+                    if (isset($typeDefinitions[$v['type']])) {
                         $v['default_category'] = $request_item['category'];
                         $v['value'] = $request_item['value'];
                         $v['to_ids'] = $request_item['to_ids'];
@@ -741,8 +743,8 @@ class MispObject extends AppModel
                         }
                         if (empty($v['categories'])) {
                             $v['categories'] = array();
-                            foreach ($this->Event->Attribute->categoryDefinitions as $catk => $catv) {
+                            foreach ($categoryDefinitions as $catk => $catv) {
-                                if (in_array($v['type'], $catv['types'])) {
+                                if (in_array($v['type'], $catv['types'], true)) {
                                     $v['categories'][] = $catk;
                                 }
                             }
@@ -751,7 +753,7 @@ class MispObject extends AppModel
                         $template['ObjectTemplateElement'][] = $v;
                         unset($v['uuid']); // force creating a new attribute if template element entry gets reused
                     } else {
-                        $template['warnings'][] = 'Missing attribute type "' . $v['type'] . '" found. Omitted template element ("' . $template_object_elements[$k]['object_relation'] . '") that would not pass validation due to this.';
+                        $template['warnings'][] = 'Missing attribute type "' . $v['type'] . '" found. Omitted template element ("' . $v['object_relation'] . '") that would not pass validation due to this.';
                     }
                 }
             }
@@ -1215,36 +1217,40 @@ class MispObject extends AppModel
         return count($orphans);
     }
 
-    public function validObjectsFromAttributeTypes($user, $event_id, $selected_attribute_ids)
+    /**
+     * @param array $user
+     * @param int $eventId
+     * @param array $selectedAttributeIds
+     * @return array|array[]
+     * @throws Exception
+     */
+    public function validObjectsFromAttributeTypes(array $user, $eventId, array $selectedAttributeIds)
     {
-        $attributes = $this->Attribute->fetchAttributes($user,
+        $attributes = $this->Attribute->fetchAttributesSimple($user, [
-            array(
+            'conditions' => [
-                'conditions' => array(
+                'Attribute.id' => $selectedAttributeIds,
-                    'Attribute.id' => $selected_attribute_ids,
+                'Attribute.event_id' => $eventId,
-                    'Attribute.event_id' => $event_id,
+                'Attribute.object_id' => 0,
-                    'Attribute.object_id' => 0
+            ],
-                ),
+        ]);
-            )
-        );
         if (empty($attributes)) {
             return array('templates' => array(), 'types' => array());
         }
-        $attribute_types = array();
+        $attributeTypes = array();
         foreach ($attributes as $i => $attribute) {
-            $attribute_types[$attribute['Attribute']['type']] = 1;
+            $attributeTypes[$attribute['Attribute']['type']] = true;
             $attributes[$i]['Attribute']['object_relation'] = $attribute['Attribute']['type'];
         }
-        $attribute_types = array_keys($attribute_types);
+        $attributeTypes = array_keys($attributeTypes);
 
-        $potential_templates = $this->ObjectTemplate->find('list', array(
+        $potentialTemplateIds = $this->ObjectTemplate->find('column', array(
             'recursive' => -1,
             'fields' => array(
                 'ObjectTemplate.id',
-                'COUNT(ObjectTemplateElement.type) as type_count'
             ),
             'conditions' => array(
                 'ObjectTemplate.active' => true,
-                'ObjectTemplateElement.type' => $attribute_types
+                'ObjectTemplateElement.type' => $attributeTypes,
             ),
             'joins' => array(
                 array(
@@ -1256,15 +1262,13 @@ class MispObject extends AppModel
                 )
             ),
             'group' => 'ObjectTemplate.id',
-            'order' => 'type_count DESC'
         ));
 
-        $potential_template_ids = array_keys($potential_templates);
+        $templates = $this->ObjectTemplate->find('all', [
-        $templates = $this->ObjectTemplate->find('all', array(
             'recursive' => -1,
-            'conditions' => array('id' => $potential_template_ids),
+            'conditions' => ['id' => $potentialTemplateIds],
-            'contain' => 'ObjectTemplateElement'
+            'contain' => ['ObjectTemplateElement' => ['fields' => ['object_relation', 'type', 'multiple']]]
-        ));
+        ]);
 
         foreach ($templates as $i => $template) {
             $res = $this->ObjectTemplate->checkTemplateConformityBasedOnTypes($template, $attributes);
@@ -1272,21 +1276,16 @@ class MispObject extends AppModel
             $templates[$i]['ObjectTemplate']['invalidTypes'] = $res['invalidTypes'];
             $templates[$i]['ObjectTemplate']['invalidTypesMultiple'] = $res['invalidTypesMultiple'];
         }
-        return array('templates' => $templates, 'types' => $attribute_types);
+        return array('templates' => $templates, 'types' => $attributeTypes);
     }
 
-    public function groupAttributesIntoObject($user, $event_id, $object, $template, $selected_attribute_ids, $selected_object_relation_mapping, $hard_delete_attribute)
+    public function groupAttributesIntoObject(array $user, $event_id, array $object, $template, array $selected_attribute_ids, array $selected_object_relation_mapping, $hard_delete_attribute)
     {
         $saved_object_id = $this->saveObject($object, $event_id, $template, $user);
         if (!is_numeric($saved_object_id)) {
             return $saved_object_id;
         }
 
-        $saved_object = $this->find('first', array(
-            'recursive' => -1,
-            'conditions' => array('Object.id' => $saved_object_id)
-        ));
-
         $existing_attributes = $this->Attribute->fetchAttributes($user, array('conditions' => array(
             'Attribute.id' => $selected_attribute_ids,
             'Attribute.event_id' => $event_id,
@@ -1299,7 +1298,7 @@ class MispObject extends AppModel
         $event = array('Event' => $existing_attributes[0]['Event']);
 
         // Duplicate the attribute and its context, otherwise connected instances will drop the duplicated UUID
-        foreach ($existing_attributes as $i => $existing_attribute) {
+        foreach ($existing_attributes as $existing_attribute) {
             if (isset($selected_object_relation_mapping[$existing_attribute['Attribute']['id']])) {
                 $sightings = $this->Event->Sighting->attachToEvent($event, $user, $existing_attribute['Attribute']['id']);
                 $object_relation = $selected_object_relation_mapping[$existing_attribute['Attribute']['id']];
@@ -1308,20 +1307,18 @@ class MispObject extends AppModel
                 unset($created_attribute['id']);
                 unset($created_attribute['uuid']);
                 $created_attribute['object_relation'] = $object_relation;
-                $created_attribute['object_id'] = $saved_object['Object']['id'];
+                $created_attribute['object_id'] = $saved_object_id;
                 if (isset($existing_attribute['AttributeTag'])) {
                     $created_attribute['AttributeTag'] = $existing_attribute['AttributeTag'];
                 }
                 if (!empty($sightings)) {
                     $created_attribute['Sighting'] = $sightings;
                 }
-                $saved_object['Attribute'][$i] = $created_attribute;
+                $this->Attribute->captureAttribute($created_attribute, $event_id, $user, $saved_object_id);
-                $this->Attribute->captureAttribute($created_attribute, $event_id, $user, $saved_object['Object']['id']);
                 $this->Attribute->deleteAttribute($existing_attribute['Attribute']['id'], $user, $hard_delete_attribute);
             }
         }
-        return $saved_object['Object']['id'];
+        return $saved_object_id;
-
     }
 
     public function resolveUpdatedTemplate($template, $object, $update_template_available = false)
@@ -1395,9 +1392,7 @@ class MispObject extends AppModel
             }
             $toReturn['updateable_attribute'] = $object['Attribute'];
             $toReturn['not_updateable_attribute'] = array();
-        } else {
+
-            $toReturn['newer_template_version'] = false;
-        }
             if (!empty($template_difference)) { // older template not completely embeded in newer
                 foreach ($template_difference as $temp_diff_element) {
                     foreach ($object['Attribute'] as $i => $attribute) {
@@ -1412,13 +1407,16 @@ class MispObject extends AppModel
                     }
                 }
             }
+        }
+
         if ($update_template_available) { // template version bump requested
             $toReturn['template'] = $newer_template; // bump the template version
         }
         return $toReturn;
     }
 
-    public function reviseObject($revised_object, $object, $template) {
+    public function reviseObject($revised_object, $object, $template)
+    {
         $revised_object_both = array('mergeable' => array(), 'notMergeable' => array());
 
         // Loop through attributes to inject and perform the correct action

app/Model/ObjectTemplate.php

@@ -206,18 +206,27 @@ class ObjectTemplate extends AppModel
         return true;
     }
 
-    public function checkTemplateConformityBasedOnTypes($template, $attributes)
+    /**
+     * @param array $template
+     * @param array $attributes
+     * @return array
+     */
+    public function checkTemplateConformityBasedOnTypes(array $template, array $attributes)
     {
         $to_return = array('valid' => true, 'missingTypes' => array());
         if (!empty($template['ObjectTemplate']['requirements'])) {
+            // construct array containing ObjectTemplateElement with object_relation as key for faster search
+            $elementsByObjectRelationName = array_column($template['ObjectTemplateElement'], null, 'object_relation');
+
             // check for all required attributes
             if (!empty($template['ObjectTemplate']['requirements']['required'])) {
                 foreach ($template['ObjectTemplate']['requirements']['required'] as $requiredField) {
-                    $requiredType = Hash::extract($template['ObjectTemplateElement'], sprintf('{n}[object_relation=%s].type', $requiredField))[0];
+                    $requiredType = $elementsByObjectRelationName[$requiredField]['type'];
                     $found = false;
                     foreach ($attributes as $attribute) {
-                        if ($attribute['Attribute']['type'] == $requiredType) {
+                        if ($attribute['Attribute']['type'] === $requiredType) {
                             $found = true;
+                            break;
                         }
                     }
                     if (!$found) {
@@ -228,24 +237,24 @@ class ObjectTemplate extends AppModel
             // check for all required one of attributes
             if (!empty($template['ObjectTemplate']['requirements']['requiredOneOf'])) {
                 $found = false;
-                $all_required_type = array();
+                $allRequiredTypes = array();
                 foreach ($template['ObjectTemplate']['requirements']['requiredOneOf'] as $requiredField) {
-                    $requiredType = Hash::extract($template['ObjectTemplateElement'], sprintf('{n}[object_relation=%s].type', $requiredField));
+                    $requiredType = $elementsByObjectRelationName[$requiredField]['type'] ?? null;
-                    $requiredType = empty($requiredType) ? NULL : $requiredType[0];
+                    $allRequiredTypes[] = $requiredType;
-                    $all_required_type[] = $requiredType;
                     foreach ($attributes as $attribute) {
-                        if ($attribute['Attribute']['type'] == $requiredType) {
+                        if ($attribute['Attribute']['type'] === $requiredType) {
                             $found = true;
+                            break;
                         }
                     }
                 }
                 if (!$found) {
-                    $to_return = array('valid' => false, 'missingTypes' => $all_required_type);
+                    $to_return = array('valid' => false, 'missingTypes' => $allRequiredTypes);
                 }
             }
         }
 
-        // at this point, an object could created; checking if all attribute are valids
+        // at this point, an object could created; checking if all attribute are valid
         $valid_types = array();
         $to_return['invalidTypes'] = array();
         $to_return['invalidTypesMultiple'] = array();
@@ -266,8 +275,8 @@ class ObjectTemplate extends AppModel
                 $to_return['invalidTypes'][] = $attribute['Attribute']['type'];
             }
         }
-        $to_return['invalidTypes'] = array_unique($to_return['invalidTypes']);
+        $to_return['invalidTypes'] = array_unique($to_return['invalidTypes'], SORT_REGULAR);
-        $to_return['invalidTypesMultiple'] = array_unique($to_return['invalidTypesMultiple']);
+        $to_return['invalidTypesMultiple'] = array_unique($to_return['invalidTypesMultiple'], SORT_REGULAR);
         if (!empty($to_return['invalidTypesMultiple'])) {
             $to_return['valid'] = false;
         }
@@ -344,6 +353,10 @@ class ObjectTemplate extends AppModel
         return FileAccessTool::readJsonFromFile($path);
     }
 
+    /**
+     * @return Generator<array>
+     * @throws Exception
+     */
     private function readTemplatesFromDisk()
     {
         foreach ($this->getTemplateDirectoryPaths() as $dirpath) {
@@ -355,8 +368,13 @@ class ObjectTemplate extends AppModel
         }
     }
 
+    /**
+     * @param bool $fullPath
+     * @return array
+     */
     private function getTemplateDirectoryPaths($fullPath=true)
     {
+        App::uses('Folder', 'Utility');
         $dir = new Folder(self::OBJECTS_DIR, false);
         return $dir->read(true, false, $fullPath)[0];
     }

app/Model/ObjectTemplateElement.php

@@ -1,5 +1,4 @@
 <?php
-
 App::uses('AppModel', 'Model');
 
 class ObjectTemplateElement extends AppModel
@@ -8,23 +7,17 @@ class ObjectTemplateElement extends AppModel
         'Containable'
     );
 
-    public $belongsTo = array(
-    );
-
-    public $validate = array(
-    );
-
     public function afterFind($results, $primary = false)
     {
-        foreach ($results as $k => $result) {
+        foreach ($results as &$result) {
-            if (isset($results[$k]['ObjectTemplateElement']['categories'])) {
+            if (isset($result['ObjectTemplateElement']['categories'])) {
-                $results[$k]['ObjectTemplateElement']['categories'] = json_decode($results[$k]['ObjectTemplateElement']['categories'], true);
+                $result['ObjectTemplateElement']['categories'] = json_decode($result['ObjectTemplateElement']['categories'], true);
             }
-            if (isset($results[$k]['ObjectTemplateElement']['values_list'])) {
+            if (isset($result['ObjectTemplateElement']['values_list'])) {
-                $results[$k]['ObjectTemplateElement']['values_list'] = json_decode($results[$k]['ObjectTemplateElement']['values_list'], true);
+                $result['ObjectTemplateElement']['values_list'] = json_decode($result['ObjectTemplateElement']['values_list'], true);
             }
-            if (isset($results[$k]['ObjectTemplateElement']['sane_default'])) {
+            if (isset($result['ObjectTemplateElement']['sane_default'])) {
-                $results[$k]['ObjectTemplateElement']['sane_default'] = json_decode($results[$k]['ObjectTemplateElement']['sane_default'], true);
+                $result['ObjectTemplateElement']['sane_default'] = json_decode($result['ObjectTemplateElement']['sane_default'], true);
             }
         }
         return $results;
@@ -37,13 +30,13 @@ class ObjectTemplateElement extends AppModel
         }
         $json_fields = array('categories', 'values_list', 'sane_default');
         foreach ($json_fields as $field) {
-            $this->data['ObjectTemplateElement'][$field] = empty($this->data['ObjectTemplateElement'][$field]) ? '[]' : json_encode($this->data['ObjectTemplateElement'][$field]);
+            $this->data['ObjectTemplateElement'][$field] = empty($this->data['ObjectTemplateElement'][$field]) ? '[]' : JsonTool::encode($this->data['ObjectTemplateElement'][$field]);
         }
         return true;
     }
 
-
+    public function getAllAvailableTypes()
-    public function getAllAvailableTypes() {
+    {
         $temp = $this->find('all', array(
             'recursive' => -1,
             'fields' => array('object_relation as type', 'description AS desc', 'categories'),

app/Model/OverCorrelatingValue.php

@@ -5,10 +5,6 @@ class OverCorrelatingValue extends AppModel
 {
     public $recursive = -1;
 
-    public $actsAs = array(
-        'Containable'
-    );
-
     public function beforeValidate($options = array())
     {
         $this->data['OverCorrelatingValue']['value'] = self::truncate($this->data['OverCorrelatingValue']['value']);
@@ -70,7 +66,7 @@ class OverCorrelatingValue extends AppModel
      */
     public function getLimit()
     {
-        return Configure::check('MISP.correlation_limit') ? Configure::read('MISP.correlation_limit') : 20;
+        return Configure::read('MISP.correlation_limit') ?: 20;
     }
 
     public function getOverCorrelations($query)
@@ -87,19 +83,13 @@ class OverCorrelatingValue extends AppModel
         return $data;
     }
 
-    public function checkValue($value)
+    public function findOverCorrelatingValues(array $valuesToCheck): array
     {
-        return $this->hasAny(['value' => self::truncate($value)]);
+        $valuesToCheck = array_unique(self::truncateValues($valuesToCheck), SORT_REGULAR);
-    }
+        return $this->find('column', [
-
+            'conditions' => ['value' => $valuesToCheck],
-    public function findOverCorrelatingValues(array $values_to_check): array
-    {
-        $values_to_check_truncated = array_unique(self::truncateValues($values_to_check));
-        $overCorrelatingValues = $this->find('column', [
-            'conditions' => ['value' => $values_to_check_truncated],
             'fields' => ['value'],
         ]);
-        return $overCorrelatingValues;
     }
 
     public function generateOccurrencesRouter()
@@ -139,14 +129,23 @@ class OverCorrelatingValue extends AppModel
         ]);
         $this->Attribute = ClassRegistry::init('Attribute');
         foreach ($overCorrelations as &$overCorrelation) {
+            $value = $overCorrelation['OverCorrelatingValue']['value'] . '%';
             $count = $this->Attribute->find('count', [
                 'recursive' => -1,
                 'conditions' => [
                     'OR' => [
-                        'Attribute.value1 LIKE' => $overCorrelation['OverCorrelatingValue']['value'] . '%',
+                        'Attribute.value1 LIKE' => $value,
-                        'Attribute.value2 LIKE' => $overCorrelation['OverCorrelatingValue']['value'] . '%'
+                        'AND' => [
-                    ]
+                            'Attribute.value2 LIKE' => $value,
-                ]
+                            'NOT' => ['Attribute.type' => Attribute::PRIMARY_ONLY_CORRELATING_TYPES]
+                        ],
+                    ],
+                    'NOT' => ['Attribute.type' => Attribute::NON_CORRELATING_TYPES],
+                    'Attribute.disable_correlation' => 0,
+                    'Event.disable_correlation' => 0,
+                    'Attribute.deleted' => 0,
+                ],
+                'contain' => ['Event'],
             ]);
             $overCorrelation['OverCorrelatingValue']['occurrence'] = $count;
         }

app/Model/Server.php

@@ -659,7 +659,7 @@ class Server extends AppModel
         }
 
         $change = sprintf(
-            '%s events, %s proposals, %s sightings and %s galaxyClusters pulled or updated. %s events failed or didn\'t need an update.',
+            '%s events, %s proposals, %s sightings and %s galaxy clusters pulled or updated. %s events failed or didn\'t need an update.',
             count($successes),
             $pulledProposals,
             $pulledSightings,
@@ -796,6 +796,7 @@ class Server extends AppModel
      * @return array
      * @throws HttpSocketHttpException
      * @throws HttpSocketJsonException
+     * @throws JsonException
      */
     public function getEventIndexFromServer(ServerSyncTool $serverSync, $ignoreFilterRules = false)
     {
@@ -809,12 +810,37 @@ class Server extends AppModel
         }
         $filterRules['minimal'] = 1;
         $filterRules['published'] = 1;
-        $eventIndex = $serverSync->eventIndex($filterRules)->json();
+
+        // Fetch event index from cache if exists and is not modified
+        $redis = RedisTool::init();
+        $indexFromCache = $redis->get("misp:event_index:{$serverSync->serverId()}");
+        if ($indexFromCache) {
+            list($etag, $eventIndex) = RedisTool::deserialize($indexFromCache);
+        } else {
+            $etag = '""';  // Provide empty ETag, so MISP will compute ETag for returned data
+        }
+
+        $response = $serverSync->eventIndex($filterRules, $etag);
+
+        if ($response->isNotModified() && $indexFromCache) {
+            return $eventIndex;
+        }
+
+        $eventIndex = $response->json();
 
         // correct $eventArray if just one event, probably this response returns old MISP
         if (isset($eventIndex['id'])) {
             $eventIndex = [$eventIndex];
         }
+
+        // Save to cache for 24 hours if ETag provided
+        if (isset($response->headers["ETag"])) {
+            $data = RedisTool::serialize([$response->headers["ETag"], $eventIndex]);
+            $redis->setex("misp:event_index:{$serverSync->serverId()}", 3600 * 24, $data);
+        } else if ($indexFromCache) {
+            $redis->del("misp:event_index:{$serverSync->serverId()}");
+        }
+
         return $eventIndex;
     }
 
@@ -5708,7 +5734,7 @@ class Server extends AppModel
                 ),
                 'custom_css' => array(
                     'level' => 2,
-                    'description' => __('If you would like to customise the css, simply drop a css file in the /var/www/MISP/app/webroot/css directory and enter the name here.'),
+                    'description' => __('If you would like to customise the CSS, simply drop a css file in the /var/www/MISP/app/webroot/css directory and enter the name here.'),
                     'value' => '',
                     'test' => 'testForStyleFile',
                     'type' => 'string',
@@ -5776,6 +5802,22 @@ class Server extends AppModel
                     'type' => 'string',
                     'redacted' => true
                 ),
+                'redis_serializer' => [
+                    'level' => self::SETTING_OPTIONAL,
+                    'description' => __('Redis serializer method. WARNING: Changing this setting will drop some cached data.'),
+                    'value' => 'JSON',
+                    'test' => null,
+                    'type' => 'string',
+                    'null' => true,
+                    'afterHook' => function () {
+                        $keysToDelete = ['taxonomies_cache:*', 'misp:warninglist_cache', 'misp:event_lock:*', 'misp:event_index:*'];
+                        $redis = RedisTool::init();
+                        foreach ($keysToDelete as $key) {
+                            $redis->unlink($redis->keys($key));
+                        }
+                        return true;
+                    },
+                ],
                 'event_view_filter_fields' => array(
                     'level' => 2,
                     'description' => __('Specify which fields to filter on when you search on the event view. Default values are : "id, uuid, value, comment, type, category, Tag.name"'),

app/Model/Taxonomy.php

@@ -126,6 +126,9 @@ class Taxonomy extends AppModel
         return (int)$result;
     }
 
+    /**
+     * @throws Exception
+     */
     private function __updateVocab(array $vocab, array $current)
     {
         $enabled = 0;
@@ -149,6 +152,9 @@ class Taxonomy extends AppModel
         }
         if (!empty($vocab['values'])) {
             foreach ($vocab['values'] as $value) {
+                if (!isset($predicateLookup[$value['predicate']])) {
+                    throw new Exception("Invalid taxonomy `{$vocab['namespace']}` provided. Predicate `{$value['predicate']}` is missing.");
+                }
                 $predicatePosition = $predicateLookup[$value['predicate']];
                 if (empty($taxonomy['Taxonomy']['TaxonomyPredicate'][$predicatePosition]['TaxonomyEntry'])) {
                     $taxonomy['Taxonomy']['TaxonomyPredicate'][$predicatePosition]['TaxonomyEntry'] = $value['entry'];
@@ -194,30 +200,34 @@ class Taxonomy extends AppModel
         foreach ($taxonomy['TaxonomyPredicate'] as $predicate) {
             if (isset($predicate['TaxonomyEntry']) && !empty($predicate['TaxonomyEntry'])) {
                 foreach ($predicate['TaxonomyEntry'] as $entry) {
-                    $temp = array('tag' => $taxonomy['Taxonomy']['namespace'] . ':' . $predicate['value'] . '="' . $entry['value'] . '"');
+                    $temp = [
-                    $temp['expanded'] = (!empty($predicate['expanded']) ? $predicate['expanded'] : $predicate['value']) . ': ' . (!empty($entry['expanded']) ? $entry['expanded'] : $entry['value']);
+                        'tag' => $taxonomy['Taxonomy']['namespace'] . ':' . $predicate['value'] . '="' . $entry['value'] . '"',
-                    if (isset($entry['description']) && !empty($entry['description'])) {
+                        'expanded' => (!empty($predicate['expanded']) ? $predicate['expanded'] : $predicate['value']) . ': ' . (!empty($entry['expanded']) ? $entry['expanded'] : $entry['value']),
+                        'exclusive_predicate' => $predicate['exclusive'],
+                    ];
+                    if (!empty($entry['description'])) {
                         $temp['description'] = $entry['description'];
                     }
-                    if (isset($entry['colour']) && !empty($entry['colour'])) {
+                    if (!empty($entry['colour'])) {
                         $temp['colour'] = $entry['colour'];
                     }
-                    if (isset($entry['numerical_value']) && $entry['numerical_value'] !== null) {
+                    if (isset($entry['numerical_value'])) {
                         $temp['numerical_value'] = $entry['numerical_value'];
                     }
-                    $temp['exclusive_predicate'] = $predicate['exclusive'];
                     $entries[] = $temp;
                 }
             } else {
-                $temp = array('tag' => $taxonomy['Taxonomy']['namespace'] . ':' . $predicate['value']);
+                $temp = [
-                $temp['expanded'] = !empty($predicate['expanded']) ? $predicate['expanded'] : $predicate['value'];
+                    'tag' => $taxonomy['Taxonomy']['namespace'] . ':' . $predicate['value'],
-                if (isset($predicate['description']) && !empty($predicate['description'])) {
+                    'expanded' => !empty($predicate['expanded']) ? $predicate['expanded'] : $predicate['value']
+                ];
+                if (!empty($predicate['description'])) {
                     $temp['description'] = $predicate['description'];
                 }
-                if (isset($predicate['colour']) && !empty($predicate['colour'])) {
+                if (!empty($predicate['colour'])) {
                     $temp['colour'] = $predicate['colour'];
                 }
-                if (isset($predicate['numerical_value']) && $predicate['numerical_value'] !== null) {
+                if (isset($predicate['numerical_value'])) {
                     $temp['numerical_value'] = $predicate['numerical_value'];
                 }
                 $entries[] = $temp;
@@ -589,9 +599,9 @@ class Taxonomy extends AppModel
             return false; // not taxonomy tag
         }
 
-        $key = 'taxonomies_cache:tagName=' . $tagName . "&" . "metaOnly=$metaOnly" . "&" . "fullTaxonomy=$fullTaxonomy";
+        $key = "taxonomies_cache:tagName=$tagName&metaOnly=$metaOnly&fullTaxonomy=$fullTaxonomy";
         $redis = $this->setupRedis();
-        $taxonomy = $redis ? json_decode($redis->get($key), true) : null;
+        $taxonomy = $redis ? RedisTool::deserialize($redis->get($key)) : null;
 
         if (!$taxonomy) {
             if (isset($splits['value'])) {
@@ -634,7 +644,7 @@ class Taxonomy extends AppModel
             }
 
             if ($redis) {
-                $redis->setex($key, 1800, json_encode($taxonomy));
+                $redis->setex($key, 1800, RedisTool::serialize($taxonomy));
             }
         }
 
@@ -753,7 +763,7 @@ class Taxonomy extends AppModel
 
     /**
      * @param string $tag
-     * @return array|null
+     * @return array|null Returns null if tag is not in taxonomy format
      */
     public function splitTagToComponents($tag)
     {

app/Model/User.php

@@ -493,14 +493,6 @@ class User extends AppModel
         return $fails;
     }
 
-    public function getOrgMemberCount($org)
-    {
-        return $this->find('count', array(
-                'conditions' => array(
-                        'org =' => $org,
-                )));
-    }
-
     /**
      * 0 - true if key is valid
      * 1 - User e-mail
@@ -838,7 +830,7 @@ class User extends AppModel
      * @param array $user
      * @param SendEmailTemplate|string $body
      * @param string|false $bodyNoEnc
-     * @param string $subject
+     * @param string|null $subject
      * @param array|false $replyToUser
      * @return bool
      * @throws Crypt_GPG_BadPassphraseException
@@ -1649,56 +1641,88 @@ class User extends AppModel
         return substr(hash('sha256', "{$user['id']}|$salt"), 0, 8);
     }
 
-    public function extractPeriodicSettingForUser($user, $decode=false): array
+    /**
+     * @param int $userId
+     * @param bool $decode
+     * @return array
+     * @throws JsonException
+     */
+    public function fetchPeriodicSettingForUser($userId, $decode = false): array
     {
-        $filter_names = ['orgc_id', 'distribution', 'sharing_group_id', 'event_info', 'tags', 'trending_for_tags'];
+        $filterNames = ['orgc_id', 'distribution', 'sharing_group_id', 'event_info', 'tags', 'trending_for_tags', 'include_correlations', 'trending_period_amount'];
-        $filter_to_decode = ['tags', 'trending_for_tags', ];
+        $filterToDecode = ['tags', 'trending_for_tags'];
-        if (is_numeric($user)) {
+        $defaultPeriodicSettings = [
-            $user = $this->find('first', [
-                'recursive' => -1,
-                'conditions' => ['User.id' => $user],
-                'contain' => [
-                    'UserSetting',
-                ]
-            ]);
-        }
-        $periodic_settings = array_values(array_filter($user['UserSetting'], function ($userSetting) {
-            return $userSetting['setting'] == self::PERIODIC_USER_SETTING_KEY;
-        }));
-        if (empty($periodic_settings)) {
-            $periodic_settings = [['value' => [
             'orgc_id' => '',
             'distribution' => -1,
             'sharing_group_id' => '',
             'event_info' => '',
             'tags' => '[]',
-                'trending_for_tags' => '[]'
+            'trending_for_tags' => '[]',
-            ]]];
+            'include_correlations' => '',
+            'trending_period_amount' => 2,
+        ];
+
+        $periodicSettings = $this->UserSetting->find('first', [
+            'recursive' => -1,
+            'conditions' => [
+                'user_id' => $userId,
+                'setting' => self::PERIODIC_USER_SETTING_KEY,
+            ],
+            'fields' => ['value'],
+        ]);
+        if (empty($periodicSettings)) {
+            $periodicSettings = $defaultPeriodicSettings;
+        } else {
+            $periodicSettings = $periodicSettings['UserSetting']['value'];
         }
-        $periodic_settings_indexed = [];
+        $periodicSettingsIndexed = [];
-        if (!empty($periodic_settings)) {
+        foreach ($filterNames as $filterName) {
-            foreach ($filter_names as $filter_name) {
+            $periodicSettingsIndexed[$filterName] = $periodicSettings[$filterName] ?? $defaultPeriodicSettings[$filterName];
-                $periodic_settings_indexed[$filter_name] = $periodic_settings[0]['value'][$filter_name];
+        }
+        if ($decode) {
+            foreach ($filterToDecode as $filter) {
+                if (!empty($periodicSettingsIndexed[$filter])) {
+                    $periodicSettingsIndexed[$filter] = JsonTool::decode($periodicSettingsIndexed[$filter]);
                 }
             }
-        foreach ($filter_to_decode as $filter) {
-            if (!empty($decode) && !empty($periodic_settings_indexed[$filter])) {
-                $periodic_settings_indexed[$filter] = JsonTool::decode($periodic_settings_indexed[$filter]);
         }
-        }
+        return $periodicSettingsIndexed;
-        return $periodic_settings_indexed;
     }
 
-    public function getUsablePeriodicSettingForUser(array $periodicSettings, $period='daily'): array
+    /**
+     * @param array $period_filters
+     * @param string $period
+     * @return array
+     */
+    private function getUsablePeriodicSettingForUser(array $period_filters, $period='daily'): array
     {
-        return $this->__getUsableFilters($periodicSettings, $period);
+        $filters = [
+            'last' => $this->__genTimerangeFilter($period),
+            'published' => true,
+        ];
+        if (!empty($period_filters['orgc_id'])) {
+            $filters['orgc_id'] = $period_filters['orgc_id'];
+        }
+        if (isset($period_filters['distribution']) && $period_filters['distribution'] >= 0) {
+            $filters['distribution'] = intval($period_filters['distribution']);
+        }
+        if (!empty($period_filters['sharing_group_id'])) {
+            $filters['sharing_group_id'] = $period_filters['sharing_group_id'];
+        }
+        if (!empty($period_filters['event_info'])) {
+            $filters['event_info'] = $period_filters['event_info'];
+        }
+        if (!empty($period_filters['tags'])) {
+            $filters['tags'] = $period_filters['tags'];
+        }
+        return $filters;
     }
 
-    public function saveNotificationSettings(int $user_id, array $data): bool
+    public function saveNotificationSettings(int $userId, array $data): bool
     {
         $existingUser = $this->find('first', [
             'recursive' => -1,
-            'conditions' => ['User.id' => $user_id],
+            'conditions' => ['User.id' => $userId],
         ]);
         if (empty($existingUser)) {
             return false;
@@ -1707,11 +1731,11 @@ class User extends AppModel
             $existingUser['User'][$notification_period] = $data['User'][$notification_period];
         }
         $success = $this->save($existingUser, [
-            'fieldList' => self::PERIODIC_NOTIFICATIONS
+            'fieldList' => array_merge(self::PERIODIC_NOTIFICATIONS, ['date_modified']),
         ]);
         if ($success) {
             $periodic_settings = $data['periodic_settings'];
-            $param_to_decode = ['tags', 'trending_for_tags', ];
+            $param_to_decode = ['tags', 'trending_for_tags'];
             foreach ($param_to_decode as $param) {
                 if (empty($periodic_settings[$param])) {
                     $periodic_settings[$param] = '[]';
@@ -1729,6 +1753,8 @@ class User extends AppModel
                 'event_info' => $periodic_settings['event_info'] ?? '',
                 'tags' => $periodic_settings['tags'] ?? '[]',
                 'trending_for_tags' => $periodic_settings['trending_for_tags'] ?? '[]',
+                'include_correlations' => $periodic_settings['include_correlations'] ?? '',
+                'trending_period_amount' => $periodic_settings['trending_period_amount'] ?? 2,
             ];
             $new_user_setting = [
                 'UserSetting' => [
@@ -1746,40 +1772,45 @@ class User extends AppModel
     {
         return $this->find('all', [
             'recursive' => -1,
-            'conditions' => ["notification_$period" => true],
+            'conditions' => [
+                "notification_$period" => true,
+                'disabled' => false,
+            ],
         ]);
     }
 
     /**
      * generatePeriodicSummary
      *
-     * @param int $user_id
+     * @param int $userId
-     * @param string $period
+     * @param string $period Can be 'daily', 'weekly' or 'monthly'
+     * @param bool $rendered When false, instance of SendEmailTemplate will returned
      * @return string|SendEmailTemplate
      * @throws NotFoundException
      * @throws InvalidArgumentException
      */
-    public function generatePeriodicSummary(int $user_id, string $period, $rendered=true)
+    public function generatePeriodicSummary(int $userId, string $period, $rendered=true)
     {
-        $existingUser = $this->getUserById($user_id);
+        $allowedPeriods = array_map(function($period) {
-        $user = $this->rearrangeToAuthForm($existingUser);
-        $allowed_periods = array_map(function($period) {
             return substr($period, strlen('notification_'));
         }, self::PERIODIC_NOTIFICATIONS);
-        if (!in_array($period, $allowed_periods)) {
+        if (!in_array($period, $allowedPeriods, true)) {
             throw new InvalidArgumentException(__('Invalid period. Must be one of %s', JsonTool::encode(self::PERIODIC_NOTIFICATIONS)));
         }
+
+        $user = $this->getAuthUser($userId);
         App::import('Tools', 'SendEmail');
-        $emailTemplate = $this->prepareEmailTemplate($period);
+        $periodicSettings = $this->fetchPeriodicSettingForUser($userId, true);
-        $periodicSettings = $this->extractPeriodicSettingForUser($user, true);
         $filters = $this->getUsablePeriodicSettingForUser($periodicSettings, $period);
         $filtersForRestSearch = $filters; // filters for restSearch are slightly different than fetchEvent
         $filters['last'] = $this->resolveTimeDelta($filters['last']);
         $filters['sgReferenceOnly'] = true;
-        $filters['includeEventCorrelations'] = false;
+        $filters['includeEventCorrelations'] = !empty($periodicSettings['include_correlations']);
+        $filters['includeGranularCorrelations'] = !empty($periodicSettings['include_correlations']);
         $filters['noSightings'] = true;
-        $filters['includeGalaxy'] = false;
+        $filters['fetchFullClusters'] = false;
-        $events = $this->__getEventsForFilters($user, $filters);
+        $filters['includeScoresOnEvent'] = true;
+        $events = $this->Event->fetchEvent($user, $filters);
 
         $elementCounter = 0;
         $renderView = false;
@@ -1792,11 +1823,10 @@ class User extends AppModel
             unset($filtersForRestSearch['tags']);
         }
         $finalContext = $this->Event->restSearch($user, 'context', $filtersForRestSearch, false, false, $elementCounter, $renderView);
-        $finalContext = json_decode($finalContext->intoString(), true);
+        $finalContext = JsonTool::decode($finalContext->intoString());
         $aggregated_context = $this->__renderAggregatedContext($finalContext);
-
+        $rollingWindows = $periodicSettings['trending_period_amount'] ?: 2;
-        $rollingWindows = 2;
+        $trendAnalysis = $this->Event->getTrendsForTagsFromEvents($events, $this->periodToDays($period), $rollingWindows, $periodicSettings['trending_for_tags']);
-        $trendAnalysis = $this->Event->getTrendsForTagsFromEvents($events, $this->__periodToDays($period), $rollingWindows, $periodicSettings['trending_for_tags']);
         $tagFilterPrefixes = $periodicSettings['trending_for_tags'] ?: array_keys($trendAnalysis['all_tags']);
         $trendData = [
             'trendAnalysis' => $trendAnalysis,
@@ -1804,17 +1834,20 @@ class User extends AppModel
         ];
         $trending_summary = $this->__renderTrendingSummary($trendData);
 
+        $emailTemplate = $this->prepareEmailTemplate($period);
         $emailTemplate->set('baseurl', $this->Event->__getAnnounceBaseurl());
         $emailTemplate->set('events', $events);
         $emailTemplate->set('filters', $filters);
+        $emailTemplate->set('periodicSettings', $periodicSettings);
+        $emailTemplate->set('period_days', $this->periodToDays($period));
         $emailTemplate->set('period', $period);
         $emailTemplate->set('aggregated_context', $aggregated_context);
         $emailTemplate->set('trending_summary', $trending_summary);
         $emailTemplate->set('analysisLevels', $this->Event->analysisLevels);
         $emailTemplate->set('distributionLevels', $this->Event->distributionLevels);
-        if (!empty($rendered)) {
+        if ($rendered) {
             $summary = $emailTemplate->render();
-            return $summary->format() == 'text' ? $summary->text : $summary->html;
+            return $summary->format() === 'text' ? $summary->text : $summary->html;
         }
         return $emailTemplate;
     }
@@ -1866,33 +1899,23 @@ class User extends AppModel
         }
         return $filters;
     }
-
     private function __genTimerangeFilter(string $period='daily'): string
     {
-        $timerange = '1d';
+        return $this->periodToDays($period) . 'd';
-        if ($period == 'weekly') {
-            $timerange = '7d';
-        } else if ($period == 'monthly'){
-            $timerange = '31d';
-        }
-        return $timerange;
     }
 
-    private function __periodToDays(string $period='daily'): int
+    private function periodToDays(string $period='daily'): int
     {
-        return ($period == 'daily' ? 1 : (
+        if ($period === 'daily') {
-            $period == 'weekly' ? 7 : 31)
+            return 1;
-        );
+        } else if ($period === 'weekly') {
+            return 7;
+        } else {
+            return 31;
+        }
     }
 
-    private function __getEventsForFilters(array $user, array $filters): array
+    private function prepareEmailTemplate(string $period = 'daily'): SendEmailTemplate
-    {
-        $this->Event = ClassRegistry::init('Event');
-        $events = $this->Event->fetchEvent($user, $filters);
-        return $events;
-    }
-
-    public function prepareEmailTemplate(string $period='daily'): SendEmailTemplate
     {
         $subject = sprintf('[%s MISP] %s %s', Configure::read('MISP.org'), Inflector::humanize($period), __('Notification - %s', (new DateTime())->format('Y-m-d')));
         $template = new SendEmailTemplate("notification_$period");

app/Model/Warninglist.php

@@ -115,7 +115,7 @@ class Warninglist extends AppModel
         }
 
         try {
-            $redis = $this->setupRedisWithException();
+            $redis = RedisTool::init();
         } catch (Exception $e) {
             // fallback to default implementation when redis is not available
             $eventWarnings = [];
@@ -182,10 +182,10 @@ class Warninglist extends AppModel
                 }
 
                 $attributeKey = $keysToGet[$pos];
-                $saveToCache[$attributeKey] = empty($store) ? '' : json_encode($store);
+                $saveToCache[$attributeKey] = empty($store) ? '' : RedisTool::serialize($store);
 
             } elseif (!empty($result)) { // skip empty string that means no warning list match
-                $matchedWarningList = json_decode($result, true);
+                $matchedWarningList = RedisTool::deserialize($result);
                 foreach ($matchedWarningList as $warninglistId => $matched) {
                     $attributes[$redisResultToAttributePos[$pos]]['warnings'][] = [
                         'value' => $matched[0],
@@ -447,12 +447,7 @@ class Warninglist extends AppModel
             $redis->del($redis->keys('misp:warninglist_entries_cache:*'));
         }
 
-        $warninglists = $this->find('all', array(
+        $warninglists = $this->getEnabledAndCacheWarninglist();
-            'contain' => array('WarninglistType'),
-            'conditions' => array('enabled' => 1),
-            'fields' => ['id', 'name', 'type', 'category'],
-        ));
-        $this->cacheWarninglists($warninglists);
 
         foreach ($warninglists as $warninglist) {
             if ($id && $warninglist['Warninglist']['id'] != $id) {
@@ -467,17 +462,34 @@ class Warninglist extends AppModel
         return true;
     }
 
-    private function cacheWarninglists(array $warninglists)
+    /**
+     * Get enable warninglists and cache them.
+     * @return array
+     */
+    private function getEnabledAndCacheWarninglist()
     {
-        $redis = $this->setupRedis();
+        $warninglists = $this->find('all', [
-        if ($redis !== false) {
+            'contain' => ['WarninglistType'],
-            $redis->del('misp:warninglist_cache');
+            'conditions' => ['enabled' => 1],
-            foreach ($warninglists as $warninglist) {
+            'fields' => ['id', 'name', 'type', 'category'],
-                $redis->sAdd('misp:warninglist_cache', json_encode($warninglist));
+        ]);
+
+        // Convert type to array
+        foreach ($warninglists as &$warninglist) {
+            $warninglist['types'] = [];
+            foreach ($warninglist['WarninglistType'] as $wt) {
+                $warninglist['types'][] = $wt['type'];
             }
-            return true;
+            unset($warninglist['WarninglistType']);
         }
-        return false;
+
+        try {
+            $redis = RedisTool::init();
+            $redis->set('misp:warninglist_cache', RedisTool::serialize($warninglists));
+        } catch (Exception $e) {
+        }
+
+        return $warninglists;
     }
 
     private function cacheWarninglistEntries(array $warninglistEntries, $id)
@@ -500,6 +512,7 @@ class Warninglist extends AppModel
 
     /**
      * @return array
+     * @throws JsonException
      */
     public function getEnabled()
     {
@@ -507,28 +520,18 @@ class Warninglist extends AppModel
             return $this->enabledCache;
         }
 
-        $redis = $this->setupRedis();
+        try {
-        if ($redis !== false && $redis->exists('misp:warninglist_cache')) {
+            $redis = RedisTool::init();
-            $warninglists = $redis->sMembers('misp:warninglist_cache');
+            $warninglists = RedisTool::deserialize($redis->get('misp:warninglist_cache'));
-            foreach ($warninglists as $k => $v) {
+        } catch (Exception $e) {
-                $warninglists[$k] = json_decode($v, true);
+            $warninglists = false;
-            }
-        } else {
-            $warninglists = $this->find('all', array(
-                'contain' => ['WarninglistType'],
-                'conditions' => ['enabled' => 1],
-                'fields' => ['id', 'name', 'type', 'category'],
-            ));
-            $this->cacheWarninglists($warninglists);
         }
 
-        foreach ($warninglists as &$warninglist) {
+        // $warninglists is false when nothing is cached
-            $warninglist['types'] = [];
+        if ($warninglists === false) {
-            foreach ($warninglist['WarninglistType'] as $wt) {
+            $warninglists = $this->getEnabledAndCacheWarninglist();
-                $warninglist['types'][] = $wt['type'];
-            }
-            unset($warninglist['WarninglistType']);
         }
+
         $this->enabledCache = $warninglists;
         return $warninglists;
     }
@@ -539,10 +542,14 @@ class Warninglist extends AppModel
      */
     private function getWarninglistEntries($id)
     {
-        $redis = $this->setupRedis();
+        try {
-        if ($redis !== false && $redis->exists('misp:warninglist_entries_cache:' . $id)) {
+            $redis = RedisTool::init();
-            return $redis->sMembers('misp:warninglist_entries_cache:' . $id);
+            $entries = $redis->sMembers('misp:warninglist_entries_cache:' . $id);
-        } else {
+            if (!empty($entries)) {
+                return $entries;
+            }
+        } catch (Exception $e) {}
+
         $entries = $this->WarninglistEntry->find('column', array(
             'conditions' => array('warninglist_id' => $id),
             'fields' => array('WarninglistEntry.value')
@@ -550,7 +557,6 @@ class Warninglist extends AppModel
         $this->cacheWarninglistEntries($entries, $id);
         return $entries;
     }
-    }
 
     /**
      * For 'hostname', 'string' and 'cidr' warninglist type, values are just in keys to save memory.

app/Model/Workflow.php

@@ -723,7 +723,7 @@ class Workflow extends AppModel
      * @return array
      * @throws ModuleNotFoundException
      */
-    public function getModuleConfigByType($module_type, $id, $throwException=false): array
+    public function getModuleConfigByType($module_type, $id, $throwException=false): ?array
     {
         $this->loadAllWorkflowModules();
         $moduleConfig = $this->loaded_modules[$module_type][$id] ?? null;

app/Test/AttributeValidationToolTest.php

@@ -135,6 +135,12 @@ class AttributeValidationToolTest extends TestCase
         $this->assertEquals('xn--hkyrky-ptac70bc.cz|127.0.0.1', AttributeValidationTool::modifyBeforeValidation('domain|ip', 'HÁČKYČÁRKY.CZ|127.0.0.1'));
     }
 
+    public function testSssdeep()
+    {
+        $this->shouldBeValid('ssdeep', ["768:+OFu8Q3w6QzfR5Jni6SQD7qSFDs6P93/q0XIc/UB5EPABWX:RFu8QAFzffJui79f13/AnB5EPAkX"]);
+        $this->shouldBeInvalid('ssdeep', ["768:+OFu8Q3w6QzfR5Jni6SQD7qSFDs6P93/q0XIc/UB5EPABWX\n\n:RFu8QAFzffJui79f13/AnB5EPAkX"]);
+    }
+
     private function shouldBeValid($type, array $values)
     {
         foreach ($values as $value) {

app/View/Correlations/over_correlations.ctp

@@ -11,17 +11,20 @@
                         [
                             'type' => 'simple',
                             'url' => $baseurl . '/correlations/overCorrelations',
-                                'text' => __('All')
+                            'text' => __('All'),
+                            'active' => $scope === 'all',
                         ],
                         [
                             'type' => 'simple',
                             'url' => $baseurl . '/correlations/overCorrelations/scope:over_correlating',
-                                'text' => __('Over-correlating')
+                            'text' => __('Over-correlating'),
+                            'active' => $scope === 'over_correlating',
                         ],
                         [
                             'type' => 'simple',
                             'url' => $baseurl . '/correlations/overCorrelations/scope:not_over_correlating',
-                                'text' => __('Not over-correlating')
+                            'text' => __('Not over-correlating'),
+                            'active' => $scope === 'not_over_correlating',
                         ],
                         [
                             'type' => 'simple',

app/View/Elements/Events/View/attribute_correlations.ctp

@@ -1,7 +1,7 @@
 <?php
     if (!empty($event['Related' . $scope][$object['id']])) {
         $i = 0;
-        $linkColour = ($scope == 'Attribute') ? 'red' : 'white';
+        $linkColour = $scope === 'Attribute' ? 'red' : 'white';
         $withPivot = isset($withPivot) ? $withPivot : false;
         // remove duplicates
         $relatedEvents = array();
@@ -15,7 +15,7 @@
         $count = count($event['Related' . $scope][$object['id']]);
         foreach ($event['Related' . $scope][$object['id']] as $relatedAttribute) {
             if ($i == 4 && $count > 5) {
-                $expandButton = __('Show %s more...', $count - 4);
+                $expandButton = __('Show %s more…', $count - 4);
                 echo sprintf(
                     '<li class="no-side-padding correlation-expand-button useCursorPointer linkButton %s">%s</li> ',
                     $linkColour,
@@ -23,8 +23,8 @@
                 );
             }
             $relatedData = array(
-                'Orgc' => !empty($orgTable[$relatedAttribute['org_id']]) ? $orgTable[$relatedAttribute['org_id']] : 'N/A',
+                'Orgc' => $orgTable[$relatedAttribute['org_id']] ?? 'N/A',
-                'Date' => isset($relatedAttribute['date']) ? $relatedAttribute['date'] : 'N/A',
+                'Date' => $relatedAttribute['date'] ?? 'N/A',
                 'Event' => $relatedAttribute['info'],
                 'Correlating Value' => $relatedAttribute['value']
             );
@@ -59,13 +59,13 @@
     }
     if (!empty($object['correlation_exclusion'])) {
         echo sprintf(
-            '<span class="bold red" title="%s">%s</span> ',
+            '<span class="red" title="%s">%s</span> ',
             __('The attribute value matches a correlation exclusion rule defined by a site-administrator for this instance. Click the magnifying glass to search for all occurrences of the value.'),
             __('Excluded.')
         );
     } else if (!empty($object['over_correlation'])) {
         echo sprintf(
-            '<span class="bold red" title="%s">%s</span> ',
+            '<span class="red" title="%s">%s</span> ',
             __('The instance threshold for the maximum number of correlations for the given attribute value has been exceeded. Click the magnifying glass to search for all occurrences of the value.'),
             __('Too many correlations.')
         );

app/View/Elements/Events/eventIndexTable.ctp

@@ -7,9 +7,7 @@
         <?php else: ?>
             <th style="padding-left:0;padding-right:0;">&nbsp;</th>
         <?php endif;?>
-        <th class="filter">
+        <th class="filter" title="<?= __('Published') ?>"><?= $this->Paginator->sort('published', '<i class="fa fa-upload"></i>', ['escape' => false]) ?></th>
-            <?php echo $this->Paginator->sort('published');?>
-        </th>
         <?php
             if (Configure::read('MISP.showorgalternate') && Configure::read('MISP.showorg')):
         ?>
@@ -37,7 +35,7 @@
         <?php if (in_array('creator_user', $columns, true)): ?><th><?= $this->Paginator->sort('user_id', __('Creator user')) ?></th><?php endif; ?>
         <th class="filter"><?= $this->Paginator->sort('date', null, array('direction' => 'desc'));?></th>
         <?php if (in_array('timestamp', $columns, true)): ?><th title="<?= __('Last modified at') ?>"><?= $this->Paginator->sort('timestamp', __('Last modified at')) ?></th><?php endif; ?>
-        <?php if (in_array('publish_timestamp', $columns, true)): ?><th title="<?= __('Last modified at') ?>"><?= $this->Paginator->sort('publish_timestamp', __('Published at')) ?></th><?php endif; ?>
+        <?php if (in_array('publish_timestamp', $columns, true)): ?><th title="<?= __('Published at') ?>"><?= $this->Paginator->sort('publish_timestamp', __('Published at')) ?></th><?php endif; ?>
         <th class="filter"><?= $this->Paginator->sort('info');?></th>
         <th title="<?= $eventDescriptions['distribution']['desc'];?>"><?= $this->Paginator->sort('distribution');?></th>
         <th class="actions"><?php echo __('Actions');?></th>
@@ -45,13 +43,13 @@
     <?php foreach ($events as $event): $eventId = (int)$event['Event']['id']; ?>
     <tr id="event_<?= $eventId ?>">
         <?php if ($isSiteAdmin || ($event['Event']['orgc_id'] == $me['org_id'])):?>
-        <td style="width:10px;">
+        <td style="width:10px">
             <input class="select" type="checkbox" data-id="<?= $eventId ?>" data-uuid="<?= h($event['Event']['uuid']) ?>">
         </td>
         <?php else: ?>
         <td style="padding-left:0;padding-right:0;"></td>
         <?php endif; ?>
-        <td class="short dblclickElement">
+        <td class="dblclickElement" style="width:30px">
             <a href="<?= "$baseurl/events/view/$eventId" ?>" title="<?= __('View') ?>" aria-label="<?= __('View') ?>">
                 <i class="fa <?= $event['Event']['published'] ? 'fa-check green' : 'fa-times grey' ?>"></i>
             </a>
@@ -105,46 +103,46 @@
                 'missingTaxonomies' => false,
                 'columnised' => true,
                 'static_tags_only' => 1,
-                    'tag_display_style' => Configure::check('MISP.full_tags_on_event_index') ? Configure::read('MISP.full_tags_on_event_index') : 1
+                'tag_display_style' => Configure::check('MISP.full_tags_on_event_index') ? Configure::read('MISP.full_tags_on_event_index') : 1,
             ]);
             ?>
         </td>
         <?php endif; ?>
         <?php if (in_array('attribute_count', $columns, true)): ?>
-        <td class="dblclickElement" style="width:30px;">
+        <td class="dblclickElement" style="width:30px">
             <?= $event['Event']['attribute_count']; ?>
         </td>
         <?php endif; ?>
         <?php if (in_array('correlations', $columns, true)): ?>
-        <td class="bold" style="width:30px;">
+        <td class="bold" style="width:30px">
             <?php if (!empty($event['Event']['correlation_count'])): ?>
-                <a href="<?php echo $baseurl."/events/view/" . $eventId . '/correlation:1';?>" title="<?= __n('%s correlation', '%s correlations', $event['Event']['correlation_count'], $event['Event']['correlation_count']), '. ' . __('Show filtered event with correlation only.');?>">
+                <a href="<?= "$baseurl/events/view/$eventId/correlation:1" ?>" title="<?= __n('%s correlation', '%s correlations', $event['Event']['correlation_count'], $event['Event']['correlation_count']), '. ' . __('Show filtered event with correlation only.');?>">
-                    <?php echo h($event['Event']['correlation_count']); ?>
+                    <?= intval($event['Event']['correlation_count']); ?>
                 </a>
             <?php endif; ?>
         </td>
         <?php endif; ?>
         <?php if (in_array('report_count', $columns, true)): ?>
-        <td class="bold" style="width:30px;">
+        <td class="bold" style="width:30px">
             <?= $event['Event']['report_count']; ?>
         </td>
         <?php endif; ?>
         <?php if (in_array('sightings', $columns, true)): ?>
-        <td class="bold" style="width:30px;">
+        <td class="bold" style="width:30px">
             <?php if (!empty($event['Event']['sightings_count'])): ?>
-                <a href="<?php echo $baseurl."/events/view/" . $eventId . '/sighting:1';?>" title="<?php echo (!empty($event['Event']['sightings_count']) ? h($event['Event']['sightings_count']) : '0') . ' sighting(s). Show filtered event with sighting(s) only.';?>">
+                <a href="<?= "$baseurl/events/view/$eventId/sighting:1" ?>" title="<?= __n("1 sighting. Show filtered event with sighting only.", "%s sightings. Show filtered event with sightings only.", $event['Event']['sightings_count'], intval($event['Event']['sightings_count'])) ?>">
                     <?= intval($event['Event']['sightings_count']) ?>
                 </a>
             <?php endif; ?>
         </td>
         <?php endif; ?>
         <?php if (in_array('proposals', $columns, true)): ?>
-        <td class="bold dblclickElement" style="width:30px;" title="<?= __n('%s proposal', '%s proposals', $event['Event']['proposals_count'], $event['Event']['proposals_count']) ?>">
+        <td class="bold dblclickElement" style="width:30px" title="<?= __n('%s proposal', '%s proposals', $event['Event']['proposals_count'], $event['Event']['proposals_count']) ?>">
             <?= !empty($event['Event']['proposals_count']) ? intval($event['Event']['proposals_count']) : ''; ?>
         </td>
         <?php endif;?>
         <?php if (in_array('discussion', $columns, true)): ?>
-        <td class="bold dblclickElement" style="width:30px;">
+        <td class="bold dblclickElement" style="width:30px">
             <?php
                 if (!empty($event['Event']['post_count'])) {
                     $post_count = h($event['Event']['post_count']);
@@ -179,7 +177,7 @@
         <td class="dblclickElement">
             <?= nl2br(h($event['Event']['info']), false) ?>
         </td>
-        <td class="short dblclickElement<?php if ($event['Event']['distribution'] == 0) echo ' privateRedText';?>" title="<?php echo $event['Event']['distribution'] != 3 ? $distributionLevels[$event['Event']['distribution']] : __('All');?>">
+        <td class="short dblclickElement<?php if ($event['Event']['distribution'] == 0) echo ' privateRedText';?>" title="<?= $event['Event']['distribution'] != 3 ? $distributionLevels[$event['Event']['distribution']] : __('All');?>">
             <?php if ($event['Event']['distribution'] == 4):?>
                 <a href="<?php echo $baseurl;?>/sharingGroups/view/<?= intval($event['SharingGroup']['id']); ?>"><?= h($event['SharingGroup']['name']) ?></a>
             <?php else:
@@ -191,7 +189,7 @@
                 '<it type="button" title="%s" class="%s" aria-hidden="true" style="font-size: x-small;" data-event-distribution="%s" data-event-distribution-name="%s" data-scope-id="%s"></it>',
                 __('Toggle advanced sharing network viewer'),
                 'fa fa-share-alt useCursorPointer distributionNetworkToggle',
-                h($event['Event']['distribution']),
+                intval($event['Event']['distribution']),
                 $event['Event']['distribution'] == 4 ? h($event['SharingGroup']['name']) : h($shortDist[$event['Event']['distribution']]),
                 $eventId
             )

app/View/Elements/Events/trending_summary.ctp

@@ -5,13 +5,14 @@ $allTags = $trendAnalysis['all_tags'];
 $allTimestamps = $trendAnalysis['all_timestamps'];
 $currentPeriod = $allTimestamps[0];
 $previousPeriod = $allTimestamps[1];
-$previousPeriod2 = $allTimestamps[2];
+$periods = $allTimestamps;
-$periods = [$previousPeriod2, $previousPeriod, $currentPeriod];
+$reversedPeriods = array_reverse($periods);
+$periodCount = count($periods);
 
 $allUniqueTagsPerPeriod = array_map(function ($tags) {
     return array_keys($tags);
 }, $clusteredTags);
-$allUniqueTags = array_unique(array_merge($allUniqueTagsPerPeriod[$currentPeriod], $allUniqueTagsPerPeriod[$previousPeriod], $allUniqueTagsPerPeriod[$previousPeriod2]));
+$allUniqueTags = array_values(array_unique(Hash::extract($allUniqueTagsPerPeriod, '{n}.{n}')));
 App::uses('ColourPaletteTool', 'Tools');
 $paletteTool = new ColourPaletteTool();
 $COLOR_PALETTE = $paletteTool->createColourPalette(max(count($allUniqueTags), 1));
@@ -29,34 +30,34 @@ $trendColorMapping = [
     '?' => '#999999',
 ];
 $now = new DateTime();
-$currentPeriodDate = DateTime::createFromFormat('U', $currentPeriod);
-$previousPeriodDate = DateTime::createFromFormat('U', $previousPeriod);
-$previousPeriod2Date = DateTime::createFromFormat('U', $previousPeriod2);
 
 $colorForTags = [];
 $chartData = [];
 $maxValue = 0;
 foreach ($allUniqueTags as $i => $tag) {
-    if (
+    $sumForTags = array_reduce($clusteredTags, function ($carry, $clusteredTagsPerPeriod) use ($tag) {
-        !empty($clusteredTags[$previousPeriod2][$tag]['occurence']) ||
+        return $carry + ($clusteredTagsPerPeriod[$tag]['occurrence'] ?? 0);
-        !empty($clusteredTags[$previousPeriod][$tag]['occurence']) ||
+    }, 0);
-        !empty($clusteredTags[$currentPeriod][$tag]['occurence'])
+    if ($sumForTags > 0) {
-    ) {
         $colorForTags[$tag] = $COLOR_PALETTE[$i];
-        $chartData[$tag] = [
+        $chartData[$tag] = array_values(array_map(function ($clusteredTagsPerPeriod) use ($tag) {
-            $clusteredTags[$previousPeriod2][$tag]['occurence'] ?? 0,
+            return $clusteredTagsPerPeriod[$tag]['occurrence'] ?? 0;
-            $clusteredTags[$previousPeriod][$tag]['occurence'] ?? 0,
+        }, $clusteredTags));
-            $clusteredTags[$currentPeriod][$tag]['occurence'] ?? 0,
+        $chartData[$tag] = array_reverse($chartData[$tag]);
-        ];
         $maxValue = max($maxValue, max($chartData[$tag]));
     }
+    $colorForTags[$tag] = $COLOR_PALETTE[$i];
 }
 $canvasWidth = 600;
 $canvasHeight = 150;
 foreach (array_keys($chartData) as $tag) {
+    $lastIndex = count($chartData[$tag]) - 1;
+    $canvasSubWidth = $lastIndex;
     $chartData[$tag][0] = [0, $canvasHeight - ($chartData[$tag][0] / $maxValue) * $canvasHeight];
-    $chartData[$tag][1] = [$canvasWidth / 2, $canvasHeight - ($chartData[$tag][1] / $maxValue) * $canvasHeight];
+    for ($i = 1; $i < $lastIndex; $i++) {
-    $chartData[$tag][2] = [$canvasWidth, $canvasHeight - ($chartData[$tag][2] / $maxValue) * $canvasHeight];
+        $chartData[$tag][$i] = [$canvasWidth * ($i / $canvasSubWidth), $canvasHeight - ($chartData[$tag][$i] / $maxValue) * $canvasHeight];
+    }
+    $chartData[$tag][$lastIndex] = [$canvasWidth, $canvasHeight - ($chartData[$tag][$lastIndex] / $maxValue) * $canvasHeight];
 }
 
 if (!function_exists('reduceTag')) {
@@ -93,6 +94,17 @@ if (!function_exists('computeLinePositions')) {
     }
 }
 
+if (!function_exists('getColorFromYlOrBr')) {
+    function getColorFromYlOrBr(float $min, float $max, float $value): string
+    {
+        $YlOrBrPalette = ["#fff7bc", "#fee391", "#fec44f", "#fe9929", "#ec7014", "#cc4c02", "#993404", "#662506"];
+        $valuePercent = $value / ($max - $min);
+        $paletteRatio = $valuePercent * count($YlOrBrPalette);
+        $paletteIndex = max(round($paletteRatio - 1), 0);
+        return $YlOrBrPalette[$paletteIndex];
+    }
+}
+
 ?>
 
 <div style="display: flex; column-gap: 20px; justify-content: space-around; margin-bottom: 40px;">
@@ -101,19 +113,19 @@ if (!function_exists('computeLinePositions')) {
             <tbody>
                 <tr>
                     <td><?= __('Period duration') ?></td>
-                    <td><?= __('%s days', $currentPeriodDate->diff($now)->format('%a')); ?></td>
+                    <td><?= __('%s days', DateTime::createFromFormat('U', $currentPeriod)->diff($now)->format('%a')); ?></td>
+                </tr>
+                <tr>
+                    <td><?= __('Period number') ?></td>
+                    <td><?= count($periods) - 1 ?></td>
                 </tr>
                 <tr>
                     <td><?= __('Starting period') ?></td>
-                    <td><?= sprintf('%s', $currentPeriodDate->format('M d, o. (\W\e\e\k W)')); ?></td>
+                    <td><?= sprintf('%s', DateTime::createFromFormat('U', $currentPeriod)->format('M d, o. (\W\e\e\k W)')); ?></td>
                 </tr>
                 <tr>
-                    <td><?= __('Previous period') ?></td>
+                    <td><?= __('Last period') ?></td>
-                    <td><?= sprintf('%s', $previousPeriodDate->format('M d, o. (\W\e\e\k W)')); ?></td>
+                    <td><?= sprintf('%s', DateTime::createFromFormat('U', $periods[$periodCount - 1])->format('M d, o. (\W\e\e\k W)')); ?></td>
-                </tr>
-                <tr>
-                    <td><?= __('Period-2') ?></td>
-                    <td><?= sprintf('%s', $previousPeriod2Date->format('M d, o. (\W\e\e\k W)')); ?></td>
                 </tr>
             </tbody>
         </table>
@@ -125,12 +137,12 @@ if (!function_exists('computeLinePositions')) {
                     <div>
                         <span class="y-axis-label" style="<?= sprintf('left: %spx; top: %spx; transform: translate(-100%%, %s%%)', 0, 0, -25) ?>"><?= h($maxValue) ?></span>
                         <span class="y-axis-label" style="<?= sprintf('left: %spx; top: %spx; transform: translate(-100%%, %s%%)', 0, ($canvasHeight - 20) / 2, 0) ?>"><?= h(round($maxValue / 2, 2)) ?></span>
-                        <span class="y-axis-label" style="<?= sprintf('left: %spx; top: %spx; transform: translate(-100%%, %s%%)', 0, ($canvasHeight - 20), 25) ?>"><?= 0 ?></span>
+                        <span class="y-axis-label" style="<?= sprintf('left: %spx; top: %spx; transform: translate(-100%%, %s%%)', 0, ($canvasHeight - 20), 25) ?>">0</span>
                     </div>
                 </div>
                 <div class="canvas">
                     <?php foreach ($chartData as $tag => $coords) : ?>
-                        <?php for ($i = 0; $i < 3; $i++) : ?>
+                        <?php for ($i = 0; $i < count($periods); $i++) : ?>
                             <?php
                             $coord = $coords[$i];
                             $previousCoord = isset($coords[$i - 1]) ? $coords[$i - 1] : false;
@@ -154,9 +166,9 @@ if (!function_exists('computeLinePositions')) {
                     <?php endforeach ?>
                 </div>
                 <div class="x-axis-container">
-                    <span class="x-axis-label" style="<?= sprintf('left: %spx; top: %spx;', 0, 0) ?>"><?= __('Period-2') ?></span>
+                    <?php foreach ($reversedPeriods as $i => $period) : ?>
-                    <span class="x-axis-label" style="<?= sprintf('left: %spx; top: %spx;', $canvasWidth / 2, 0) ?>"><?= __('Previous period') ?></span>
+                        <span class="x-axis-label" style="<?= sprintf('left: %spx; top: %spx;', $i * $canvasWidth / $canvasSubWidth, 0) ?>"><?= DateTime::createFromFormat('U', $period)->format('M. d, o') ?></span>
-                    <span class="x-axis-label" style="<?= sprintf('left: %spx; top: %spx;', $canvasWidth, 0) ?>"><?= __('Starting period') ?></span>
+                    <?php endforeach; ?>
                 </div>
             </div>
         </div>
@@ -170,54 +182,24 @@ if (!function_exists('computeLinePositions')) {
         <thead>
             <tr>
                 <th></th>
+                <?php foreach ($reversedPeriods as $i => $period) : ?>
                     <th>
                         <span>
-                        <div><?= __('Period-2') ?></div>
+                            <div><?= DateTime::createFromFormat('U', $period)->format('M. d, o') ?></div>
-                        <div style="font-weight: normal;"><?= __('%s events', h($clusteredEvents[$previousPeriod2])) ?></div>
+                            <div style="font-weight: normal;"><?= __('%s events', h($clusteredEvents[$period])) ?></div>
                         </span>
                         <table class="trending-table-data">
                             <thead style="font-size: small;">
                                 <tr>
-                                <td>#</td>
+                                    <td title="<?= __('Occurrence per events') ?>">#</td>
-                                <td>⥮</td>
+                                    <td title="<?= __('Raw change') ?>">⥮</td>
-                                <td>%</td>
+                                    <td title="<?= __('Percent change') ?>">%</td>
-                                <td></td>
-                            </tr>
-                        </thead>
-                    </table>
-                </th>
-                <th>
-                    <span>
-                        <div><?= __('Previous period') ?></div>
-                        <div style="font-weight: normal;"><?= __('%s events', h($clusteredEvents[$previousPeriod])) ?></div>
-                    </span>
-                    <table class="trending-table-data">
-                        <thead style="font-size: small;">
-                            <tr>
-                                <td>#</td>
-                                <td>⥮</td>
-                                <td>%</td>
-                                <td></td>
-                            </tr>
-                        </thead>
-                    </table>
-                </th>
-                <th>
-                    <span>
-                        <div><?= __('Starting period') ?></div>
-                        <div style="font-weight: normal;"><?= __('%s events', h($clusteredEvents[$currentPeriod])) ?></div>
-                    </span>
-                    <table class="trending-table-data">
-                        <thead style="font-size: small;">
-                            <tr>
-                                <td>#</td>
-                                <td>⥮</td>
-                                <td>%</td>
                                     <td></td>
                                 </tr>
                             </thead>
                         </table>
                     </th>
+                <?php endforeach; ?>
             </tr>
         </thead>
         <?php foreach ($tagFilterPrefixes as $tagPrefix) : ?>
@@ -238,59 +220,34 @@ if (!function_exists('computeLinePositions')) {
                             <span class="tag-legend" style="background-color: <?= $colorForTags[$tagName] ?>;"></span>
                             <code><?= h(reduceTag($tagName, count(explode(':', $tagPrefix)))) ?></code>
                         </td>
+                        <?php foreach ($reversedPeriods as $i => $period) : ?>
                             <td>
                                 <table class="table-condensed no-border trending-table-data">
                                     <tbody>
                                         <tr>
-                                        <td><?= h($clusteredTags[$previousPeriod2][$tagName]['occurence'] ?? '-') ?></td>
+                                            <td title="<?= __('Occurrence per events') ?>"><?= h($clusteredTags[$period][$tagName]['occurrence'] ?? '-') ?></td>
-                                        <td><?= h($clusteredTags[$previousPeriod2][$tagName]['raw_change'] ?? '-') ?></td>
+                                            <td title="<?= __('Raw change') ?>"><?= h($clusteredTags[$period][$tagName]['raw_change'] ?? '-') ?></td>
-                                        <td><?= h($clusteredTags[$previousPeriod2][$tagName]['percent_change'] ?? '-') ?>%</td>
+                                            <td title="<?= __('Percent change') ?>"><?= h($clusteredTags[$period][$tagName]['percent_change'] ?? '-') ?>%</td>
-                                        <td style="font-size: large; color: <?= $trendColorMapping[$clusteredTags[$previousPeriod2][$tagName]['change_sign'] ?? '?'] ?>"><?= $trendIconMapping[$clusteredTags[$previousPeriod2][$tagName]['change_sign'] ?? '?'] ?></td>
+                                            <?php if ($i > 0) : ?>
-                                    </tr>
+                                                <td title="<?= __('Evolution') ?>" style="font-size: large; color: <?= $trendColorMapping[$clusteredTags[$period][$tagName]['change_sign'] ?? '?'] ?>"><?= $trendIconMapping[$clusteredTags[$period][$tagName]['change_sign'] ?? '?'] ?></td>
-                                </tbody>
+                                            <?php endif; ?>
-                            </table>
-                        </td>
-                        <td>
-                            <table class="table-condensed no-border trending-table-data">
-                                <tbody>
-                                    <tr>
-                                        <td><?= h($clusteredTags[$previousPeriod][$tagName]['occurence'] ?? '-') ?></td>
-                                        <td><?= h($clusteredTags[$previousPeriod][$tagName]['raw_change'] ?? '-') ?></td>
-                                        <td><?= h($clusteredTags[$previousPeriod][$tagName]['percent_change'] ?? '-') ?>%</td>
-                                        <td style="font-size: large; color: <?= $trendColorMapping[$clusteredTags[$previousPeriod][$tagName]['change_sign'] ?? '?'] ?>"><?= $trendIconMapping[$clusteredTags[$previousPeriod][$tagName]['change_sign'] ?? '?'] ?></td>
-                                    </tr>
-                                </tbody>
-                            </table>
-                        </td>
-                        <td>
-                            <table class="table-condensed no-border trending-table-data">
-                                <tbody>
-                                    <tr>
-                                        <td><?= h($clusteredTags[$currentPeriod][$tagName]['occurence'] ?? '-') ?></td>
-                                        <td><?= h($clusteredTags[$currentPeriod][$tagName]['raw_change'] ?? '-') ?></td>
-                                        <td><?= h($clusteredTags[$currentPeriod][$tagName]['percent_change'] ?? '-') ?>%</td>
-                                        <td style="font-size: large; color: <?= $trendColorMapping[$clusteredTags[$currentPeriod][$tagName]['change_sign'] ?? '?'] ?>"><?= $trendIconMapping[$clusteredTags[$currentPeriod][$tagName]['change_sign'] ?? '?'] ?></td>
                                         </tr>
                                     </tbody>
                                 </table>
                             </td>
+                        <?php endforeach; ?>
                     </tr>
                     <td style="padding: 0;"></td>
-                    <td colspan="3" style="padding: 0;">
+                    <td colspan="<?= count($periods) ?>" style="padding: 0;">
                         <?php
-                            $low = '#fee8c8';
-                            $medium = '#f09c8f';
-                            $high = '#bc2f1a';
-                            $periodColorRatio = $clusteredTags[$currentPeriod][$tagName]['occurence'] / $maxValue;
                         $colorGradient = [];
-                            foreach ($periods as $i => $period) {
+                        foreach ($reversedPeriods as $i => $period) {
-                                $ratio = ($clusteredTags[$period][$tagName]['occurence'] ?? 0) / $maxValue;
+                            $color = getColorFromYlOrBr(0, $maxValue, $clusteredTags[$period][$tagName]['occurrence'] ?? 0);
-                                $color = $ratio <= 0.33 ? $low : ($ratio >= 0.66 ? $high : $medium);
                             $length = 100 * $i / (count($periods) - 1);
+                            $length = $i > 0 ? $length - 5 : $length; // Small offset to better align colors on the table period header
                             $colorGradient[] = sprintf('%s %s%%', $color, $length);
                         }
                         ?>
-
                         <div class="heatbar" style="background: <?= sprintf('linear-gradient(90deg, %s);', implode(', ', $colorGradient)) ?>;"></div>
                     </td>
                 <?php endforeach; ?>
@@ -303,6 +260,7 @@ if (!function_exists('computeLinePositions')) {
     table.trending-table table.trending-table-data {
         width: 150px;
     }
+
     table.trending-table th:not(:first-child) {
         width: 150px;
     }

app/View/Emails/notification_common.ctp

@@ -9,6 +9,7 @@
  *  - `detailed-summary-type`
  *  - `detailed-summary-tags`
  *  - `detailed-summary-events`
+ *  - `detailed-summary-correlations`
  *  - `aggregated-context`
  * 
  * Additional variables:
@@ -21,6 +22,8 @@ if (empty($this->__vars)) {
 $default_vars = [
     'event_table_include_basescore' => true,
     'event_table_max_event_count' => 30,
+    'correlation_table_advanced_ui' => 10,
+    'correlation_table_max_count' => 50,
     'additional_taxonomy_event_list' => [
         'PAP' => 'PAP:'
     ],
@@ -28,7 +31,7 @@ $default_vars = [
 $vars = array_merge($default_vars, $this->__vars);
 
 $now = new DateTime();
-$start_date = new DateTime('7 days ago');
+$start_date = new DateTime($period_days . ' days ago');
 $event_number = count($events);
 $attribute_number = 0;
 $object_number = 0;
@@ -49,6 +52,8 @@ $mitre_galaxy_tag_prefix = 'misp-galaxy:mitre-attack-pattern="';
 $reportLink = sprintf('%s/users/viewPeriodicSummary/%s', $baseurl, $period);
 $eventLink = sprintf('%s/events/index/searchpublished:1/searchPublishTimestamp:%s/searchPublishTimestamp:%s', $baseurl, h($start_date->format('Y-m-d H:i:s')), h($now->format('Y-m-d H:i:s')));
 
+$processed_correlations = [];
+$new_correlations = [];
 foreach ($events as $event) {
     $unique_tag_per_event = [];
     $attribute_number += count($event['Attribute']);
@@ -56,11 +61,10 @@ foreach ($events as $event) {
     $event_report_number += count($event['EventReport']);
     $proposal_number += count($event['ShadowAttribute']);
 
-
     foreach ($event['EventTag'] as $event_tag) {
         $tag = $event_tag['Tag'];
 
-        if (!empty($unique_tag_per_event[$tag['name']])) {
+        if (isset($unique_tag_per_event[$tag['name']])) {
             continue; // Only one instance of tag per event
         }
         $unique_tag_per_event[$tag['name']] = true;
@@ -69,24 +73,29 @@ foreach ($events as $event) {
             $all_tag_amount[$tag['name']] = 0;
             $tag_color_mapping[$tag['name']] = $tag['colour'];
         }
-        $all_tag_amount[$tag['name']] += 1;
+        $all_tag_amount[$tag['name']]++;
 
-        if (!empty($tag['is_galaxy']) && substr($tag['name'], 0, strlen($mitre_galaxy_tag_prefix)) === $mitre_galaxy_tag_prefix) {
+        if ($tag['is_galaxy'] && substr($tag['name'], 0, strlen($mitre_galaxy_tag_prefix)) === $mitre_galaxy_tag_prefix) {
             $technique = substr($tag['name'], strlen($mitre_galaxy_tag_prefix), strlen($tag['name']) - strlen($mitre_galaxy_tag_prefix) - 1);
             $mitre_attack_techniques[$technique] = $event_tag;
         }
     }
 
+    $attribute_light_by_id = [];
     foreach ($event['Attribute'] as $attribute) {
+        $attribute_light_by_id[$attribute['id']] = [
+            'timestamp' => $attribute['timestamp'],
+            'type' => $attribute['type'],
+        ];
         if (empty($attribute_types[$attribute['type']])) {
             $attribute_types[$attribute['type']] = 0;
         }
-        $attribute_types[$attribute['type']] += 1;
+        $attribute_types[$attribute['type']]++;
 
         foreach ($attribute['AttributeTag'] as $attribute_tag) {
             $tag = $attribute_tag['Tag'];
 
-            if (!empty($unique_tag_per_event[$tag['name']])) {
+            if (isset($unique_tag_per_event[$tag['name']])) {
                 continue; // Only one instance of tag per event
             }
             $unique_tag_per_event[$tag['name']] = true;
@@ -95,9 +104,9 @@ foreach ($events as $event) {
                 $all_tag_amount[$tag['name']] = 0;
                 $tag_color_mapping[$tag['name']] = $tag['colour'];
             }
-            $all_tag_amount[$tag['name']] += 1;
+            $all_tag_amount[$tag['name']]++;
 
-            if (!empty($tag['is_galaxy']) && substr($tag['name'], 0, strlen($mitre_galaxy_tag_prefix)) === $mitre_galaxy_tag_prefix) {
+            if ($tag['is_galaxy'] && substr($tag['name'], 0, strlen($mitre_galaxy_tag_prefix)) === $mitre_galaxy_tag_prefix) {
                 $technique = substr($tag['name'], strlen($mitre_galaxy_tag_prefix), strlen($tag['name']) - strlen($mitre_galaxy_tag_prefix) - 1);
                 $mitre_attack_techniques[$technique] = $attribute_tag;
             }
@@ -108,19 +117,23 @@ foreach ($events as $event) {
         if (empty($object_types[$object['name']])) {
             $object_types[$object['name']] = 0;
         }
-        $object_types[$object['name']] += 1;
+        $object_types[$object['name']]++;
 
         $attribute_number += count($object['Attribute']);
         foreach ($object['Attribute'] as $attribute) {
+            $attribute_light_by_id[$attribute['id']] = [
+                'timestamp' => $attribute['timestamp'],
+                'type' => $attribute['type'],
+            ];
             if (empty($attribute_types[$attribute['type']])) {
                 $attribute_types[$attribute['type']] = 0;
             }
-            $attribute_types[$attribute['type']] += 1;
+            $attribute_types[$attribute['type']]++;
 
             foreach ($attribute['AttributeTag'] as $attribute_tag) {
                 $tag = $attribute_tag['Tag'];
 
-                if (!empty($unique_tag_per_event[$tag['name']])) {
+                if (isset($unique_tag_per_event[$tag['name']])) {
                     continue; // Only one instance of tag per event
                 }
                 $unique_tag_per_event[$tag['name']] = true;
@@ -129,9 +142,9 @@ foreach ($events as $event) {
                     $all_tag_amount[$tag['name']] = 0;
                     $tag_color_mapping[$tag['name']] = $tag['colour'];
                 }
-                $all_tag_amount[$tag['name']] += 1;
+                $all_tag_amount[$tag['name']]++;
 
-                if (!empty($tag['is_galaxy']) && substr($tag['name'], 0, strlen($mitre_galaxy_tag_prefix)) === $mitre_galaxy_tag_prefix) {
+                if ($tag['is_galaxy'] && substr($tag['name'], 0, strlen($mitre_galaxy_tag_prefix)) === $mitre_galaxy_tag_prefix) {
                     $technique = substr($tag['name'], strlen($mitre_galaxy_tag_prefix), strlen($tag['name']) - strlen($mitre_galaxy_tag_prefix) - 1);
                     $mitre_attack_techniques[$technique] = $attribute_tag;
                 }
@@ -147,6 +160,33 @@ foreach ($events as $event) {
             'event_info' => $event['Event']['info'],
         ];
     }
+
+    if (!empty($event['RelatedEvent'])) {
+        $related_event_by_id = [];
+        foreach ($event['RelatedEvent'] as $related_event) {
+            $related_event_by_id[$related_event['Event']['id']] = $related_event['Event'];
+        }
+
+        foreach ($event['RelatedAttribute'] as $attribute_id => $related_attributes) {
+            $has_attribute_been_modified_since_last_period = intval($attribute_light_by_id[$attribute_id]['timestamp']) >= intval($start_date->format('U'));
+            foreach ($related_attributes as $related_attribute) {
+                $correlation_id = sprintf('%s-%s', $related_attribute['attribute_id'], $attribute_id);
+                $reversed_correlation_id = sprintf('%s-%s', $attribute_id, $related_attribute['attribute_id']);
+                $has_correlation_been_processed = isset($processed_correlations[$correlation_id]); // We already added the correlation the other way around
+                if ($has_attribute_been_modified_since_last_period && !$has_correlation_been_processed) {
+                    $source_event = $event['Event'];
+                    $source_event['Orgc'] = $event['Orgc'];
+                    $new_correlations[] = [
+                        'source_event' => $source_event,
+                        'target_event' => $related_event_by_id[$related_attribute['id']],
+                        'attribute_value' => $related_attribute['value'],
+                        'attribute_type' => $attribute_light_by_id[$attribute_id]['type'],
+                    ];
+                    $processed_correlations[$reversed_correlation_id] = true;
+                }
+            }
+        }
+    }
 }
 
 if (!function_exists('findAndBuildTag')) {
@@ -161,12 +201,14 @@ if (!function_exists('findAndBuildTag')) {
     }
 }
 
-$unique_tag_number = count(array_keys($all_tag_amount));
+$unique_tag_number = count($all_tag_amount);
 
 arsort($attribute_types);
 arsort($object_types);
 arsort($all_tag_amount);
-arsort($mitre_attack_techniques);
+uasort($mitre_attack_techniques, function($tag1, $tag2) use ($all_tag_amount) {
+    return ($all_tag_amount[$tag1['Tag']['name']] < $all_tag_amount[$tag2['Tag']['name']]) ? 1 : -1;
+});
 
 array_splice($attribute_types, 10);
 array_splice($object_types, 10);
@@ -190,18 +232,16 @@ array_splice($mitre_attack_techniques, 10);
                 <tbody>
                     <tr>
                         <td><?= __('Summary period') ?></td>
-                        <td><?= h($period) ?></td>
+                        <td><?= h(ucfirst($period)) ?></td>
                     </tr>
                     <tr>
                         <td><?= __('Summary for dates') ?></td>
                         <td>
-                            <?=
+                            <?= __('<strong>%s</strong> (Week %s) ➞ <strong>%s</strong> (Week %s)',
-                            sprintf('<strong>%s</strong> (Week %s) ➞ <strong>%s</strong> (Week %s)',
                                 $start_date->format('M d, o'),
                                 $start_date->format('W'),
                                 $now->format('M d, o'),
-                                $now->format('W'),
+                                $now->format('W')
-                                $start_date->format('M d, o')
                             )
                             ?>
                         </td>
@@ -211,7 +251,7 @@ array_splice($mitre_attack_techniques, 10);
                         <td><?= date("c"); ?></td>
                     </tr>
                     <tr>
-                        <td><?= __('Events #') ?></td>
+                        <td><?= __('Published Events #') ?></td>
                         <td><?= $event_number ?></td>
                     </tr>
                     <tr>
@@ -234,6 +274,12 @@ array_splice($mitre_attack_techniques, 10);
                         <td><?= __('Unique tags #') ?></td>
                         <td><?= $unique_tag_number ?></td>
                     </tr>
+                    <?php if (!empty($periodicSettings['include_correlations'])): ?>
+                        <tr>
+                            <td><?= __('New correlation #') ?></td>
+                            <td><?= count($new_correlations) ?></td>
+                        </tr>
+                    <?php endif; ?>
                 </tbody>
             </table>
             ⮞ <a href="<?= h($reportLink) ?>"><?= __('View this report in MISP') ?></a>
@@ -253,7 +299,7 @@ array_splice($mitre_attack_techniques, 10);
                 <?= $this->fetch('detailed-summary-mitre-attack'); ?>
             <?php else: ?>
                 <?php if (!empty($mitre_attack_techniques)) : ?>
-                    <h4><?= __('Top 10 Mitre Att&ck techniques') ?></h4>
+                    <h4><?= __('Top 10 MITRE ATT&CK techniques') ?></h4>
                     <ul>
                         <?php foreach ($mitre_attack_techniques as $technique => $tag) : ?>
                             <li>
@@ -310,7 +356,7 @@ array_splice($mitre_attack_techniques, 10);
             <?php else: ?>
                 <h4><?= __('Top 10 Tags') ?></h4>
                 <ul>
-                    <?php foreach ($all_tag_amount as $tag_name => $amount) : ?>
+                    <?php array_splice($all_tag_amount, 10); foreach ($all_tag_amount as $tag_name => $amount) : ?>
                         <li>
                             <span class="tag" style="background-color: #999; color: #fff; border-radius: 9px; padding: 2px 8px;">
                                 <?= $amount ?>
@@ -405,9 +451,78 @@ array_splice($mitre_attack_techniques, 10);
                     <a href="<?= h($eventLink) ?>"><?= __('View all events in MISP') ?></a>
                 <?php endif; ?>
             <?php endif; ?>
+
+            <?php if ($this->fetch('detailed-summary-correlations')) : ?>
+            <?php else: ?>
+                <?php if (!empty($new_correlations)) : ?>
+                    <h4><?= __('New correlations') ?><small style="color: #999999;"><?= sprintf(' (%s)', count($new_correlations)) ?></small></h4>
+                    <div>
+                        <?php if (count($new_correlations) < $vars['correlation_table_advanced_ui']) : ?>
+                            <?php foreach ($new_correlations as $correlation): ?>
+                                <div style="display: flex; flex-wrap: nowrap; align-items: center; margin-top: 0.5em;">
+                                    <span>
+                                        <span class="correlating-event-container">
+                                            <span>
+                                                <a href="<?= sprintf('%s/events/view/%s', $baseurl, h($correlation['source_event']['id'])) ?>"><?= h($correlation['source_event']['info']) ?></a>
+                                            </span>
+                                            <span class="org-date">
+                                                <span><?= h($correlation['source_event']['date']) ?></span>
+                                                <span><?= h($correlation['source_event']['Orgc']['name']) ?></span>
+                                            </span>
+                                        </span>
+                                    </span>
+                                    <span class="correlating-attribute-container">
+                                        <span class="correlating-attribute">
+                                            <?= h($correlation['attribute_type']); ?> :: <b><?= h($correlation['attribute_value']) ?></b>
+                                        </span>
+                                    </span>
+                                    <span>
+                                        <span class="correlating-event-container">
+                                            <span>
+                                                <a href="<?= sprintf('%s/events/view/%s', $baseurl, h($correlation['target_event']['id'])) ?>"><?= h($correlation['target_event']['info']) ?></a>
+                                            </span>
+                                            <span class="org-date">
+                                                <span><?= h($correlation['target_event']['date']) ?></span>
+                                                <span><?= h($correlation['target_event']['Orgc']['name']) ?></span>
+                                            </span>
+                                        </span>
+                                    </span>
                                 </div>
+                            <?php endforeach; ?>
+                        <?php else: ?>
+                            <table class="table table-xcondensed">
+                                <thead>
+                                    <tr>
+                                        <th><?= __('First event info') ?></th>
+                                        <th><?= __('Value') ?></th>
+                                        <th><?= __('Second event info') ?></th>
+                                    </tr>
+                                </thead>
+                                <tbody>
+                                    <?php foreach (array_slice($new_correlations, 0, $vars['correlation_table_max_count']) as $correlation): ?>
+                                        <tr>
+                                            <td><a href="<?= sprintf('%s/events/view/%s', $baseurl, h($correlation['source_event']['id'])) ?>"><?= h($correlation['source_event']['info']) ?></a></td>
+                                            <td><b><?= h($correlation['attribute_value']) ?></b></td>
+                                            <td><a href="<?= sprintf('%s/events/view/%s', $baseurl, h($correlation['target_event']['id'])) ?>"><?= h($correlation['target_event']['info']) ?></a></td>
+                                        </tr>
+                                    <?php endforeach; ?>
+                                </tbody>
+                            </table>
+                            <?php
+                            if (count($new_correlations) > $vars['correlation_table_max_count']) {
+                                echo '⮞ ' . __n(
+                                    '%s correlation not displayed.',
+                                    '%s correlations not displayed.',
+                                    count($new_correlations) - $vars['correlation_table_max_count'],
+                                    sprintf('<strong>%s</strong>', count($new_correlations) - $vars['correlation_table_max_count'])
+                                );
+                            }
+                            ?>
+                        <?php endif; ?>
+                    </div>
+                <?php endif; ?>
+            <?php endif; ?>
         </div>
-    <?php endif; // detailed-summary-full 
     ?>
 
     <?php if ($this->fetch('trending-summary')) : ?>
@@ -488,6 +603,41 @@ array_splice($mitre_attack_techniques, 10);
             border-radius: 3px;
         }
 
+        .correlating-attribute {
+            padding: 3px 5px;
+            border: 1px solid #ccc;
+            border-radius: 3px;
+            white-space: nowrap;
+        }
+        .correlating-attribute-container {
+            display: flex;
+            box-sizing: border-box;
+            margin: 0 0;
+            align-items: center;
+            min-width: 400px;
+        }
+        .correlating-attribute-container::before,
+        .correlating-attribute-container::after {
+            display: inline-block;
+            content: ' ';
+            height: 2px;
+            width: 100%;
+            background-color: #ccc;
+        }
+
+        .correlating-event-container {
+            display: flex;
+            flex-direction: column;
+            min-width: 180px;
+            border: 1px solid #ccc;
+            border-radius: 3px;
+            padding: 3px 5px;
+        }
+        .correlating-event-container > .org-date {
+            display: flex;
+            justify-content: space-between;
+        }
+
         .no-overflow {
             display: inline-block;
             white-space: nowrap;

app/View/Events/index.ctp

@@ -44,7 +44,7 @@
             'proposals' => __('Proposals'),
             'discussion' => __('Posts'),
             'report_count' => __('Report count'),
-            'timestamp' => __('Last change at'),
+            'timestamp' => __('Last modified at'),
             'publish_timestamp' => __('Published at')
         ];
 

app/View/Events/module_views/context_view.ctp

@@ -48,29 +48,28 @@
         <h2><?= __('Galaxy Clusters') ?></h2>
         <div>
             <?php
-            $htmlClusters = '';
             foreach ($clusters as $tagname => $entries) {
-                $htmlClusters .= sprintf(
+                echo sprintf(
                     '<div><h4>%s %s</h4></div>',
                     sprintf('<i class="%s"></i>', $this->FontAwesome->getClass($entries[0]['Galaxy']['icon'])),
                     h($entries[0]['Galaxy']['name'])
                 );
                 if (!empty($entries[0]['Galaxy']['description'])) {
-                    $htmlClusters .= sprintf('<div><i>%s</i></div>', h($entries[0]['Galaxy']['description']));
+                    echo sprintf('<div><i>%s</i></div>', h($entries[0]['Galaxy']['description']));
                 }
-                $htmlClusters .= '<ul>';
+                echo '<ul>';
                 foreach ($entries as $cluster) {
-                    $htmlClusters .= sprintf(
+                    $description = $this->Markdown->toText($cluster['GalaxyCluster']['description']);
+                    echo sprintf(
                         '<li><strong><a href="%s" target="_blank">%s</a></strong></li> %s',
                         $baseurl . '/galaxy_clusters/view/' . h($cluster['GalaxyCluster']['id']),
                         h($cluster['GalaxyCluster']['value']),
-                        strlen(h($cluster['GalaxyCluster']['description'])) > 300 ?
+                        strlen($description) > 300 ?
-                            (substr(h($cluster['GalaxyCluster']['description']), 0, 300) . '...') : h($cluster['GalaxyCluster']['description'])
+                            (h(mb_substr($description, 0, 300)) . '...') : h($description)
                     );
                 }
-                $htmlClusters .= '</ul>';
+                echo '</ul>';
             }
-            echo $htmlClusters;
             ?>
         </div>
     <?php endif; ?>

app/View/ObjectReferences/ajax/add.ctp

@@ -13,7 +13,7 @@
                                     'div' => false
                                 ));
                         ?>
-                            <div id="" class="hidden">
+                            <div class="hidden">
                                 <label for="ObjectReferenceRelationshipTypeSelect"><?php echo __('Relationship type');?></label>
                                 <?php
                                     echo $this->Form->input('relationship_type', array(
@@ -44,7 +44,7 @@
                                     'style' => 'width:320px;'
                                 ));
                             ?>
-                            <br />
+                            <br>
 
                             <?php
                                 $items = array();
@@ -115,7 +115,6 @@
                                 );
                                 echo $this->element('generic_picker', array('items' => $items, 'options' => $options));
                             ?>
-
                         </div>
                         <div class="span6">
                             <label for="selectedData"><?php echo __('Target Details');?></label>
@@ -146,7 +145,7 @@
 </div>
 <script type="text/javascript">
     var targetEvent = <?php echo json_encode($event); ?>;
-    $(document).ready(function() {
+    $(function() {
         $('#ObjectReferenceReferencedUuid').on('input', function() {
             objectReferenceInput();
         });
@@ -160,4 +159,3 @@
         $('#ObjectReferenceRelationshipTypeSelect').chosen({ width: "100%" });
     });
 </script>
-<?php echo $this->Js->writeBuffer(); // Write cached scripts

app/View/Objects/group_attributes_into_object.ctp

@@ -59,22 +59,22 @@ echo $this->Form->create('Object', array('url' => $baseurl . '/objects/groupAttr
                 <th><?php echo __('Distribution'); ?></th>
             </tr>
         </thead>
-        <tbody id='attributeMappingBody'>
+        <tbody id="attributeMappingBody">
             <?php foreach ($attributes as $attribute): ?>
                 <tr>
-                    <td id="isAttributeId"><?php echo h($attribute['Attribute']['id']); ?></td>
+                    <td class="attributeId"><?= intval($attribute['Attribute']['id']); ?></td>
                     <td>
                         <span style="display: block;">
-                            <select id="isAttributeMapping" style="margin-bottom: 5px;" onchange="updateObjectRelationDescription(this);">
+                            <select class="attributeMapping" style="margin-bottom: 5px;">
                                 <?php foreach ($object_relations[$attribute['Attribute']['type']] as $object_relation): ?>
                                     <option value="<?php echo h($object_relation['object_relation']); ?>" title="<?php echo h($object_relation['description']); ?>"><?php echo h($object_relation['object_relation']); ?></option>
                                 <?php endforeach; ?>
                             </select>
                             :: <?php echo h($attribute['Attribute']['type']); ?>
                         </span>
-                        <i id="objectRelationDescription" class="apply_css_arrow"><?php echo h($object_relations[$attribute['Attribute']['type']][0]['description']); ?></i>
+                        <i class="objectRelationDescription apply_css_arrow"><?php echo h($object_relations[$attribute['Attribute']['type']][0]['description']); ?></i>
                     </td>
-                    <td style="min-width: 75px;"><?php echo h(date('Y-m-d', $attribute['Attribute']['timestamp'])); ?></td>
+                    <td style="min-width: 75px;"><?= $this->Time->date($attribute['Attribute']['timestamp']); ?></td>
                     <td><?php echo h($attribute['Attribute']['category']); ?></td>
                     <td style="white-space: nowrap;"><?php echo h($attribute['Attribute']['value']); ?></td>
                     <td><?php echo h($distributionLevels[$attribute['Attribute']['distribution']]); ?></td>
@@ -126,7 +126,7 @@ echo $this->Form->create('Object', array('url' => $baseurl . '/objects/groupAttr
         <button class="btn btn-primary" onclick="submitMergeAttributeIntoObjectForm(this);"><?php echo __('Merge above Attributes into an Object'); ?></button>
     </div>
     <span class="red bold" data-original-title="" title="">
-        <?php echo sprintf(__('Selected Attributes will be %s deleted'), '<strong style="font-size: medium">' . ($hard_delete_attribute ? __('hard') : __('soft')) . '</strong>'); ?>
+        <?php echo __('Selected Attributes will be %s deleted', '<strong style="font-size: medium">' . ($hard_delete_attribute ? __('hard') : __('soft')) . '</strong>'); ?>
     </span>
 </div>
 
@@ -135,25 +135,31 @@ $(".Object_distribution_select").change(function() {
     checkAndEnable($(this).parent().find('.Object_sharing_group_id_select'), $(this).val() == 4);
 });
 
+$(".attributeMapping").change(function() {
+    var $select = $(this);
+    var text = $select.find(":selected").attr('title');
+    $select.parent().parent().find('.objectRelationDescription').text(text);
+});
+
 function submitMergeAttributeIntoObjectForm(btn) {
     var $btn = $(btn);
     var $form = $('#ObjectGroupAttributesIntoObjectForm');
     var attribute_mapping = {};
     $('#attributeMappingBody').find('tr').each(function() {
         var $tr = $(this);
-        var attr_id = $tr.find('#isAttributeId').text();
+        var attr_id = $tr.find('.attributeId').text();
-        var attr_mapping = $tr.find('#isAttributeMapping').val();
+        var attr_mapping = $tr.find('.attributeMapping').val();
         attribute_mapping[attr_id] = attr_mapping;
     });
     $('#ObjectSelectedObjectRelationMapping').val(JSON.stringify(attribute_mapping));
     var btn_text_backup = '';
     $.ajax({
         data: $form.serialize(),
-        beforeSend: function (XMLHttpRequest) {
+        beforeSend: function () {
             btn_text_backup = $btn.text();
             $btn.html('<it class="fa fa-spinner fa-spin"></it>');
         },
-        success:function (data, textStatus) {
+        success: function (data) {
             if (data.errors !== undefined) {
                 showMessage('fail', responseArray.errors);
                 $btn.text(btn_text_backup);
@@ -170,10 +176,4 @@ function submitMergeAttributeIntoObjectForm(btn) {
         url: $form.attr('action')
     });
 }
-
-function updateObjectRelationDescription(changed) {
-    var $select = $(changed);
-    var text = $select.find(":selected").attr('title');
-    $select.parent().parent().find('#objectRelationDescription').text(text);
-}
 </script>

app/View/Objects/propose_objects_from_attributes.ctp

@@ -15,7 +15,7 @@
                 <th><?php echo __('Object name'); ?></th>
                 <th><?php echo __('Category'); ?></th>
                 <th><?php echo __('Description'); ?></th>
-                <th title="<?php echo __('Compatiblity or Attribute type missing from the selection'); ?>"><?php echo __('Compatiblity'); ?></th>
+                <th title="<?php echo __('Compatibility or Attribute type missing from the selection'); ?>"><?php echo __('Compatibility'); ?></th>
             </tr>
         </thead>
         <tbody>
@@ -28,7 +28,7 @@
                     <td><?php echo h($potential_template['ObjectTemplate']['meta-category']); ?></td>
                     <?php
                         $v = h($potential_template['ObjectTemplate']['description']);
-                        $v = strlen($v) > 100 ? substr($v, 0, 100) . '...' : $v;
+                        $v = strlen($v) > 100 ? mb_substr($v, 0, 100) . '&hellip;' : $v;
                     ?>
                     <td style="max-width: 500px;" title="<?php echo h($potential_template['ObjectTemplate']['description']); ?>">
                         <?php echo $v; ?>

app/View/UserSettings/set_setting.ctp

@@ -10,7 +10,7 @@
                 array(
                     'div' => 'clear',
                     'class' => 'input input-xxlarge',
-                    'options' => array($users),
+                    'options' => $users,
                     'disabled' => count($users) === 1
                 )
             ),
@@ -40,18 +40,16 @@
 ?>
 <script type="text/javascript">
     var validSettings = <?= json_encode($validSettings); ?>;
+
     $(function() {
         loadUserSettingValue();
         changeUserSettingPlaceholder();
-        $('#UserSettingSetting').on('change', function() {
+        $('#UserSettingSetting, #UserSettingUserId').on('change', function() {
-            loadUserSettingValue();
-            changeUserSettingPlaceholder();
-        });
-        $('#UserSettingUserId').on('change', function() {
             loadUserSettingValue();
             changeUserSettingPlaceholder();
         });
     });
+
     function loadUserSettingValue() {
         var user_id = $('#UserSettingUserId').val();
         var setting = $('#UserSettingSetting').val();
@@ -66,6 +64,13 @@
                     data = JSON.stringify(data, undefined, 4);
                 }
                 $('#UserSettingValue').val(data);
+            },
+            error: function (xhr) {
+                if (xhr.status === 404) {
+                    $('#UserSettingValue').val('');
+                } else {
+                    xhrFailCallback(xhr);
+                }
             }
         });
     }

app/View/Users/admin_add.ctp

@@ -38,7 +38,8 @@
                 $passwordPopover = '<span class="blue bold">' . __('Minimal length') . '</span>: ' . h($length) . '<br>';
                 $passwordPopover .= '<span class="blue bold">' . __('Complexity') . '</span>: ' . h($complexity);
                 echo $this->Form->input('password', array(
-                    'label' => __('Password') . ' <span id="PasswordPopover" data-content="' . h($passwordPopover) . '" class="fas fa-info-circle"></span>'
+                    'label' => __('Password') . ' <span id="PasswordPopover" data-content="' . h($passwordPopover) . '" class="fas fa-info-circle"></span>',
+                    'autocomplete' => 'new-password'
                 ));
                 echo $this->Form->input('confirm_password', array('type' => 'password', 'div' => array('class' => 'input password required')));
             ?>

app/View/Users/admin_edit.ctp

@@ -46,7 +46,8 @@
                 $passwordPopover = '<span class="blue bold">' . __('Length') .'</span>: ' . h($length) . '<br>';
                 $passwordPopover .= '<span class="blue bold">' . __('Complexity') .'</span>: ' . h($complexity);
                 echo $this->Form->input('password', array(
-                    'label' => __('Password') . ' <span id="PasswordPopover" data-content="' . h($passwordPopover) .'" class="fas fa-info-circle"></span>'
+                    'label' => __('Password') . ' <span id="PasswordPopover" data-content="' . h($passwordPopover) .'" class="fas fa-info-circle"></span>',
+                    'autocomplete' => 'new-password'
                 ));
                 echo $this->Form->input('confirm_password', array('type' => 'password', 'div' => array('class' => 'input password required')));
             ?>

app/View/Users/notification_settings.ctp

@@ -67,6 +67,18 @@ echo $this->element('genericElements/Form/genericForm', [
                 'type' => 'textarea',
                 'placeholder' => '["misp-galaxy:mitre-attack-pattern", "admiralty-scale"]',
             ],
+            [
+                'field' => 'periodic_settings.trending_period_amount',
+                'label' => __('Trending Period Amount'),
+                'class' => 'span6',
+                'type' => 'number',
+            ],
+            [
+                'field' => 'periodic_settings.include_correlations',
+                'label' => __('Include events correlations'),
+                'default' => 0,
+                'type' => 'checkbox'
+            ],
         ],
         'submit' => [
             'action' => $this->request->params['action'],

app/files/community-metadata/defaults.json

@@ -183,7 +183,7 @@
     "uuid": "53a830e6-8d07-4dd8-98d5-18cc4e66fc8c",
     "org_uuid": "",
     "org_name": "",
-    "description": "The ultimate goal of the project is to develop a NATO capability, available to all NATO nations, through which nations commit to sharing their information. ",
+    "description": "The ultimate goal of the project is to develop a NATO capability, available to all NATO nations, through which nations commit to sharing their information. This community is only open for official government entities, sponsored by their nation representative in the NATO Multinational MISP Steering Board.",
     "url": "https://misp.ncirc.nato.int",
     "sector": "Governmental",
     "nationality": "International",

app/files/misp-galaxy

@@ -1 +1 @@
-Subproject commit 1c8d82cfcc6ca14791d2c3311181170449de19dc
+Subproject commit eacab6ca27e1d1996bb28b7c617943052a41e3fd

app/files/taxonomies

@@ -1 +1 @@
-Subproject commit 7d379245b7a62831cc4b5d32e73e1f0d923f2624
+Subproject commit 57b125782cd372c8a762db7ce34f8a405752c6c1

app/files/warninglists

@@ -1 +1 @@
-Subproject commit 81d122e2df112888f93fd07f7a006e1ee432532b
+Subproject commit 1b026ee5115e5a6c7fda4cb7a6032c01e6f69a9c

app/webroot/js/misp.js

@@ -13,6 +13,10 @@ if (!String.prototype.startsWith) {
 }
 
 function escapeHtml(unsafe) {
+    if (typeof unsafe === "boolean" || typeof unsafe === "number") {
+        return unsafe;
+    }
+
     return unsafe
         .replace(/&/g, "&")
         .replace(/</g, "&lt;")
@@ -52,7 +56,9 @@ function rgb2hex(rgb) {
 }
 
 function xhrFailCallback(xhr) {
-    if (xhr.status === 401) {
+    if (xhr.status === 0) {
+        showMessage('fail', 'Something went wrong – server is not responding.');
+    } if (xhr.status === 401) {
         showMessage('fail', 'Unauthorized. Please reload page to log again.');
     } else if (xhr.status === 403 || xhr.status === 405) {
         showMessage('fail', 'Not allowed.');
@@ -425,7 +431,7 @@ function eventUnpublish() {
     $('.notPublished').show();
 }
 
-function updateIndex(id, context) {
+function updateIndex(id, context, callback) {
     var url, div;
     if (context === 'event') {
         if (typeof currentUri == 'undefined') {
@@ -442,6 +448,10 @@ function updateIndex(id, context) {
         dataType: "html",
         success: function (data) {
             $(div).html(data);
+            if (typeof callback !== "undefined") {
+                callback("success");
+            }
+
             if (typeof genericPopupCallback !== "undefined") {
                 genericPopupCallback("success");
             } else {
@@ -1275,10 +1285,8 @@ function submitPopoverForm(context_id, referer, update_context_id, modal, popove
                 $.get(baseurl + "/sightings/listSightings/" + id + "/attribute", function(data) {
                     $("#sightingsData").html(data);
                 }).fail(xhrFailCallback);
-                $('.sightingsToggle').removeClass('btn-primary');
+                $('.sightingsToggle').removeClass('btn-primary').addClass('btn-inverse');
-                $('.sightingsToggle').addClass('btn-inverse');
+                $('#sightingsListAllToggle').removeClass('btn-inverse').addClass('btn-primary');
-                $('#sightingsListAllToggle').removeClass('btn-inverse');
-                $('#sightingsListAllToggle').addClass('btn-primary');
             }
             if (referer === 'addEventReport' && typeof window.reloadEventReportTable === 'function') {
                 reloadEventReportTable()
@@ -1286,8 +1294,8 @@ function submitPopoverForm(context_id, referer, update_context_id, modal, popove
             }
             if (
                 (
-                    context == 'event' &&
+                    context === 'event' &&
-                    (referer == 'add' || referer == 'massEdit' || referer == 'replaceAttributes' || referer == 'addObjectReference' || referer == 'quickAddAttributeForm')
+                    (referer === 'add' || referer === 'massEdit' || referer === 'replaceAttributes' || referer === 'addObjectReference' || referer === 'quickAddAttributeForm')
                 )
             ){
                 eventUnpublish();
@@ -1340,10 +1348,15 @@ function handleAjaxModalResponse(response, context_id, url, referer, context, co
 
 function handleAjaxPopoverResponse(response, context_id, url, referer, context, contextNamingConvention) {
     responseArray = response;
-    var message = null;
     var result = "fail";
     if (responseArray.saved) {
-        updateIndex(context_id, context);
+        var callback = function() {
+            // Scroll to edited object after index is updated
+            if (referer === 'quickAddAttributeForm') {
+                scrollToElementIfNotVisible($("#Object_" + context_id + "_tr"));
+            }
+        }
+        updateIndex(context_id, context, callback);
         if (responseArray.success) {
             showMessage("success", responseArray.success);
             result = "success";
@@ -3666,6 +3679,24 @@ function pivotObjectReferences(url, uuid) {
     fetchAttributes(currentUri, {"focus": uuid});
 }
 
+function scrollToElementIfNotVisible($el) {
+    var isInViewport = function($el) {
+        var elementTop = $el.offset().top;
+        var elementBottom = elementTop + $el.outerHeight();
+
+        var viewportTop = $(window).scrollTop();
+        var viewportBottom = viewportTop + $(window).height();
+
+        return elementBottom > viewportTop && elementTop < viewportBottom;
+    };
+
+    if ($el.length && !isInViewport($el)) {
+        $([document.documentElement, document.body]).animate({
+            scrollTop: $el.offset().top - 45, // 42px is #topBar size, so make little bit more space
+        });
+    }
+}
+
 // Attribute filtering
 function filterAttributes(filter) {
     var data;
@@ -4500,38 +4531,43 @@ function add_basic_auth() {
 }
 
 function changeObjectReferenceSelectOption(selected, additionalData) {
+    var keys = {
+        "uuid": "UUID",
+        "category": "Category",
+        "type": "Type",
+        "value": "Value",
+        "to_ids": "To IDS",
+        "name": "Name",
+        "meta-category": "Meta category",
+    };
+
     var uuid = selected;
     var type = additionalData.itemOptions[uuid].type;
     $('#ObjectReferenceReferencedUuid').val(uuid);
-    if (type == "Attribute") {
+    var $targetData = $('#targetData');
-        $('#targetData').html("");
+    if (type === "Attribute") {
+        $targetData.html("");
         for (var k in targetEvent[type][uuid]) {
             if ($.inArray(k, ['uuid', 'category', 'type', 'value', 'to_ids']) !== -1) {
-                $('#targetData').append('<div><span id="' + uuid + '_' + k + '_key" class="bold"></span>: <span id="' + uuid + '_' + k + '_data"></span></div>');
+                $targetData.append('<div><span class="bold">' + keys[k] +  '</span>: ' + escapeHtml(targetEvent[type][uuid][k]) + '</div>');
-                $('#' + uuid + '_' + k + '_key').text(k);
-                $('#' + uuid + '_' + k + '_data').text(targetEvent[type][uuid][k]);
             }
         }
     } else {
-        $('#targetData').html("");
+        $targetData.html("");
         for (var k in targetEvent[type][uuid]) {
-            if (k == 'Attribute') {
+            if (k === 'Attribute') {
-                $('#targetData').append('<br /><div><span id="header" class="bold">Attributes:</span>');
+                $targetData.append('<br><div><span id="header" class="bold">Attributes:</span>');
-                for (attribute in targetEvent[type][uuid]['Attribute']) {
+                for (var attribute in targetEvent[type][uuid]['Attribute']) {
-                    for (k2 in targetEvent[type][uuid]['Attribute'][attribute]) {
+                    for (var k2 in targetEvent[type][uuid]['Attribute'][attribute]) {
                         if ($.inArray(k2, ['category', 'type', 'value', 'to_ids']) !== -1) {
-                            $('#targetData').append('<div class="indent"><span id="' + targetEvent[type][uuid]['Attribute'][attribute]['uuid'] + '_' + k2 + '_key" class="bold"></span>: <span id="' + targetEvent[type][uuid]['Attribute'][attribute]['uuid'] + '_' + k2 + '_data"></span></div>');
+                            $targetData.append('<div class="indent"><span class="bold">' + keys[k2] + '</span>: ' + escapeHtml(targetEvent[type][uuid]['Attribute'][attribute][k2]) + '</div>');
-                            $('#' + targetEvent[type][uuid]['Attribute'][attribute]['uuid'] + '_' + k2 + '_key').text(k2);
-                            $('#' + targetEvent[type][uuid]['Attribute'][attribute]['uuid'] + '_' + k2 + '_data').text(targetEvent[type][uuid]['Attribute'][attribute][k2]);
                         }
                     }
-                    $('#targetData').append('<br />');
+                    $targetData.append('<br>');
                 }
             } else {
                 if ($.inArray(k, ['name', 'uuid', 'meta-category']) !== -1) {
-                    $('#targetData').append('<div><span id="' + uuid + '_' + k + '_key" class="bold"></span>: <span id="' + uuid + '_' + k + '_data"></span></div>');
+                    $targetData.append('<div><span class="bold">' + keys[k] +  '</span>: ' + escapeHtml(targetEvent[type][uuid][k]) + '</div>');
-                    $('#' + uuid + '_' + k + '_key').text(k);
-                    $('#' + uuid + '_' + k + '_data').text(targetEvent[type][uuid][k]);
                 }
             }
         }

app/webroot/js/workflows-editor/workflows-editor.js

@@ -723,9 +723,9 @@ function loadWorkflow(workflow) {
     Object.values(workflow.data).forEach(function (node) {
         var module = all_modules_by_id[node.data.id] || all_triggers_by_id[node.data.id]
         if (!module) {
-            console.error('Tried to add node for unknown module ' + node.data.module_data.id + ' (' + node.id + ')')
+            console.error('Tried to add node for unknown module ' + node.data.id + ' (' + node.id + ')')
             var html = window['dotBlock_error']({
-                error: 'Invalid module id`' + node.data.module_data.id + '` (' + node.id + ')',
+                error: 'Invalid module id`' + node.data.id + '` (' + node.id + ')',
                 data: JSON.stringify(node.data.indexed_params, null, 2)
             })
             editor.addNode(

db_schema.json

@@ -8938,7 +8938,8 @@
         "object_template_elements": {
             "id": true,
             "object_relation": false,
-            "type": false
+            "type": false,
+            "object_template_id": false
         },
         "organisations": {
             "id": true,
@@ -9149,5 +9150,5 @@
             "uuid": false
         }
     },
-    "db_version": "97"
+    "db_version": "98"
 }

tests/testlive_comprehensive_local.py

@@ -536,6 +536,10 @@ class TestComprehensive(unittest.TestCase):
             for event in (first, second):
                 check_response(self.admin_misp_connector.delete_event(event))
 
+    def test_correlations_noacl(self):
+        with MISPSetting(self.admin_misp_connector, {"MISP.correlation_engine": "NoAcl"}):
+            self.test_correlations()
+
     def test_advanced_correlations(self):
         with MISPSetting(self.admin_misp_connector, {"MISP.enable_advanced_correlations": True}):
             first = create_simple_event()