fix: [Collections] path pluralisation fix inb acl check for collections, fixes #9745

- no longer breaks collections index

.github/workflows/main.yml

@@ -184,20 +184,8 @@ jobs:
           app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" 1
           app/Console/cake Admin setSetting "Plugin.ZeroMQ_audit_notifications_enable" 1
 
-      - name: Update Galaxies
-        run: app/Console/cake Admin updateGalaxies
-
-      - name: Update Taxonomies
-        run: app/Console/cake Admin updateTaxonomies
-
-      - name: Update Warninglists
-        run: app/Console/cake Admin updateWarningLists --verbose
-
-      - name: Update Noticelists
-        run: app/Console/cake Admin updateNoticeLists
-
-      - name: Update Object Templates
-        run: app/Console/cake Admin updateObjectTemplates 1
+      - name: Update JSON
+        run: app/Console/cake Admin updateJSON
 
       - name: Turn MISP live
         run: app/Console/cake Admin live 1
@@ -269,13 +257,16 @@ jobs:
       - name: Check requirements.txt
         run: python tests/check_requirements.py
 
-      - name: Logs
+      - name: System logs
         if: ${{ always() }}
         # update logs_test.sh when adding more logsources here
         run: |
           tail -n +1 `pwd`/app/tmp/logs/*
           tail -n +1 /var/log/apache2/*.log
 
+      - name: Application logs
+        if: ${{ always() }}
+        run: |
           app/Console/cake Log export /tmp/logs.json.gz --without-changes
           zcat /tmp/logs.json.gz
 

.gitignore

@@ -95,6 +95,7 @@ app/Lib/EventWarning/Custom/*
 /app/Config/database.php
 /app/Config/core.php
 /app/Config/config.php
+/app/Config/hmac_key.php
 /app/Console/Command/training.json
 /app/Lib/cakephp
 /app/webroot/gpg.asc

INSTALL/MYSQL.sql

@@ -1428,7 +1428,7 @@ CREATE TABLE IF NOT EXISTS `users` (
 -- Table structure for table `user_login_profiles`
 --
 
-CREATE TABLE `user_login_profiles` (
+CREATE TABLE IF NOT EXISTS `user_login_profiles` (
   `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
   `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
   `user_id` int(11) NOT NULL,
@@ -1698,4 +1698,4 @@ INSERT IGNORE INTO `org_blocklists` (`org_uuid`, `created`, `org_name`, `comment
 
 INSERT IGNORE INTO `admin_settings` (`setting`, `value`) VALUES 
 ('fix_login', NOW()),
-('default_role', 3);
+('default_role', 3);

PyMISP

@@ -1 +1 @@
-Subproject commit 4715f91d2ab948fb44640426be6a40099f94c910
+Subproject commit 8b4f98ac4c2e6c8cc1dba064f937dac816b67d0f

README.md

@@ -46,6 +46,8 @@ The objective of MISP is to foster the sharing of structured information within
 </tr>
 </table>
 
+[![CLA FREE initiative](https://raw.githubusercontent.com/ossbase-org/ossbase.org/main/logos/cla-free-small.png)](https://ossbase.org/initiatives/cla-free/)
+
 Core functions
 ------------------
 - An **efficient IOC and indicators** database, allowing to store technical and non-technical information about malware samples, incidents, attackers and intelligence.

VERSION.json

@@ -1 +1 @@
-{"major":2, "minor":4, "hotfix":186}
+{"major":2, "minor":4, "hotfix":192}

app/Config/bootstrap.default.php

@@ -180,3 +180,7 @@ CakeLog::config('error', array(
 CakePlugin::loadAll(array(
 	'CakeResque' => array('bootstrap' => true)
 ));
+
+
+// Enable the additional exception logging for certain failures (timeouts, out of memory, etc)
+Configure::write('Exception.renderer', 'AppExceptionRenderer');

app/Console/Command/AdminShell.php

@@ -302,16 +302,22 @@ class AdminShell extends AppShell
     public function updateJSON()
     {
         $this->out('Updating all JSON structures.');
-        $results = $this->Server->updateJSON();
-        foreach ($results as $type => $result) {
+        $overallSuccess = true;
+        foreach ($this->Server->updateJSON() as $type => $result) {
             $type = Inflector::pluralize(Inflector::humanize($type));
-            if ($result !== false) {
-                $this->out(__('%s updated.', $type));
+            if ($result['success']) {
+                $this->out(__('%s updated in %.2f seconds.', $type, $result['duration']));
             } else {
                 $this->out(__('Could not update %s.', $type));
+                $this->out($result['result']);
+                $overallSuccess = false;
             }
         }
-        $this->out('All JSON structures updated. Thank you and have a very safe and productive day.');
+        if ($overallSuccess) {
+            $this->out('All JSON structures updated. Thank you and have a very safe and productive day.');
+        } else {
+            $this->error('Some structure could no be updated');
+        }
     }
 
     public function updateGalaxies()
@@ -616,9 +622,9 @@ class AdminShell extends AppShell
         try {
             $redis = RedisTool::init();
             for ($i = 0; $i < 10; $i++) {
-                $persistence = $redis->info('persistence');
-                if (isset($persistence['loading']) && $persistence['loading']) {
-                    $this->out('Redis is still loading...');
+                $pong = $redis->ping();
+                if ($pong !== true) {
+                    $this->out('Redis is still loading... ' . $pong);
                     sleep(1);
                 } else {
                     break;

app/Console/Command/AppShell.php

@@ -18,6 +18,7 @@
 
 App::uses('AppModel', 'Model');
 App::uses('BackgroundJobsTool', 'Tools');
+App::uses('BenchmarkTool', 'Tools');
 
 require_once dirname(__DIR__) . '/../Model/Attribute.php';   // FIXME workaround bug where Vendor/symfony/polyfill-php80/Resources/stubs/Attribute.php is loaded instead
 
@@ -38,7 +39,18 @@ abstract class AppShell extends Shell
     {
         $configLoad = $this->Tasks->load('ConfigLoad');
         $configLoad->execute();
-
+        if (Configure::read('Plugin.Benchmarking_enable')) {
+            $Benchmark = new BenchmarkTool(ClassRegistry::init('User'));
+            $start_time = $Benchmark->startBenchmark();
+            register_shutdown_function(function () use ($start_time, $Benchmark) {
+                $Benchmark->stopBenchmark([
+                    'user' => 0,
+                    'controller' => 'Shell::' . $this->modelClass,
+                    'action' => $this->command,
+                    'start_time' => $start_time
+                ]);
+            });
+        }
         parent::initialize();
     }
 

app/Console/Command/Ls22Shell.php

@@ -20,7 +20,8 @@ class Ls22Shell extends AppShell
                 $this->__servers[] = [
                     'Server' => [
                         'url' => trim($fields[0]),
-                        'authkey' => trim($fields[2])
+                        'authkey' => trim($fields[2]),
+                        'self_signed' => true,
                     ]
                 ];
             }
@@ -30,6 +31,7 @@ class Ls22Shell extends AppShell
     public function getOptionParser()
     {
         $this->stdout->styles('green', array('text' => 'green'));
+        $this->stdout->styles('black', array('text' => 'black'));
 
         $parser = parent::getOptionParser();
         $parser->addSubcommand('enableTaxonomy', [
@@ -414,6 +416,13 @@ class Ls22Shell extends AppShell
 
     public function scores()
     {
+        $MITIGATION_DETECTION_OBJECT_UUIDs = [
+            'b5acf82e-ecca-4868-82fe-9dbdf4d808c3', # yara
+            '3c177337-fb80-405a-a6c1-1b2ddea8684a', # suricata
+            'aa21a3cd-ab2c-442a-9999-a5e6626591ec', # sigma
+            '6bce7d01-dbec-4054-b3c2-3655a19382e2', # script
+            '35b4dd03-4fa9-4e0e-97d8-a2867b11c956', # yabin
+        ];
         $results = [];
         $this->__getInstances($this->param('instances'));
         $server = null;
@@ -464,6 +473,7 @@ class Ls22Shell extends AppShell
             $params = [
                 'org' => $org_id,
                 'includeWarninglistHits' => true,
+                // 'includeAnalystData' => true,
             ];
             if (!empty($time_range)) {
                 $params['publish_timestamp'] = $time_range;
@@ -486,6 +496,8 @@ class Ls22Shell extends AppShell
                 'warnings' => 0,
                 'events_extended' => 0,
                 'extending_events' => 0,
+                'mitigation_detection_rules_count' => 0,
+                'analyst_data_count' => 0,
             ];
             foreach ($events['response'] as $event) {
                 $event_uuid_per_org[$event['Event']['uuid']] = $event['Event']['Orgc']['name'];
@@ -511,6 +523,11 @@ class Ls22Shell extends AppShell
                         }
                     }
                 }
+
+                #  ['Note' => 0, 'Opinion' => 0, 'Relationship' => 0,]
+                $analystDataCount = $this->countAnalystData($event['Event'], $org_name);
+                $results[$org_name]['analyst_data_count'] = $analystDataCount['Note'] + $analystDataCount['Opinion'] + $analystDataCount['Relationship'];
+
                 foreach ($event['Event']['Attribute'] as $attribute) {
                     if (!empty($attribute['referenced_by'])) {
                         $results[$org_name]['connected_elements'] +=1;
@@ -533,6 +550,9 @@ class Ls22Shell extends AppShell
                     foreach ($event['Event']['Object'] as $object) {
                         $results[$org_name]['attribute_count'] += count($object['Attribute']);
                         $results[$org_name]['object_count'] += 1;
+                        if (in_array($object['template_uuid'], $MITIGATION_DETECTION_OBJECT_UUIDs)) {
+                            $results[$org_name]['mitigation_detection_rules_count'] += 1;
+                        }
                         if (!empty($object['ObjectReference'])) {
                             $results[$org_name]['connected_elements'] += 1;
                         }
@@ -547,6 +567,9 @@ class Ls22Shell extends AppShell
                                 }
                             }
                         }
+                        if (!empty($attribute['warnings'])) {
+                            $results[$org_name]['warnings'] += 1;
+                        }
                     }
                 }
 
@@ -579,13 +602,24 @@ class Ls22Shell extends AppShell
                 } else {
                     $results[$k]['metrics']['warnings'] = 0;
                 }
+                $results[$k]['metrics']['mitigation_detection_rules'] = 100 * ($result['mitigation_detection_rules_count'] / ($result['event_count']));
                 $results[$k]['metrics']['connectedness'] = 100 * ($result['connected_elements'] / ($result['attribute_count'] + $result['object_count']));
                 $results[$k]['metrics']['attack_weight'] = 100 * (2*($result['attack']) + $result['attribute_attack']) / ($result['attribute_count'] + $result['object_count']);
                 $results[$k]['metrics']['other_weight'] = 100 * (2*($result['other']) + $result['attribute_other']) / ($result['attribute_count'] + $result['object_count']);
-                $results[$k]['metrics']['collaboration'] = 100 * ((2*$result['events_extended'] + $result['extending_events']) / $result['event_count']);
-                $results[$k]['metrics']['collaboration'] = 100 * (2*(2*$result['events_extended'] + $result['extending_events']) / $result['event_count']);
+                // $results[$k]['metrics']['collaboration'] = 100 * ((2*$result['events_extended'] + $result['extending_events']) / $result['event_count']);
+                // $results[$k]['metrics']['collaboration'] = 100 * (2*(2*$result['events_extended'] + $result['extending_events']) / $result['event_count']);
+                $results[$k]['metrics']['collaboration'] = 100 * (($result['events_extended'] + $result['extending_events']));
+                
+                # Math magic to give lots of points of you extend or have your events extended. You quickly get point, but it slows down very quick
+                if (($result['events_extended'] + $result['extending_events']) == 0) {
+                    $results[$k]['metrics']['collaboration'] = 0;
+                } else {
+                    $results[$k]['metrics']['collaboration'] = min(5*log(($result['events_extended'] + $result['extending_events']), 1.17), 100);
+                }
+
+                $results[$k]['metrics']['collaboration_analyst'] = $result['analyst_data_count'] > 0 ? 100 : 0;
             }
-            foreach (['connectedness', 'attack_weight', 'other_weight', 'warnings', 'collaboration'] as $metric) {
+            foreach (['connectedness',  'mitigation_detection_rules', 'attack_weight', 'other_weight', 'warnings', 'collaboration', 'collaboration_analyst'] as $metric) {
                 if (empty($results[$k]['metrics'][$metric])) {
                     $results[$k]['metrics'][$metric] = 0;
                 }
@@ -594,18 +628,21 @@ class Ls22Shell extends AppShell
                 }
             }
             $results[$k]['score'] = round(
-                    20 * $results[$k]['metrics']['warnings'] +
-                    20 * $results[$k]['metrics']['connectedness'] +
+                    15 * $results[$k]['metrics']['warnings'] +
+                    15 * $results[$k]['metrics']['mitigation_detection_rules'] +
+                    10 * $results[$k]['metrics']['connectedness'] +
                     40 * $results[$k]['metrics']['attack_weight'] +
                     10 * $results[$k]['metrics']['other_weight'] +
+                    // 7 * $results[$k]['metrics']['collaboration'] + 3 * $results[$k]['metrics']['collaboration_analyst']
                     10 * $results[$k]['metrics']['collaboration']
                 ) / 100;
             $scores[$k]['total'] = $results[$k]['score'];
-            $scores[$k]['warnings'] = round(20 * $results[$k]['metrics']['warnings']);
-            $scores[$k]['connectedness'] = round(20 * $results[$k]['metrics']['connectedness']);
+            $scores[$k]['warnings'] = round(15 * $results[$k]['metrics']['warnings']);
+            $scores[$k]['mitigation_detection_rules'] = round(15 * $results[$k]['metrics']['mitigation_detection_rules']);
+            $scores[$k]['connectedness'] = round(10 * $results[$k]['metrics']['connectedness']);
             $scores[$k]['attack_weight'] = round(40 * $results[$k]['metrics']['attack_weight']);
             $scores[$k]['other_weight'] = round(10 * $results[$k]['metrics']['other_weight']);
-            $scores[$k]['collaboration'] = round(10 * $results[$k]['metrics']['collaboration']);
+            $scores[$k]['collaboration'] = round(7 * $results[$k]['metrics']['collaboration']) + round(3 * $results[$k]['metrics']['collaboration_analyst']);
         }
         arsort($scores, SORT_DESC);
         $this->out(str_repeat('=', 128), 1, Shell::NORMAL);
@@ -618,20 +655,22 @@ class Ls22Shell extends AppShell
         $this->out(str_repeat('=', 128), 1, Shell::NORMAL);
         foreach ($scores as $org => $score) {
             $score_string[0] = str_repeat('█', round($score['warnings']/100));
-            $score_string[1] = str_repeat('█', round($score['connectedness']/100));
-            $score_string[2] = str_repeat('█', round($score['attack_weight']/100));
-            $score_string[3] = str_repeat('█', round($score['other_weight']/100));
-            $score_string[4] = str_repeat('█', round($score['collaboration']/100));
+            $score_string[1] = str_repeat('█', round($score['mitigation_detection_rules']/100));
+            $score_string[2] = str_repeat('█', round($score['connectedness']/100));
+            $score_string[3] = str_repeat('█', round($score['attack_weight']/100));
+            $score_string[4] = str_repeat('█', round($score['other_weight']/100));
+            $score_string[5] = str_repeat('█', round($score['collaboration']/100));
             $this->out(sprintf(
                 '| %s | %s | %s |',
                 str_pad($org, 10, ' ', STR_PAD_RIGHT),
                 sprintf(
-                    '<error>%s</error><warning>%s</warning><question>%s</question><info>%s</info><green>%s</green>%s',
+                    '<error>%s</error><black>%s</black><warning>%s</warning><question>%s</question><info>%s</info><green>%s</green>%s',
                     $score_string[0],
                     $score_string[1],
                     $score_string[2],
                     $score_string[3],
                     $score_string[4],
+                    $score_string[5],
                     str_repeat(' ', 100 - mb_strlen(implode('', $score_string)))
                 ),
                 str_pad($score['total'] . '%', 8, ' ', STR_PAD_RIGHT)
@@ -639,8 +678,9 @@ class Ls22Shell extends AppShell
         }
         $this->out(str_repeat('=', 128), 1, Shell::NORMAL);
         $this->out(sprintf(
-            '| Legend: %s %s %s %s %s |',
+            '| Legend: %s %s %s %s %s %s |',
             '<error>█: Warnings</error>',
+            '<black>█: Detection/Mitigation Rules</black>',
             '<warning>█: Connectedness</warning>',
             '<question>█: ATT&CK context</question>',
             '<info>█: Other Context</info>',
@@ -650,4 +690,30 @@ class Ls22Shell extends AppShell
         $this->out(str_repeat('=', 128), 1, Shell::NORMAL);
         file_put_contents(APP . 'tmp/report.json', json_encode($results, JSON_PRETTY_PRINT));
     }
+
+    private function countAnalystData($data, $orgc_name): array {
+        $analystTypes = ['Note', 'Opinion', 'Relationship'];
+        $counts = [
+            'Note' => 0,
+            'Opinion' => 0,
+            'Relationship' => 0,
+        ];
+
+        foreach ($analystTypes as $type) {
+            if (!empty($data[$type])) {
+                foreach ($data[$type] as $entry) {
+                    if ($entry['Orgc']['name'] == $orgc_name) {
+                        $counts[$type] += 1;
+                    }
+                }
+                foreach ($data[$type] as $child) {
+                    $nestedCounts = $this->countAnalystData($child, $orgc_name);
+                    $counts['Note'] += $nestedCounts['Note'];
+                    $counts['Opinion'] += $nestedCounts['Opinion'];
+                    $counts['Relationship'] += $nestedCounts['Relationship'];
+                }
+            }
+        }
+        return $counts;
+    }
 }

app/Console/Command/OrganisationShell.php

@@ -0,0 +1,64 @@
+<?php
+
+/**
+ * @property User $User
+ * @property Log $Log
+ * @property UserLoginProfile $UserLoginProfile
+ */
+class OrganisationShell extends AppShell
+{
+    public $uses = ['Organisation'];
+
+    public function getOptionParser()
+    {
+        $parser = parent::getOptionParser();
+        $parser->addSubcommand('list', [
+            'help' => __('Get list of organisations.'),
+            'parser' => [
+                'arguments' => [
+                    'filter' => ['help' => __('Filter the list by name.'), 'required' => false],
+                    'local' => ['help' => __('Filter the list by local/known organisations.'), 'required' => false],
+                ],
+                'options' => [
+                    'json' => ['help' => __('Output as JSON.'), 'boolean' => true],
+                ],
+            ]
+        ]);
+        return $parser;
+    }
+
+    public function list()
+    {
+        $filter = $this->args[0] ?? null;
+        $local = isset($this->args[1]) ? $this->args[1] : null;
+        $conditions = [];
+        if ($filter) {
+            $conditions = ['OR' => [
+                'Organisation.name LIKE' => "%$filter%",
+                'Organisation.uuid LIKE' => "%$filter%"
+            ]];
+        }
+        if ($local !== null) {
+            $conditions['OR'][] = [
+                'Organisation.local' => $local
+            ];
+        }
+        $organisations = $this->Organisation->find('all', [
+            'recursive' => -1,
+            'conditions' => $conditions
+        ]);
+        if ($this->params['json']) {
+            $this->out($this->json($organisations));
+        } else {
+            foreach ($organisations as $organisation) {
+                $this->out(sprintf(
+                    '%d. [%s] %s',
+                    $organisation['Organisation']['id'],
+                    $organisation['Organisation']['uuid'],
+                    $organisation['Organisation']['name']
+                ));
+            }
+            $this->out(count($organisations) . ' hits.');
+        }
+    }
+}

app/Console/Command/RoleShell.php

@@ -0,0 +1,56 @@
+<?php
+
+/**
+ * @property User $User
+ * @property Log $Log
+ * @property UserLoginProfile $UserLoginProfile
+ */
+class RoleShell extends AppShell
+{
+    public $uses = ['Role'];
+
+    public function getOptionParser()
+    {
+        $parser = parent::getOptionParser();
+        $parser->addSubcommand('list', [
+            'help' => __('Get list of the roles.'),
+            'parser' => [
+                'arguments' => [
+                    'filter' => ['help' => __('Filter list by name.'), 'required' => false],
+                ],
+                'options' => [
+                    'json' => ['help' => __('Output as JSON.'), 'boolean' => true],
+                ],
+            ]
+        ]);
+        return $parser;
+    }
+
+    public function list()
+    {
+        $filter = $this->args[0] ?? null;
+        if ($filter) {
+            $conditions = ['OR' => [
+                'Role.name LIKE' => "%$filter%"
+            ]];
+        } else {
+            $conditions = [];
+        }
+        $roles = $this->Role->find('all', [
+            'recursive' => -1,
+            'conditions' => $conditions
+        ]);
+        if ($this->params['json']) {
+            $this->out($this->json($roles));
+        } else {
+            foreach ($roles as $role) {
+                $this->out(sprintf(
+                    '%d. %s',
+                    $role['Role']['id'],
+                    $role['Role']['name']
+                ));
+            }
+        }
+        $this->out(count($roles) . ' hits.');
+    }
+}

app/Console/Command/ServerShell.php

@@ -125,7 +125,6 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Pull'] . PHP_EOL);
         }
-
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $serverId = $this->args[1];
@@ -145,6 +144,10 @@ class ServerShell extends AppShell
         if (!empty($this->args[4]) && $this->args[4] === 'force') {
             $force = true;
         }
+
+        // Try to enable garbage collector as pulling events can use a lot of memory
+        gc_enable();
+
         try {
             $result = $this->Server->pull($user, $technique, $server, $jobId, $force);
             if (is_array($result)) {
@@ -166,7 +169,7 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Push'] . PHP_EOL);
         }
-
+        
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $serverId = $this->args[1];
@@ -370,7 +373,7 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Fetch feeds as local data'] . PHP_EOL);
         }
-
+        
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $feedId = $this->args[1];
@@ -426,7 +429,7 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Cache server'] . PHP_EOL);
         }
-
+        
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $scope = $this->args[1];
@@ -489,7 +492,7 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Cache feeds for quick lookups'] . PHP_EOL);
         }
-
+        
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $scope = $this->args[1];
@@ -735,6 +738,7 @@ class ServerShell extends AppShell
 
     public function sendPeriodicSummaryToUsers()
     {
+        
         $periods = $this->__getPeriodsForToday();
         $start_time = time();
         echo __n('Started periodic summary generation for the %s period', 'Started periodic summary generation for periods: %s', count($periods), implode(', ', $periods)) . PHP_EOL;
@@ -800,7 +804,7 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Push Taxii'] . PHP_EOL);
         }
-
+        
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $serverId = $this->args[1];

app/Console/Command/StatisticsShell.php

@@ -303,10 +303,10 @@ class StatisticsShell extends AppShell {
         $this->out(json_encode([
             'events' => $this->Event->find('count'),
             'attributes' => $this->Event->Attribute->find('count',
-                ['conditions' => ['Attribute.deleted' => 0], 'recursive' => -1]
+                ['recursive' => -1]
             ),
             'objects' => $this->Event->Object->find('count',
-                ['conditions' => ['Object.deleted' => 0], 'recursive' => -1]
+                ['recursive' => -1]
             ),
             'correlations' => $this->Correlation->find('count') / 2,
             'users' => $this->User->find('count',

app/Console/Command/UserShell.php

@@ -16,7 +16,21 @@ class UserShell extends AppShell
             'help' => __('Get list of user accounts.'),
             'parser' => [
                 'arguments' => [
-                    'userId' => ['help' => __('User ID or e-mail address.'), 'required' => true],
+                    'userId' => ['help' => __('User ID or e-mail address to filter.'), 'required' => false],
+                ],
+                'options' => [
+                    'json' => ['help' => __('Output as JSON.'), 'boolean' => true],
+                ],
+            ]
+        ]);
+        $parser->addSubcommand('create', [
+            'help' => __('Create a new user account.'),
+            'parser' => [
+                'arguments' => [
+                    'email' => ['help' => __('E-mail address (also used as the username.'), 'required' => true],
+                    'role_id' => ['help' => __('Role ID of the user. For a list of available roles, use `cake Roles list`.'), 'required' => true],
+                    'org_id' => ['help' => __('Organisation under which the user should be created'), 'required' => true],
+                    'password' => ['help' => __('Enter a password to assign to the user (optional) - if none is set, the user will receive a temporary password.')]
                 ],
                 'options' => [
                     'json' => ['help' => __('Output as JSON.'), 'boolean' => true],
@@ -82,6 +96,15 @@ class UserShell extends AppShell
                 ],
             ],
         ]);
+        $parser->addSubcommand('change_role', [
+            'help' => __('Change user role.'),
+            'parser' => [
+                'arguments' => [
+                    'userId' => ['help' => __('User ID or e-mail address.'), 'required' => true],
+                    'new_role' => ['help' => __('Role ID or Role name.'), 'required' => true],
+                ]
+            ],
+        ]);
         $parser->addSubcommand('change_authkey', [
             'help' => __('Change authkey. When advanced authkeys are enabled, old authkeys will be disabled.'),
             'parser' => [
@@ -180,6 +203,48 @@ class UserShell extends AppShell
         }
     }
 
+    public function create()
+    {
+        if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
+            $this->err('Invalid input. Usage: `User create [email] [role_id] [org_id] [password:optional]`');
+        }
+        $user = [
+            'email' => $this->args[0],
+            'role_id' => $this->args[1],
+            'org_id' => $this->args[2],
+            'change_pw' => true
+        ];
+        if (!empty($this->args[3])) {
+            $user['password'] = $this->args[3];
+            $user['confirm_password'] = $this->args[3];
+            $user['change_pw'] = true;
+        }
+        $this->User->create();
+        $result = $this->User->save($user);
+        // do not fetch sensitive or big values
+        $schema = $this->User->schema();
+        unset($schema['authkey']);
+        unset($schema['password']);
+        unset($schema['gpgkey']);
+        unset($schema['certif_public']);
+
+        $fields = array_keys($schema);
+        $fields[] = 'Role.*';
+        $fields[] = 'Organisation.*';
+
+        $user = $this->User->find('first', [
+            'recursive' => -1,
+            'fields' => $fields,
+            'conditions' => ['User.id' => $this->User->id],
+            'contain' => ['Organisation', 'Role', 'UserSetting'],
+        ]);
+        if ($this->params['json']) {
+            $this->out($this->json($user));
+        } else {
+            $this->out('User created.');
+        }
+    }
+
     public function init()
     {
         if (!Configure::read('Security.salt')) {
@@ -443,6 +508,35 @@ class UserShell extends AppShell
         }
     }
 
+    public function change_role()
+    {
+        list($userId, $newRole) = $this->args;
+        $user = $this->getUser($userId);
+
+        if (is_numeric($newRole)) {
+            $conditions = ['Role.id' => $newRole];
+        } else {
+            $conditions = ['Role.name' => $newRole];
+        }
+
+        $newRoleFromDb = $this->User->Role->find('first', [
+            'conditions' => $conditions,
+            'fields' => ['Role.id'],
+        ]);
+
+        if (empty($newRoleFromDb)) {
+            $this->error("Role `$newRole` not found.");
+        }
+
+        if ($newRoleFromDb['Role']['id'] == $user['role_id']) {
+            $this->error("Role `$newRole` is already assigned to {$user['email']}.");
+        }
+
+        $this->User->updateField($user, 'role_id', $newRoleFromDb['Role']['id']);
+
+        $this->out("Role changed from `{$user['role_id']}` to `{$newRoleFromDb['Role']['id']}`.");
+    }
+
     public function user_ips()
     {
         list($userId) = $this->args;
@@ -575,7 +669,7 @@ class UserShell extends AppShell
     }
 
     /**
-     * @param string|int $userId
+     * @param string|int $userId User ID or User e-mail
      * @return array
      */
     private function getUser($userId)

app/Console/Command/WorkflowShell.php

@@ -17,6 +17,9 @@ class WorkflowShell extends AppShell {
         $data = JsonTool::decode($this->args[1]);
         $logging = JsonTool::decode($this->args[2]);
         $jobId = $this->args[3];
+        if (!empty($this->args[4])) {
+            Configure::write('CurrentUserId', JsonTool::decode($this->args[4]));
+        }
 
         $blockingErrors = [];
         $executionSuccess = $this->Workflow->executeWorkflowForTrigger($trigger_id, $data, $blockingErrors);

app/Controller/AnalystDataController.php

@@ -320,6 +320,11 @@ class AnalystDataController extends AppController
                 $this->AnalystData = $this->{$vt};
                 $this->modelClass = $vt;
                 $this->{$vt}->current_user = $this->Auth->user();
+                if (!empty($this->request->data)) {
+                    if (!isset($this->request->data[$type])) {
+                        $this->request->data = [$type => $this->request->data];
+                    }
+                }
                 return $vt;
             }
         }

app/Controller/AppController.php

@@ -33,13 +33,19 @@ class AppController extends Controller
 
     public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
 
-    private $__queryVersion = '159';
-    public $pyMispVersion = '2.4.186';
+    private $__queryVersion = '162';
+    public $pyMispVersion = '2.4.190';
     public $phpmin = '7.2';
     public $phprec = '7.4';
     public $phptoonew = '8.0';
     private $isApiAuthed = false;
 
+    /** @var redis */
+    private $redis = null;
+
+    /** @var benchmark_results */
+    private $benchmark_results = null;
+
     public $baseurl = '';
 
     public $restResponsePayload = null;
@@ -57,9 +63,14 @@ class AppController extends Controller
     /** @var ACLComponent */
     public $ACL;
 
+    /** @var BenchmarkComponent */
+    public $Benchmark;
+
     /** @var RestResponseComponent */
     public $RestResponse;
 
+    public $start_time;
+
     public function __construct($request = null, $response = null)
     {
         parent::__construct($request, $response);
@@ -97,13 +108,19 @@ class AppController extends Controller
 
     public function beforeFilter()
     {
-        $controller = $this->request->params['controller'];
-        $action = $this->request->params['action'];
-
         if (Configure::read('MISP.system_setting_db')) {
             App::uses('SystemSetting', 'Model');
             SystemSetting::setGlobalSetting();
         }
+        
+        $this->User = ClassRegistry::init('User');
+        if (Configure::read('Plugin.Benchmarking_enable')) {
+            App::uses('BenchmarkTool', 'Tools');
+            $this->Benchmark = new BenchmarkTool($this->User);
+            $this->start_time = $this->Benchmark->startBenchmark();
+        }
+        $controller = $this->request->params['controller'];
+        $action = $this->request->params['action'];
 
         $this->_setupBaseurl();
         $this->Auth->loginRedirect = $this->baseurl . '/users/routeafterlogin';
@@ -147,8 +164,6 @@ class AppController extends Controller
             Configure::write('Config.language', 'eng');
         }
 
-        $this->User = ClassRegistry::init('User');
-
         if (!empty($this->request->params['named']['disable_background_processing'])) {
             Configure::write('MISP.background_jobs', 0);
         }
@@ -226,6 +241,7 @@ class AppController extends Controller
         ) {
             // REST authentication
             if ($this->_isRest() || $this->_isAutomation()) {
+                
                 // disable CSRF for REST access
                 $this->Security->csrfCheck = false;
                 $loginByAuthKeyResult = $this->__loginByAuthKey();
@@ -633,21 +649,21 @@ class AppController extends Controller
         }
 
         // Check if user accepted terms and conditions
-        if (!$user['termsaccepted'] && !empty(Configure::read('MISP.terms_file')) && !$this->_isControllerAction(['users' => ['terms', 'logout', 'login', 'downloadTerms']])) {
+        if (!$user['termsaccepted'] && !empty(Configure::read('MISP.terms_file')) && !$this->_isControllerAction(['users' => ['terms', 'logout', 'login', 'downloadTerms', 'totp_new', 'email_otp']])) {
             //if ($this->_isRest()) throw new MethodNotAllowedException('You have not accepted the terms of use yet, please log in via the web interface and accept them.');
             $this->redirect(array('controller' => 'users', 'action' => 'terms', 'admin' => false));
             return false;
         }
 
         // Check if user must change password
-        if ($user['change_pw'] && !$this->_isControllerAction(['users' => ['terms', 'change_pw', 'logout', 'login']])) {
+        if ($user['change_pw'] && !$this->_isControllerAction(['users' => ['terms', 'change_pw', 'logout', 'login', 'totp_new', 'email_otp']])) {
             //if ($this->_isRest()) throw new MethodNotAllowedException('Your user account is expecting a password change, please log in via the web interface and change it before proceeding.');
             $this->redirect(array('controller' => 'users', 'action' => 'change_pw', 'admin' => false));
             return false;
         }
 
         // Check if user must read news
-        if (!$this->_isControllerAction(['news' => ['index'], 'users' => ['terms', 'change_pw', 'login', 'logout']])) {
+        if (!$this->_isControllerAction(['news' => ['index'], 'users' => ['terms', 'change_pw', 'login', 'logout', 'totp_new', 'email_otp']])) {
             $this->loadModel('News');
             $latestNewsCreated = $this->News->latestNewsTimestamp();
             if ($latestNewsCreated && $user['newsread'] < $latestNewsCreated) {
@@ -863,6 +879,21 @@ class AppController extends Controller
 
     public function afterFilter()
     {
+        // benchmarking
+        if (Configure::read('Plugin.Benchmarking_enable') && isset($this->Benchmark)) {
+            $this->Benchmark->stopBenchmark([
+                'user' => $this->Auth->user('id'),
+                'controller' => $this->request->params['controller'],
+                'action' => $this->request->params['action'],
+                'start_time' => $this->start_time
+            ]);
+
+            //if ($redis && !$redis->exists('misp:auth_fail_throttling:' . $key)) {
+                //$redis->setex('misp:auth_fail_throttling:' . $key, 3600, 1);
+                //return true;
+            //}
+            
+        }
         if ($this->isApiAuthed && $this->_isRest() && !Configure::read('Security.authkey_keep_session')) {
             $this->Session->destroy();
         }
@@ -961,31 +992,6 @@ class AppController extends Controller
         return $this->userRole['perm_site_admin'];
     }
 
-    protected function _getApiAuthUser($key, &$exception)
-    {
-        if (strlen($key) === 40) {
-            // check if the key is valid -> search for users based on key
-            $user = $this->_checkAuthUser($key);
-            if (!$user) {
-                $exception = $this->RestResponse->throwException(
-                    401,
-                    __('This authentication key is not authorized to be used for exports. Contact your administrator.')
-                );
-                return false;
-            }
-        } else {
-            $user = $this->Auth->user();
-            if (!$user) {
-                $exception = $this->RestResponse->throwException(
-                    401,
-                    __('You have to be logged in to do that.')
-                );
-                return false;
-            }
-        }
-        return $user;
-    }
-
     private function __captureParam($data, $param, $value)
     {
         if ($this->modelClass->checkParam($param)) {
@@ -1033,7 +1039,19 @@ class AppController extends Controller
                     $data = array_merge($data, $temp);
                 } else {
                     foreach ($options['paramArray'] as $param) {
-                        if (isset($temp[$param])) {
+                        if (substr($param, -1) == '*') {
+                            $root = substr($param, 0, strlen($param)-1);
+                            foreach ($temp as $existingParamKey => $v) {
+                                $leftover = substr($existingParamKey, strlen($param)-1);
+                                if (
+                                    $root == substr($existingParamKey, 0, strlen($root)) &&
+                                    preg_match('/^[\w_-. ]+$/', $leftover) == 1
+                                ) {
+                                    $data[$existingParamKey] = $temp[$existingParamKey];
+                                    break;
+                                }
+                            }
+                        } else if (isset($temp[$param])) {
                             $data[$param] = $temp[$param];
                         }
                     }
@@ -1101,6 +1119,23 @@ class AppController extends Controller
 
     protected function _checkAuthUser($authkey)
     {
+        if (Configure::read('Security.api_key_quick_lookup')) {
+            $redis = RedisTool::init();
+            if (file_exists(APP . 'Config/hmac_key.php')) {
+                include(APP . 'Config/hmac_key.php');
+                $hashed_authkey = hash_hmac('sha512', $authkey, $hmac_key);
+                if ($redis && $redis->exists('misp:fast_authkey_lookup:' . $hashed_authkey)) {
+                    $user = RedisTool::deserialize($redis->get('misp:fast_authkey_lookup:' . $hashed_authkey));
+                    if ($user) {
+                        return $user;
+                    }
+                }
+            } else {
+                App::uses('RandomTool', 'Tools');
+                $hmac_key = RandomTool::random_str(true, 40);
+                file_put_contents(APP . 'Config/hmac_key.php', sprintf('<?php%s$hmac_key = \'%s\';', PHP_EOL, $hmac_key));
+            }
+        }
         if (Configure::read('Security.advanced_authkeys')) {
             $user = $this->User->AuthKey->getAuthUserByAuthKey($authkey);
         } else {
@@ -1114,6 +1149,13 @@ class AppController extends Controller
             return false;
         }
         $user['logged_by_authkey'] = true;
+        if (Configure::read('Security.api_key_quick_lookup') && !empty($hmac_key) && $redis) {
+            $expiration = Configure::read('Security.api_key_quick_lookup_expiration') ? Configure::read('Security.api_key_quick_lookup_expiration') : 180;
+            if ($redis) {
+                $hashed_authkey = hash_hmac('sha512', $authkey, $hmac_key);
+                $redis->setex('misp:fast_authkey_lookup:' . $hashed_authkey, $expiration, RedisTool::serialize($user));
+            }
+        }
         return $user;
     }
 
@@ -1162,12 +1204,11 @@ class AppController extends Controller
                         null);
                     $this->__preAuthException($authName . ' authentication failed. Contact your MISP support for additional information at: ' . Configure::read('MISP.contact'));
                 }
-                $temp = $this->_checkExternalAuthUser($server[$headerNamespace . $header]);
-                $user['User'] = $temp;
-                if ($user['User']) {
-                    $this->User->updateLoginTimes($user['User']);
+                $user = $this->_checkExternalAuthUser($server[$headerNamespace . $header]);
+                if ($user) {
+                    $this->User->updateLoginTimes($user);
                     //$this->Session->renew();
-                    $this->Session->write(AuthComponent::$sessionKey, $user['User']);
+                    $this->Session->write(AuthComponent::$sessionKey, $user);
                     if (Configure::read('MISP.log_auth')) {
                         $this->Log = ClassRegistry::init('Log');
                         $change = $this->User->UserLoginProfile->_getUserProfile();
@@ -1177,7 +1218,7 @@ class AppController extends Controller
                             $user,
                             'auth',
                             'User',
-                            $user['User']['id'],
+                            $user['id'],
                             'Successful authentication using ' . $authName . ' key',
                             json_encode($change));
                     }
@@ -1327,13 +1368,8 @@ class AppController extends Controller
         if ($filters === false) {
             return $exception;
         }
-        $key = empty($filters['key']) ? $filters['returnFormat'] : $filters['key'];
-        $user = $this->_getApiAuthUser($key, $exception);
-        if ($user === false) {
-            return $exception;
-        }
 
-        session_write_close(); // Rest search can be longer, so close session to allow concurrent requests
+        $user = $this->_closeSession();
 
         if (isset($filters['returnFormat'])) {
             $returnFormat = $filters['returnFormat'];
@@ -1520,10 +1556,20 @@ class AppController extends Controller
      * Close session without writing changes to them and return current user.
      * @return array
      */
-    protected function _closeSession()
+    protected function _closeSession($saveSession = false)
     {
         $user = $this->Auth->user();
-        session_abort();
+
+        // Hack to store user info in static AuthComponent::$_user variable to avoid starting session again by calling
+        // $this->Auth->user()
+        AuthComponent::$sessionKey = null;
+        $this->Auth->login($user);
+
+        if ($saveSession) {
+            @session_write_close();
+        } else {
+            session_abort();
+        }
         return $user;
     }
 

app/Controller/AttributesController.php

@@ -33,38 +33,16 @@ class AttributesController extends AppController
     {
         parent::beforeFilter();
 
-        $this->Auth->allow('restSearch');
-        $this->Auth->allow('returnAttributes');
-        $this->Auth->allow('downloadAttachment');
-        $this->Auth->allow('text');
-        $this->Auth->allow('rpz');
-        $this->Auth->allow('bro');
-
         // permit reuse of CSRF tokens on the search page.
         if ('search' === $this->request->params['action']) {
             $this->Security->csrfCheck = false;
         }
         $this->Security->unlockedActions[] = 'getMassEditForm';
         $this->Security->unlockedActions[] = 'search';
+
         if ($this->request->action === 'add_attachment') {
             $this->Security->unlockedFields = array('values');
-        }
-
-        // convert uuid to id if present in the url and overwrite id field
-        if (isset($this->request->params->query['uuid'])) {
-            $params = array(
-                    'conditions' => array('Attribute.uuid' => $this->request->params->query['uuid']),
-                    'recursive' => 0,
-                    'fields' => 'Attribute.id'
-                    );
-            $result = $this->Attribute->find('first', $params);
-            if (isset($result['Attribute']) && isset($result['Attribute']['id'])) {
-                $id = $result['Attribute']['id'];
-                $this->params->addParams(array('pass' => array($id))); // FIXME find better way to change id variable if uuid is found. params->url and params->here is not modified accordingly now
-            }
-        }
-
-        if ($this->request->action === 'viewPicture') {
+        } elseif ($this->request->action === 'viewPicture') {
             $this->Security->doNotGenerateToken = true;
         }
     }
@@ -793,12 +771,14 @@ class AttributesController extends AppController
                 $result = $this->Attribute->save($this->request->data, array('fieldList' => Attribute::EDITABLE_FIELDS));
                 if ($result) {
                     $this->Attribute->AttributeTag->handleAttributeTags($this->Auth->user(), $this->request->data['Attribute'], $attribute['Event']['id'], $capture=true);
+                    $this->Attribute->Event->captureAnalystData($this->Auth->user(), $this->request->data['Attribute'], 'Attribute', $existingAttribute['Attribute']['uuid']);
                 }
                 $this->Attribute->Object->updateTimestamp($existingAttribute['Attribute']['object_id']);
             } else {
                 $result = $this->Attribute->save($this->request->data, array('fieldList' => Attribute::EDITABLE_FIELDS));
                 if ($result) {
                     $this->Attribute->AttributeTag->handleAttributeTags($this->Auth->user(), $this->request->data['Attribute'], $attribute['Event']['id'], $capture=true);
+                    $this->Attribute->Event->captureAnalystData($this->Auth->user(), $this->request->data['Attribute'], 'Attribute', $existingAttribute['Attribute']['uuid']);
                 }
                 if ($this->request->is('ajax')) {
                     $this->autoRender = false;
@@ -1576,10 +1556,11 @@ class AttributesController extends AppController
 
             // Force index for performance reasons see #3321
             if (isset($filters['value'])) {
-                $this->paginate['forceIndexHint'] = '(value1, value2)';
+                $this->paginate['forceIndexHint'] = 'value1, value2';
             }
 
             $this->paginate['conditions'] = $params['conditions'];
+            $this->paginate['ignoreIndexHint'] = 'deleted';
             $attributes = $this->paginate();
             $this->Attribute->attachTagsToAttributes($attributes, ['includeAllTags' => true]);
 
@@ -3038,4 +3019,35 @@ class AttributesController extends AppController
             ];
         }
     }
+
+    public function viewAnalystData($id, $seed = null)
+    {
+        $this->Attribute->includeAnalystDataRecursive = true;
+        $attribute = $this->Attribute->fetchAttributes(
+            $this->Auth->user(),
+            [
+                'conditions' => $this->__idToConditions($id),
+                'flatten' => true
+            ]
+        );
+        if(empty($attribute)) {
+            throw new NotFoundException(__('Invalid Attribute.'));
+        } else {
+            $attribute[0]['Attribute'] = array_merge_recursive($attribute[0]['Attribute'], $this->Attribute->attachAnalystData($attribute[0]['Attribute']));
+        }
+        if ($this->_isRest()) {
+            $validFields = ['Note', 'Opinion', 'Relationship'];
+            $results = [];
+            foreach ($validFields as $field) {
+                if (!empty($attribute[0]['Attribute'][$field])) {
+                    $results[$field] = $attribute[0]['Attribute'][$field];
+                }
+            }
+            return $this->RestResponse->viewData($results, $this->response->type());
+        }
+        $this->layout = null;
+        $this->set('shortDist', $this->Attribute->shortDist);
+        $this->set('object', $attribute[0]['Attribute']);
+        $this->set('seed', $seed);
+    }
 }

app/Controller/BenchmarksController.php

@@ -0,0 +1,123 @@
+<?php
+App::uses('AppController', 'Controller');
+
+class BenchmarksController extends AppController
+{
+    public $components = array('Session', 'RequestHandler');
+
+    public $paginate = [
+        'limit' => 60,
+        'maxLimit' => 9999,
+
+    ];
+
+    public function beforeFilter()
+    {
+        parent::beforeFilter();
+    }
+
+    public function index()
+    {
+        $this->set('menuData', ['menuList' => 'admin', 'menuItem' => 'index']);
+        $this->loadModel('User');
+        App::uses('BenchmarkTool', 'Tools');
+        $this->Benchmark = new BenchmarkTool($this->User);
+        $passedArgs = $this->passedArgs;
+        $this->paginate['order'] = 'value';
+        $defaults = [
+            'days' => null,
+            'average' => false,
+            'aggregate' => false,
+            'scope' => null,
+            'field' => null,
+            'key' => null,
+            'quickFilter' => null
+        ];
+        $filters = $this->IndexFilter->harvestParameters(array_keys($defaults));
+        foreach ($defaults as $key => $value) {
+            if (!isset($filters[$key])) {
+                $filters[$key] = $defaults[$key];
+            }
+        }
+        $temp = $this->Benchmark->getAllTopLists(
+            $filters['days'] ?? null,
+            $filters['limit'] ?? 100,
+            $filters['average'] ?? null,
+            $filters['aggregate'] ?? null
+        );
+        $settings = $this->Benchmark->getSettings();
+        $units = $this->Benchmark->getUnits();
+        $this->set('settings', $settings);
+        $data = [];
+        $userLookup = [];
+        foreach ($temp as $scope => $t) {
+            if (!empty($filters['scope']) && $filters['scope'] !== 'all' && $scope !== $filters['scope']) {
+                continue;
+            }
+            foreach ($t as $field => $t2) {
+                if (!empty($filters['field']) && $filters['field'] !== 'all' && $field !== $filters['field']) {
+                    continue;
+                }
+                foreach ($t2 as $date => $t3) {
+                    foreach ($t3 as $key => $value) {
+                        if ($scope == 'user') {
+                            if ($key === 'SYSTEM') {
+                                $text = 'SYSTEM';
+                            } else if (isset($userLookup[$key])) {
+                                $text = $userLookup[$key];
+                            } else {
+                                $user = $this->User->find('first', [
+                                    'fields' => ['User.id', 'User.email'],
+                                    'recursive' => -1,
+                                    'conditions' => ['User.id' => $key]
+                                ]);
+                                if (empty($user)) {
+                                    $text = '(' . $key . ') ' . __('Invalid user');
+                                } else {
+                                    $text = '(' . $key . ') ' . $user['User']['email'];
+                                }
+                                $userLookup[$key] = $text;
+                            }
+                        } else {
+                            $text = $key;
+                        }
+                        if (!empty($filters['quickFilter'])) {
+                            $q = strtolower($filters['quickFilter']);
+                            if (
+                                strpos(strtolower($scope), $q) === false &&
+                                strpos(strtolower($field), $q) === false &&
+                                strpos(strtolower($key), $q) === false &&
+                                strpos(strtolower($value), $q) === false &&
+                                strpos(strtolower($date), $q) === false &&
+                                strpos(strtolower($text), $q) === false
+                            ) {
+                                continue;
+                            }
+                        }
+                        if (empty($filters['key']) || $key == $filters['key']) {
+                            $data[] = [
+                                'scope' => $scope,
+                                'field' => $field,
+                                'date' => $date,
+                                'key' => $key,
+                                'text' => $text,
+                                'value' => $value,
+                                'unit' => $units[$field]
+                            ];    
+                        }
+                    }
+                }
+            }
+        }
+        if ($this->_isRest()) {
+            return $this->RestResponse->viewData($data, $this->response->type());
+        }
+        App::uses('CustomPaginationTool', 'Tools');
+        $customPagination = new CustomPaginationTool();
+        $customPagination->truncateAndPaginate($data, $this->params, $this->modelClass, true);
+        $this->set('data', $data);
+        $this->set('passedArgs', json_encode($passedArgs));
+        $this->set('filters', $filters);
+    }
+
+}

app/Controller/Component/CRUDComponent.php

@@ -289,6 +289,9 @@ class CRUDComponent extends Component
                 }
             }
         }
+        if (isset($params['afterFind'])) {
+            $data = $params['afterFind']($data);
+        }
         if (isset($params['beforeDelete'])) {
             $data = $params['beforeDelete']($data);
             if (empty($data)) {

app/Controller/Component/CompressedRequestHandlerComponent.php

@@ -37,14 +37,17 @@ class CompressedRequestHandlerComponent extends Component
     private function decodeGzipEncodedContent(Controller $controller)
     {
         if (function_exists('gzdecode')) {
-            $decoded = gzdecode($controller->request->input());
+            $input = $controller->request->input();
+            if (empty($input)) {
+                throw new BadRequestException('Request data should be gzip encoded, but request is empty.');
+            }
+            $decoded = gzdecode($input);
             if ($decoded === false) {
                 throw new BadRequestException('Invalid compressed data.');
             }
             return $decoded;
-        } else {
-            throw new BadRequestException("This server doesn't support GZIP compressed requests.");
         }
+        throw new BadRequestException("This server doesn't support GZIP compressed requests.");
     }
 
     /**

app/Controller/Component/RestResponseComponent.php

@@ -671,9 +671,10 @@ class RestResponseComponent extends Component
         }
 
         if ($response instanceof TmpFileTool) {
-            if (isset($_SERVER['HTTP_IF_NONE_MATCH'])) {
+            $requestEtag = $this->requestEtag();
+            if ($requestEtag !== null) {
                 $etag = '"' . $response->hash('sha1') . '"';
-                if ($_SERVER['HTTP_IF_NONE_MATCH'] === $etag) {
+                if ($requestEtag === $etag) {
                     return new CakeResponse(['status' => 304]);
                 }
                 $headers['ETag'] = $etag;
@@ -689,9 +690,10 @@ class RestResponseComponent extends Component
             }
         } else {
             // Check if resource was changed when `If-None-Match` header is send and return 304 Not Modified
-            if (isset($_SERVER['HTTP_IF_NONE_MATCH'])) {
+            $requestEtag = $this->requestEtag();
+            if ($requestEtag !== null) {
                 $etag = '"' . sha1($response) . '"';
-                if ($_SERVER['HTTP_IF_NONE_MATCH'] === $etag) {
+                if ($requestEtag === $etag) {
                     return new CakeResponse(['status' => 304]);
                 }
                 // Generate etag just when HTTP_IF_NONE_MATCH is set
@@ -724,6 +726,25 @@ class RestResponseComponent extends Component
         return $cakeResponse;
     }
 
+    /**
+     * Return etag from If-None-Match HTTP request header without compression marks added by Apache
+     * @return string|null
+     */
+    private function requestEtag()
+    {
+        if (isset($_SERVER['HTTP_IF_NONE_MATCH'])) {
+            // Remove compression marks that adds Apache for compressed content
+            $requestEtag = $_SERVER['HTTP_IF_NONE_MATCH'];
+            $etagWithoutQuotes = trim($requestEtag, '"');
+            $dashPos = strrpos($etagWithoutQuotes, '-');
+            if ($dashPos && in_array(substr($etagWithoutQuotes, $dashPos + 1), ['br', 'gzip'], true)) {
+                return '"' . substr($etagWithoutQuotes, 0, $dashPos) . '"';
+            }
+            return $requestEtag;
+        }
+        return null;
+    }
+
     /**
      * @param string $response
      * @return string Signature as base64 encoded string

app/Controller/Component/RestSearchComponent.php

@@ -144,7 +144,11 @@ class RestSearchComponent extends Component
             'retry',
             'expiry',
             'minimum_ttl',
-            'ttl'
+            'ttl',
+            'org.sector',
+            'org.local',
+            'org.nationality',
+            'galaxy.*',
         ],
         'Object' => [
             'returnFormat',

app/Controller/EventReportsController.php

@@ -213,10 +213,13 @@ class EventReportsController extends AppController
 
     public function extractAllFromReport($reportId)
     {
-        if (!$this->request->is('ajax')) {
+        if (!$this->request->is('ajax') && !$this->_isRest()) {
             throw new MethodNotAllowedException(__('This function can only be reached via AJAX.'));
         }
         if ($this->request->is('post')) {
+            if (!isset($this->data['EventReport'])) {
+                $this->data = ['EventReport' => $this->data];
+            }
             $report = $this->EventReport->fetchIfAuthorized($this->Auth->user(), $reportId, 'edit', $throwErrors=true, $full=false);
             $results = $this->EventReport->getComplexTypeToolResultWithReplacements($this->Auth->user(), $report);
             $report['EventReport']['content'] = $results['replacementResult']['contentWithReplacements'];
@@ -299,13 +302,16 @@ class EventReportsController extends AppController
 
     public function importReportFromUrl($event_id)
     {
-        if (!$this->request->is('ajax')) {
-            throw new MethodNotAllowedException(__('This function can only be reached via AJAX.'));
+        if (!$this->request->is('ajax') && !$this->_isRest()) {
+            throw new MethodNotAllowedException(__('This function can only be reached via AJAX and via the API.'));
         }
         $fetcherModule = $this->EventReport->isFetchURLModuleEnabled();
         if ($this->request->is('post')) {
+            if (empty($this->data['EventReport'])) {
+                $this->data = ['EventReport' => $this->data];
+            }
             if (empty($this->data['EventReport']['url'])) {
-                throw new MethodNotAllowedException(__('An URL must be provided'));
+                throw new MethodNotAllowedException(__('A URL must be provided'));
             }
             $url = $this->data['EventReport']['url'];
             $format = 'html';
@@ -316,7 +322,6 @@ class EventReportsController extends AppController
                     $format = $parsed_format;
                 }
             }
-            
             $content = $this->EventReport->downloadMarkdownFromURL($event_id, $url, $format);
 
             $errors = [];
@@ -563,6 +568,12 @@ class EventReportsController extends AppController
                 $savedReport['EventReport'][$field] = $newReport['EventReport'][$field];
             }
         }
+        $this->loadModel('AnalystData');
+        foreach ($this->AnalystData::ANALYST_DATA_TYPES as $type) {
+            if (!empty($newReport['EventReport'][$type])) {
+                $savedReport['EventReport'][$type] = $newReport['EventReport'][$type];
+            }
+        }
         return $savedReport;
     }
 }

app/Controller/EventsController.php

@@ -14,16 +14,16 @@ class EventsController extends AppController
     );
 
     public $paginate = array(
-            'limit' => 60,
-            'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page.
-            'order' => array(
-                    'Event.timestamp' => 'DESC'
-            ),
-            'contain' => array(
-                    'Org' => array('fields' => array('id', 'name', 'uuid')),
-                    'Orgc' => array('fields' => array('id', 'name', 'uuid')),
-                    'SharingGroup' => array('fields' => array('id', 'name', 'uuid'))
-            )
+        'limit' => 60,
+        'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page.
+        'order' => array(
+            'Event.timestamp' => 'DESC'
+        ),
+        'contain' => array(
+            'Org' => array('fields' => array('id', 'name', 'uuid')),
+            'Orgc' => array('fields' => array('id', 'name', 'uuid')),
+            'SharingGroup' => array('fields' => array('id', 'name', 'uuid'))
+        )
     );
 
     // private
@@ -61,46 +61,14 @@ class EventsController extends AppController
         'publish_timestamp'
     ];
 
-    public $paginationFunctions = array('index', 'proposalEventIndex');
-
     public function beforeFilter()
     {
         parent::beforeFilter();
 
-        // what pages are allowed for non-logged-in users
-        $this->Auth->allow('xml');
-        $this->Auth->allow('csv');
-        $this->Auth->allow('nids');
-        $this->Auth->allow('hids_md5');
-        $this->Auth->allow('hids_sha1');
-        $this->Auth->allow('text');
-        $this->Auth->allow('restSearch');
-        $this->Auth->allow('stix');
-        $this->Auth->allow('stix2');
-
         $this->Security->unlockedActions[] = 'viewEventAttributes';
 
-        // TODO Audit, activate logable in a Controller
-        if (count($this->uses) && $this->{$this->modelClass}->Behaviors->attached('SysLogLogable')) {
-            $this->{$this->modelClass}->setUserData($this->activeUser);
-        }
-
-        // convert uuid to id if present in the url, and overwrite id field
-        if (isset($this->request->params->query['uuid'])) {
-            $params = array(
-                    'conditions' => array('Event.uuid' => $this->params->query['uuid']),
-                    'recursive' => 0,
-                    'fields' => 'Event.id'
-            );
-            $result = $this->Event->find('first', $params);
-            if (isset($result['Event']) && isset($result['Event']['id'])) {
-                $id = $result['Event']['id'];
-                $this->params->addParams(array('pass' => array($id))); // FIXME find better way to change id variable if uuid is found. params->url and params->here is not modified accordingly now
-            }
-        }
-
         // if not admin or own org, check private as well..
-        if (!$this->_isSiteAdmin() && in_array($this->request->action, $this->paginationFunctions, true)) {
+        if (!$this->_isSiteAdmin() && in_array($this->request->action, ['index', 'proposalEventIndex'], true)) {
             $conditions = $this->Event->createEventConditions($this->Auth->user());
             if ($this->userRole['perm_sync'] && $this->Auth->user('Server')['push_rules']) {
                 $conditions['AND'][] = $this->Event->filterRulesToConditions($this->Auth->user('Server')['push_rules']);
@@ -497,6 +465,11 @@ class EventsController extends AppController
                         continue 2;
                     }
                     $pieces = is_array($v) ? $v : explode('|', $v);
+                    $isANDed = false;
+                    if (count($pieces) == 1 && strpos($pieces[0], '&') !== false) {
+                        $pieces = explode('&', $pieces[0]);
+                        $isANDed = count($pieces) > 1;
+                    }
                     $filterString = "";
                     $expectOR = false;
                     $tagRules = [];
@@ -563,10 +536,19 @@ class EventsController extends AppController
                     }
 
                     if (!empty($tagRules['include'])) {
-                        $include = $this->Event->EventTag->find('column', array(
-                            'conditions' => array('EventTag.tag_id' => $tagRules['include']),
-                            'fields' => ['EventTag.event_id'],
-                        ));
+                        if ($isANDed) {
+                            $include = $this->Event->EventTag->find('column', array(
+                                'conditions' => ['EventTag.tag_id' => $tagRules['include']],
+                                'fields' => ['EventTag.event_id'],
+                                'group' => ['EventTag.event_id'],
+                                'having' => ['COUNT(*) =' => count($tagRules['include'])],
+                            ));
+                        } else {
+                            $include = $this->Event->EventTag->find('column', array(
+                                'conditions' => array('EventTag.tag_id' => $tagRules['include']),
+                                'fields' => ['EventTag.event_id'],
+                            ));
+                        }
                         if (!empty($include)) {
                             $this->paginate['conditions']['AND'][] = 'Event.id IN (' . implode(",", $include) . ')';
                         } else {
@@ -755,8 +737,8 @@ class EventsController extends AppController
         if ($nothing) {
             $this->paginate['conditions']['AND'][] = ['Event.id' => -1]; // do not fetch any event
         }
-        $this->Event->includeAnalystData = true;
-        $this->paginate['includeAnalystData'] = true;
+        $this->Event->includeAnalystData = isset($passedArgs['includeAnalystData']) ? $passedArgs['includeAnalystData'] : false;
+        $this->paginate['includeAnalystData'] = isset($passedArgs['includeAnalystData']) ? $passedArgs['includeAnalystData'] : false;
         $events = $this->paginate();
 
         if (count($events) === 1 && isset($this->passedArgs['searchall'])) {
@@ -812,7 +794,7 @@ class EventsController extends AppController
             $rules = [
                 'contain' => ['EventTag'],
                 'fields' => array_keys($fieldNames),
-                'includeAnalystData' => isset($passedArgs['includeAnalystData']) ? $passedArgs['includeAnalystData'] : true,
+                'includeAnalystData' => isset($passedArgs['includeAnalystData']) ? $passedArgs['includeAnalystData'] : false,
             ];
         }
         if (isset($passedArgs['sort']) && isset($fieldNames[$passedArgs['sort']])) {
@@ -1416,7 +1398,8 @@ class EventsController extends AppController
         $exception = false;
         $warningTagConflicts = array();
         $filters = $this->_harvestParameters($filterData, $exception);
-
+        $analystData = $this->Event->attachAnalystData($event['Event']);
+        $event['Event'] = array_merge($event['Event'], $analystData); 
         $emptyEvent = (empty($event['Object']) && empty($event['Attribute']));
         $this->set('emptyEvent', $emptyEvent);
 
@@ -1490,7 +1473,6 @@ class EventsController extends AppController
                 $containsProposals = true;
             }
         }
-
         foreach ($event['Object'] as $k => $object) {
             $modDate = date("Y-m-d", $object['timestamp']);
             $modificationMap[$modDate] = !isset($modificationMap[$modDate])? 1 : $modificationMap[$modDate] + 1;
@@ -1522,7 +1504,6 @@ class EventsController extends AppController
                 }
             }
         }
-
         if ($containsProposals && $this->__canPublishEvent($event, $user)) {
             $mess = $this->Session->read('Message');
             if (empty($mess)) {
@@ -1696,8 +1677,8 @@ class EventsController extends AppController
         }
 
         $namedParams = $this->request->params['named'];
-        $conditions['includeAnalystData'] = true;
         if ($this->_isRest()) {
+            $conditions['includeAnalystData'] = true;
             $conditions['includeAttachments'] = isset($namedParams['includeAttachments']) ? $namedParams['includeAttachments'] : true;
         } else {
             $conditions['includeAllTags'] = true;
@@ -2369,7 +2350,13 @@ class EventsController extends AppController
                     }
 
                     $isXml = $ext === 'xml';
-                    $data = FileAccessTool::readFromFile($file['tmp_name'], $file['size']);
+                    $matches = null;
+                    $tmp_name = $file['tmp_name'];
+                    if (preg_match_all('/[\w\/\-\.]*/', $tmp_name, $matches) && file_exists($file['tmp_name'])) {
+                        $data = FileAccessTool::readFromFile($matches[0][0], $file['size']);
+                    } else {
+                        throw new NotFoundException(__('Invalid file.'));    
+                    }
                 } else {
                     throw new MethodNotAllowedException(__('No file uploaded.'));
                 }
@@ -2378,7 +2365,6 @@ class EventsController extends AppController
                     && (isset($this->request->data['Event']['takeownership']) && $this->request->data['Event']['takeownership'] == 1);
 
                 $publish = $this->request->data['Event']['publish'] ?? false;
-
                 try {
                     $results = $this->Event->addMISPExportFile($this->Auth->user(), $data, $isXml, $takeOwnership, $publish);
                 } catch (Exception $e) {
@@ -2409,7 +2395,7 @@ class EventsController extends AppController
                 if (isset($this->params['named']['distribution'])) {
                     $distribution = intval($this->params['named']['distribution']);
                     if (!array_key_exists($distribution, $distributionLevels)) {
-                        throw new MethodNotAllowedException(__('Wrong distribution level'));
+                        throw new BadRequestException(__('Wrong distribution level'));
                     }
                 } else {
                     $distribution = $initialDistribution;
@@ -2417,11 +2403,11 @@ class EventsController extends AppController
                 $sharingGroupId = null;
                 if ($distribution == 4) {
                     if (!isset($this->params['named']['sharing_group_id'])) {
-                        throw new MethodNotAllowedException(__('The sharing group id is needed when the distribution is set to 4 ("Sharing group").'));
+                        throw new BadRequestException(__('The sharing group id is needed when the distribution is set to 4 ("Sharing group").'));
                     }
                     $sharingGroupId = intval($this->params['named']['sharing_group_id']);
                     if (!array_key_exists($sharingGroupId, $sgs)) {
-                        throw new MethodNotAllowedException(__('Please select a valid sharing group id.'));
+                        throw new BadRequestException(__('Please select a valid sharing group id.'));
                     }
                 }
                 $clusterDistribution = $initialDistribution;
@@ -2431,15 +2417,15 @@ class EventsController extends AppController
                     if (isset($this->params['name']['cluster_distribution'])) {
                         $clusterDistribution = intval($this->params['named']['cluster_distribution']);
                         if (!array_key_exists($clusterDistribution, $distributionLevels)) {
-                            throw new MethodNotAllowedException(__('Wrong cluster distribution level'));
+                            throw new BadRequestException(__('Wrong cluster distribution level'));
                         }
                         if ($clusterDistribution == 4) {
                             if (!isset($this->params['named']['cluster_sharing_group_id'])) {
-                                throw new MethodNotAllowedException(__('The cluster sharing group id is needed when the cluster distribution is set to 4 ("Sharing group").'));
+                                throw new BadRequestException(__('The cluster sharing group id is needed when the cluster distribution is set to 4 ("Sharing group").'));
                             }
                             $clusterSharingGroupId = intval($this->params['named']['cluster_sharing_group_id']);
                             if (!array_key_exists($clusterSharingGroupId, $sgs)) {
-                                throw new MethodNotAllowedException(__('Please select a valid cluster sharing group id.'));
+                                throw new BadRequestException(__('Please select a valid cluster sharing group id.'));
                             }
                         }
                     }
@@ -2471,8 +2457,8 @@ class EventsController extends AppController
                 } else {
                     return $this->RestResponse->saveFailResponse('Events', 'upload_stix', false, $result, $this->response->type());
                 }
-            } else {
-                $original_file = !empty($this->data['Event']['original_file']) ? $this->data['Event']['stix']['name'] : '';
+            } else { // not REST request
+                $originalFile = !empty($this->data['Event']['original_file']) ? $this->data['Event']['stix']['name'] : '';
                 if (isset($this->data['Event']['stix']) && $this->data['Event']['stix']['size'] > 0 && is_uploaded_file($this->data['Event']['stix']['tmp_name'])) {
                     $filePath = FileAccessTool::createTempFile();
                     if (!move_uploaded_file($this->data['Event']['stix']['tmp_name'], $filePath)) {
@@ -2485,12 +2471,12 @@ class EventsController extends AppController
                         $this->Auth->user(),
                         $filePath,
                         $stix_version,
-                        $original_file,
+                        $originalFile,
                         $this->data['Event']['publish'],
                         $this->data['Event']['distribution'],
                         $this->data['Event']['sharing_group_id'] ?? null,
-                        $this->data['Event']['galaxies_handling'],
-                        $this->data['Event']['cluster_distribution'],
+                        $this->data['Event']['galaxies_handling'] ?? false,
+                        $this->data['Event']['cluster_distribution'] ?? 0,
                         $this->data['Event']['cluster_sharing_group_id'] ?? null,
                         $debug
                     );
@@ -3107,9 +3093,9 @@ class EventsController extends AppController
                         $errors['Module'] = 'Module failure.';
                     }
                 } else {
+                    $errors['failed_servers'] = $result;
                     $lastResult = array_pop($result);
                     $resultString = (count($result) > 0) ? implode(', ', $result) . ' and ' . $lastResult : $lastResult;
-                    $errors['failed_servers'] = $result;
                     $message = __('Event published but not pushed to %s, re-try later. If the issue persists, make sure that the correct sync user credentials are used for the server link and that the sync user on the remote server has authentication privileges.', $resultString);
                 }
             } else {
@@ -4156,7 +4142,13 @@ class EventsController extends AppController
                 }
             }
             $this->Event->Attribute->fetchRelated($this->Auth->user(), $resultArray);
-            $typeCategoryMapping = array();
+            $typeCategoryMapping = [
+                'ip-src/ip-dst' => [
+                    'Network activity' => 'Network activity',
+                    'Payload delivery' => 'Payload delivery',
+                    'External analysis' => 'External analysis'
+                ],
+            ];
             foreach ($this->Event->Attribute->categoryDefinitions as $k => $cat) {
                 foreach ($cat['types'] as $type) {
                     $typeCategoryMapping[$type][$k] = $k;
@@ -4857,16 +4849,18 @@ class EventsController extends AppController
     public function updateGraph($id, $type = 'event')
     {
         $user = $this->_closeSession();
+
         $validTools = array('event', 'galaxy', 'tag');
         if (!in_array($type, $validTools, true)) {
             throw new MethodNotAllowedException(__('Invalid type.'));
         }
+
         $this->loadModel('Taxonomy');
         $this->loadModel('GalaxyCluster');
         App::uses('CorrelationGraphTool', 'Tools');
-        $grapher = new CorrelationGraphTool();
+
         $data = $this->request->is('post') ? $this->request->data : array();
-        $grapher->construct($this->Event, $this->Taxonomy, $this->GalaxyCluster, $user, $data);
+        $grapher = new CorrelationGraphTool($this->Event, $this->Taxonomy, $this->GalaxyCluster, $user, $data);
         $json = $grapher->buildGraphJson($id, $type);
         array_walk_recursive($json, function (&$item, $key) {
             if (!mb_detect_encoding($item, 'utf-8', true)) {

app/Controller/FeedsController.php

@@ -74,6 +74,8 @@ class FeedsController extends AppController
                 );
             }
         }
+        $loggedUser = $this->Auth->user();
+        $this->loadModel('TagCollection');
 
         $this->CRUD->index([
             'filters' => [
@@ -92,7 +94,7 @@ class FeedsController extends AppController
                 'source_format'
             ],
             'conditions' => $conditions,
-            'afterFind' => function (array $feeds) {
+            'afterFind' => function (array $feeds) use ($loggedUser) {
                 if ($this->_isSiteAdmin()) {
                     $feeds = $this->Feed->attachFeedCacheTimestamps($feeds);
                 }
@@ -106,6 +108,19 @@ class FeedsController extends AppController
                     }
                 }
 
+                foreach ($feeds as &$feed) {
+                    if (!empty($feed['Feed']['tag_collection_id'])) {
+                        $tagCollection = $this->TagCollection->fetchTagCollection($loggedUser, [
+                            'conditions' => [
+                                'TagCollection.id' => $feed['Feed']['tag_collection_id'],
+                            ]
+                        ]);
+                        if (!empty($tagCollection)) {
+                            $feed['TagCollection'] = $tagCollection;
+                        }
+                    }
+                }
+
                 return $feeds;
             }
         ]);
@@ -294,6 +309,10 @@ class FeedsController extends AppController
         }
         $tags = $this->Event->EventTag->Tag->find('list', array('fields' => array('Tag.name'), 'order' => array('lower(Tag.name) asc')));
         $tags[0] = 'None';
+        $this->loadModel('TagCollection');
+        $tagCollections = $this->TagCollection->fetchTagCollection($this->Auth->user());
+        $tagCollections = Hash::combine($tagCollections, '{n}.TagCollection.id', '{n}.TagCollection.name');
+        $tagCollections[0] = 'None';
 
         $this->loadModel('Server');
         $allTypes = $this->Server->getAllTypes();
@@ -304,6 +323,7 @@ class FeedsController extends AppController
                 'order' => 'LOWER(name)'
             )),
             'tags' => $tags,
+            'tag_collections' => $tagCollections,
             'feedTypes' => $this->Feed->getFeedTypesOptions(),
             'sharingGroups' => $sharingGroups,
             'distributionLevels' => $distributionLevels,
@@ -340,6 +360,7 @@ class FeedsController extends AppController
                 'distribution',
                 'sharing_group_id',
                 'tag_id',
+                'tag_collection_id',
                 'event_id',
                 'publish',
                 'delta_merge',
@@ -442,8 +463,17 @@ class FeedsController extends AppController
         if (empty(Configure::read('Security.disable_local_feed_access'))) {
             $inputSources['local'] = 'Local';
         }
+        $tags = $this->Event->EventTag->Tag->find('all', [
+            'recursive' => -1,
+            'fields' => ['Tag.name', 'Tag.id'],
+            'order' => ['lower(Tag.name) asc']
+        ]);
         $tags = $this->Event->EventTag->Tag->find('list', array('fields' => array('Tag.name'), 'order' => array('lower(Tag.name) asc')));
         $tags[0] = 'None';
+        $this->loadModel('TagCollection');
+        $tagCollections = $this->TagCollection->fetchTagCollection($this->Auth->user());
+        $tagCollections = Hash::combine($tagCollections, '{n}.TagCollection.id', '{n}.TagCollection.name');
+        $tagCollections[0] = 'None';
 
         $this->loadModel('Server');
         $allTypes = $this->Server->getAllTypes();
@@ -457,6 +487,7 @@ class FeedsController extends AppController
                 'order' => 'LOWER(name)'
             )),
             'tags' => $tags,
+            'tag_collections' => $tagCollections,
             'feedTypes' => $this->Feed->getFeedTypesOptions(),
             'sharingGroups' => $sharingGroups,
             'distributionLevels' => $distributionLevels,

app/Controller/GalaxiesController.php

@@ -202,7 +202,7 @@ class GalaxiesController extends AppController
         }
         $result = $this->Galaxy->save($galaxy);
         if ($result) {
-            $message = __('Galaxy enabled');
+            $message = __('Galaxy %s', $enabled ? __('enabled') : __('disabled'));
             if ($this->_isRest()) {
                 return $this->RestResponse->saveSuccessResponse('Galaxy', 'toggle', false, $this->response->type(), $message);
             } else {

app/Controller/GalaxyClustersController.php

@@ -59,6 +59,8 @@ class GalaxyClustersController extends AppController
             $contextConditions['GalaxyCluster.default'] = true;
         } elseif ($filters['context'] == 'custom') {
             $contextConditions['GalaxyCluster.default'] = false;
+        } elseif ($filters['context'] == 'orgc') {
+            $contextConditions['GalaxyCluster.orgc_id'] = $this->Auth->user('org_id');
         } elseif ($filters['context'] == 'org') {
             $contextConditions['GalaxyCluster.org_id'] = $this->Auth->user('org_id');
         } elseif ($filters['context'] == 'deleted') {

app/Controller/LogsController.php

@@ -117,9 +117,15 @@ class LogsController extends AppController
             $this->paginate['conditions']['Log.action'] = $validFilters[$this->params['named']['filter']]['values'];
         }
         foreach ($filters as $key => $value) {
+            if ($key == 'page' || $key == 'limit') { // These should not be part of the condition parameter
+                continue;
+            }
             if ($key === 'created') {
                 $key = 'created >=';
             }
+            if ($key == 'page' || $key == 'limit') {
+                continue;
+            }
             $this->paginate['conditions']["Log.$key"] = $value;
         }
         $this->set('validFilters', $validFilters);

app/Controller/ObjectsController.php

@@ -1410,4 +1410,34 @@ class ObjectsController extends AppController
         }
         return $conditions;
     }
+
+    public function viewAnalystData($id, $seed = null)
+    {
+        $this->MispObject->includeAnalystDataRecursive = true;
+        $object = $this->MispObject->fetchObjects(
+            $this->Auth->user(),
+            [
+                'conditions' => $this->__objectIdToConditions($id)
+            ]
+        );
+        if(empty($object)) {
+            throw new NotFoundException(__('Invalid Object.'));
+        } else {
+            $object[0]['Object'] = array_merge_recursive($object[0]['Object'], $this->MispObject->attachAnalystData($object[0]['Object']));
+        }
+        if ($this->_isRest()) {
+            $validFields = ['Note', 'Opinion', 'Relationship'];
+            $results = [];
+            foreach ($validFields as $field) {
+                if (!empty($object[0]['Object'][$field])) {
+                    $results[$field] = $object[0]['Object'][$field];
+                }
+            }
+            return $this->RestResponse->viewData($results, $this->response->type());
+        }
+        $this->layout = null;
+        $this->set('shortDist', $this->MispObject->Attribute->shortDist);
+        $this->set('object', $object[0]['Object']);
+        $this->set('seed', $seed);
+    }
 }

app/Controller/OrganisationsController.php

@@ -490,8 +490,14 @@ class OrganisationsController extends AppController
                 $this->Flash->error(__('Invalid file extension, Only PNG and SVG images are allowed.'));
                 return false;
             }
-
-            $imgMime = mime_content_type($logo['tmp_name']);
+            $matches = null;
+            $tmp_name = $logo['tmp_name'];
+            if (preg_match_all('/[\w\/\-\.]*/', $tmp_name, $matches) && file_exists($logo['tmp_name'])) {
+                $tmp_name = $matches[0][0];
+                $imgMime = mime_content_type($tmp_name);
+            } else {
+                throw new NotFoundException(__('Invalid file.'));    
+            }
             if ($extension === 'png' && (function_exists('exif_imagetype') && !exif_imagetype($logo['tmp_name']))) {
                 $this->Flash->error(__('This is not a valid PNG image.'));
                 return false;
@@ -507,8 +513,8 @@ class OrganisationsController extends AppController
                 return false;
             }
 
-            if (!empty($logo['tmp_name']) && is_uploaded_file($logo['tmp_name'])) {
-                return move_uploaded_file($logo['tmp_name'], APP . 'files/img/orgs/' . $filename);
+            if (!empty($tmp_name) && is_uploaded_file($tmp_name)) {
+                return move_uploaded_file($tmp_name, APP . 'files/img/orgs/' . $filename);
             }
         }
 

app/Controller/ServersController.php

@@ -2084,7 +2084,10 @@ class ServersController extends AppController
 
     public function updateJSON()
     {
-        $results = $this->Server->updateJSON();
+        $results = [];
+        foreach ($this->Server->updateJSON() as $type => $result) {
+            $results[$type] = $results['success'];
+        }
         return $this->RestResponse->viewData($results, $this->response->type());
     }
 

app/Controller/SightingBlocklistsController.php

@@ -0,0 +1,47 @@
+<?php
+App::uses('AppController', 'Controller');
+
+class SightingBlocklistsController extends AppController
+{
+    public $components = array('Session', 'RequestHandler', 'BlockList');
+
+    public function beforeFilter()
+    {
+        parent::beforeFilter();
+        if (!$this->_isSiteAdmin()) {
+            $this->redirect('/');
+        }
+        if (Configure::check('MISP.enableSightingBlocklisting') && !Configure::read('MISP.enableSightingBlocklisting') !== false) {
+            $this->Flash->info(__('Sighting BlockListing is not currently enabled on this instance.'));
+            $this->redirect('/');
+        }
+    }
+
+    public $paginate = array(
+            'limit' => 60,
+            'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page.
+            'order' => array(
+                    'SightingBlocklist.created' => 'DESC'
+            ),
+    );
+
+    public function index()
+    {
+        return $this->BlockList->index($this->_isRest());
+    }
+
+    public function add()
+    {
+        return $this->BlockList->add($this->_isRest());
+    }
+
+    public function edit($id)
+    {
+        return $this->BlockList->edit($this->_isRest(), $id);
+    }
+
+    public function delete($id)
+    {
+        return $this->BlockList->delete($this->_isRest(), $id);
+    }
+}

app/Controller/TagCollectionsController.php

@@ -357,7 +357,7 @@ class TagCollectionsController extends AppController
             if (!$tagCollection) {
                 throw new NotFoundException(__('Invalid tag collection.'));
             }
-            if ($this->ACL->canModifyTagCollection($this->Auth->user(), $tagCollection)) {
+            if (!$this->ACL->canModifyTagCollection($this->Auth->user(), $tagCollection)) {
                 throw new ForbiddenException(__('You dont have a permission to do that'));
             }
             $tagCollectionTag = $this->TagCollection->TagCollectionTag->find('first', [

app/Controller/TagsController.php

@@ -820,6 +820,15 @@ class TagsController extends AppController
                     }
                     $message = 'Global tag ' . $existingTag['Tag']['name'] . '(' . $existingTag['Tag']['id'] . ') successfully attached to ' . $objectType . '(' . $object[$objectType]['id'] . ').';
                 }
+                $this->loadModel('Log');
+                $this->Log->createLogEntry(
+                    $this->Auth->user(),
+                    'attachTagToObject',
+                    $objectType,
+                    $object[$objectType]['id'],
+                    $message,
+                    null
+                );
                 $successes++;
             } else {
                 $fails[] = __('Failed to attach tag to object.');
@@ -896,6 +905,17 @@ class TagsController extends AppController
         $result = $this->$objectType->$connectorObject->delete($existingAssociation[$connectorObject]['id']);
         if ($result) {
             $message = __('%s tag %s (%s) successfully removed from %s(%s).', $local ? __('Local') : __('Global'), $existingTag['Tag']['name'], $existingTag['Tag']['id'], $objectType, $object[$objectType]['id']);
+            $this->loadModel('Log');
+            $this->Log->createLogEntry(
+                $this->Auth->user(),
+                'removeTagFromObject',
+                $objectType,
+                $object[$objectType]['id'],
+                $message,
+                __(
+                    '',
+                )
+            );
             if (!$local) {
                 if ($objectType === 'Attribute') {
                     $this->Attribute->touch($object['Attribute']['id']);

app/Controller/UsersController.php

@@ -1,5 +1,5 @@
 <?php
-App::uses('AppController', 'Controller', 'OTPHP\TOTP');
+App::uses('AppController', 'Controller');
 
 /**
  * @property User $User
@@ -1214,11 +1214,13 @@ class UsersController extends AppController
                     $this->Auth->constructAuthenticate();
                 }
                 // user has TOTP token, check creds and redirect to TOTP validation
-                if (!empty($unauth_user['User']['totp']) && !$unauth_user['User']['disabled'] && class_exists('\OTPHP\TOTP')) {
-                    $user = $this->Auth->identify($this->request, $this->response);
-                    if ($user && !$user['disabled']) {
-                        $this->Session->write('otp_user', $user);
-                        return $this->redirect('otp');
+                if (!Configure::read('Security.otp_disabled')) {
+                    if (!empty($unauth_user['User']['totp']) && !$unauth_user['User']['disabled'] && class_exists('\OTPHP\TOTP')) {
+                        $user = $this->Auth->identify($this->request, $this->response);
+                        if ($user && !$user['disabled']) {
+                            $this->Session->write('otp_user', $user);
+                            return $this->redirect('otp');
+                        }
                     }
                 }
             }
@@ -1811,6 +1813,7 @@ class UsersController extends AppController
             $this->Flash->error(__("The required PHP libraries to support TOTP are not installed. Please contact your administrator to address this."));
             $this->redirect($this->referer());
         }
+
         // only allow the users themselves to generate a TOTP secret.
         // If TOTP is enforced they will be invited to generate it at first login
         $user = $this->User->find('first', array(
@@ -1880,8 +1883,9 @@ class UsersController extends AppController
         $this->set('secret', $secret);
     }
 
-    public function totp_delete($id) {
-        if ($this->request->is('post') || $this->request->is('delete')) {
+    public function totp_delete($id)
+    {
+        if ($this->request->is(['post', 'delete'])) {
             $user = $this->User->find('first', array(
                 'conditions' => $this->__adminFetchConditions($id),
                 'recursive' => -1,
@@ -1988,8 +1992,7 @@ class UsersController extends AppController
     // shows some statistics about the instance
     public function statistics($page = 'data')
     {
-        $user = $this->Auth->user();
-        @session_write_close(); // loading this page can take long time, so we can close session
+        $user = $this->_closeSession(true); // loading this page can take long time, so we can close session
 
         if (!$this->_isRest()) {
             $pages = [
@@ -2071,7 +2074,7 @@ class UsersController extends AppController
 
         $stats['attribute_count'] = $this->User->Event->Attribute->find('count', array('conditions' => array('Attribute.deleted' => 0), 'recursive' => -1));
         $stats['attribute_count_month'] = $this->User->Event->Attribute->find('count', array('conditions' => array('Attribute.timestamp >' => $this_month, 'Attribute.deleted' => 0), 'recursive' => -1));
-        $stats['attributes_per_event'] = round($stats['attribute_count'] / $stats['event_count']);
+        $stats['attributes_per_event'] = $stats['event_count'] != 0 ? round($stats['attribute_count'] / $stats['event_count']) : 0;
 
         $stats['correlation_count'] = $this->User->Event->Attribute->Correlation->find('count', array('recursive' => -1));
 
@@ -2082,7 +2085,7 @@ class UsersController extends AppController
         $stats['org_count'] = count($orgs);
         $stats['local_org_count'] = $local_orgs_count;
         $stats['contributing_org_count'] = $this->User->Event->find('count', array('recursive' => -1, 'group' => array('Event.orgc_id')));
-        $stats['average_user_per_org'] = round($stats['user_count'] / $stats['local_org_count'], 1);
+        $stats['average_user_per_org'] = $stats['local_org_count'] != 0 ?  round($stats['user_count'] / $stats['local_org_count'], 1) : 0;
 
         $this->loadModel('Thread');
         $stats['thread_count'] = $this->Thread->find('count', array('conditions' => array('Thread.post_count >' => 0), 'recursive' => -1));
@@ -2603,6 +2606,7 @@ class UsersController extends AppController
                 'org_name',
                 'org_uuid',
                 'message',
+                'pgp',
                 'custom_perms',
                 'perm_sync',
                 'perm_publish',
@@ -3186,10 +3190,6 @@ class UsersController extends AppController
 
     public function forgot()
     {
-        if (empty(Configure::read('Security.allow_password_forgotten'))) {
-            $this->Flash->error(__('This feature is disabled.'));
-            $this->redirect('/');
-        }
         if (!empty($this->Auth->user()) && !$this->_isRest()) {
             $this->Flash->info(__('You are already logged in, no need to ask for a password reset. Log out first.'));
             $this->redirect('/');
@@ -3215,10 +3215,6 @@ class UsersController extends AppController
 
     public function password_reset($token)
     {
-        if (empty(Configure::read('Security.allow_password_forgotten'))) {
-            $this->Flash->error(__('This feature is disabled.'));
-            $this->redirect('/');
-        }
         $this->loadModel('Server');
         $this->set('complexity', !empty(Configure::read('Security.password_policy_complexity')) ? Configure::read('Security.password_policy_complexity') : $this->Server->serverSettings['Security']['password_policy_complexity']['value']);
         $this->set('length', !empty(Configure::read('Security.password_policy_length')) ? Configure::read('Security.password_policy_length') : $this->Server->serverSettings['Security']['password_policy_length']['value']);
@@ -3235,7 +3231,7 @@ class UsersController extends AppController
                 $this->redirect('/');
             }
         }
-        if ($this->request->is('post') || $this->request->is('put')) {
+        if ($this->request->is(['post', 'put'])) {
             $abortPost = false;
             return $this->__pw_change(['User' => $user], 'password_reset', $abortPost, $token, true);
         }

app/Error/AppExceptionRenderer.php

@@ -0,0 +1,77 @@
+<?php
+
+App::uses('ExceptionRenderer', 'Error');
+
+class AppExceptionRenderer extends ExceptionRenderer {
+
+    public function __construct($exception) {
+        $this->controller = $this->_getController($exception);
+
+        if (method_exists($this->controller, 'appError')) {
+            $this->controller->appError($exception);
+            return;
+        }
+        $method = $template = Inflector::variable(str_replace('Exception', '', get_class($exception)));
+        $code = $exception->getCode();
+
+        $methodExists = method_exists($this, $method);
+        if ($exception instanceof CakeException && !$methodExists) {
+            $this->_customErrorLogging($exception);
+            $method = '_cakeError';
+            if (empty($template) || $template === 'internalError') {
+                $template = 'error500';
+            }
+        } elseif ($exception instanceof PDOException) {
+            $method = 'pdoError';
+            $template = 'pdo_error';
+            $code = 500;
+        } elseif (!$methodExists) {
+            $method = 'error500';
+            if ($code >= 400 && $code < 500) {
+                $method = 'error400';
+            }
+        }
+
+        $isNotDebug = !Configure::read('debug');
+        if ($isNotDebug && $method === '_cakeError') {
+            $method = 'error400';
+        }
+        if ($isNotDebug && $code == 500) {
+            $method = 'error500';
+        }
+        $this->template = $template;
+        $this->method = $method;
+        $this->error = $exception;
+    }
+
+    protected function _customErrorLogging($exception): bool
+    {
+        $message = $exception->getMessage();
+        $errorDetection = [
+            'Maximum execution time of' => 'timeout',
+            'Allowed memory size of' => 'out of memory'
+        ];
+        $user = $this->controller->Auth->user();
+        $ua = env('HTTP_USER_AGENT');
+        $source = $this->controller->IndexFilter->isRest() ? 'API' : 'web UI';
+        if (strpos($ua, 'MISP ') !== false) {
+            $source = 'MISP sync';
+        }
+        foreach ($errorDetection as $search => $errorName) {
+            if (strpos($message, $search) !== false) {
+                $logMessage = sprintf(
+                    '%s %s error triggered by User %s (%s) via the %s on %s.',
+                    date('Y-m-d H:i:s'),
+                    $errorName,
+                    $user['id'],
+                    $user['email'],
+                    $source,
+                    $this->controller->request->here()
+                );
+                file_put_contents(LOGS . 'fatal_error.log', $logMessage . PHP_EOL, FILE_APPEND);
+                return true;
+            }
+        }
+        return true;
+    }
+}

app/Lib/Dashboard/AchievementsWidget.php

@@ -191,4 +191,3 @@ class AchievementsWidget
         return $result;
     }
 }
-?>

app/Lib/Dashboard/AttackWidget.php

@@ -35,4 +35,3 @@ class AttackWidget
         return $data;
     }
 }
-?>

app/Lib/Dashboard/BenchmarkTopListWidget.php

@@ -0,0 +1,115 @@
+<?php
+
+
+class BenchmarkTopListWidget
+{
+    public $title = 'Benchmark top list';
+    public $render = 'MultiLineChart';
+    public $width = 3;
+    public $height = 3;
+    public $description = 'A graph showing the top list for a given scope and field in the captured metrics.';
+    public $cacheLifetime = false;
+    public $autoRefreshDelay = 30;
+    public $params = array(
+        'days' => 'Number of days to consider for the graph. There will be a data entry for each day (assuming the benchmarking has been enabled). Defaults to returning all data.',
+        'weeks' => 'Number of weeks to consider for the graph. There will be a data entry for each day (assuming the benchmarking has been enabled). Defaults to returning all data.',
+        'months' => 'Number of months to consider for the graph. There will be a data entry for each day (assuming the benchmarking has been enabled). Defaults to returning all data.',
+        'scope' => 'The scope of the benchmarking refers to what was being tracked. The following scopes are valid: user, endpoint, user_agent',
+        'field' => 'The individual metric to be queried from the benchmark results. Valid values are: time, sql_time, sql_queries, memory, endpoint',
+        'average' => 'If you wish to view the averages per scope/field, set this variable to true. It will divide the result by the number of executions recorded for the scope/field combination for the given day.'
+    );
+    public $Benchmark;
+    public $User;
+
+    public $placeholder =
+        '{
+    "days": "30",
+    "scope": "endpoints",
+    "field": "sql_time"
+}';
+
+    public function handler($user, $options = array())
+    {
+        $this->User = ClassRegistry::init('User');
+        $currentTime = strtotime("now");
+        $endOfDay = strtotime("tomorrow", $currentTime) - 1;
+        if (!empty($options['days'])) {
+            $limit = (int)($options['days']);
+            $delta = 'day';
+        } else if (!empty($options['weeks'])) {
+            $limit = (int)($options['weeks']);
+            $delta = 'week';
+        } else if (!empty($options['months'])) {
+            $limit = (int)($options['months']);
+            $delta = 'month';
+        } else {
+            $limit = 30;
+            $delta = 'day';
+        }
+        $axis_info = [
+            'time' => 'Total time taken (ms)',
+            'sql_time' => 'SQL time taken (ms)',
+            'sql_queries' => 'Queries (#)',
+            'memory' => 'Memory (MB)',
+            'endpoint' => 'Queries to endpoint (#)'
+        ];
+        $y_axis = $axis_info[isset($options['field']) ? $options['field'] : 'time'];
+        $data = ['y-axis' => $y_axis];
+        $data['data'] = array();
+        // Add total users data for all timestamps
+        
+        for ($i = 0; $i < $limit; $i++) {
+            $itemTime = strtotime('- ' . $i . $delta, $endOfDay);
+            $item = array();
+            $date = strftime('%Y-%m-%d', $itemTime);
+            $item = $this->getData($date, $options);
+            if (!empty($item)) {
+                $item['date'] = $date;
+                $data['data'][] = $item;
+            }
+        }
+        $keys = [];
+        foreach ($data['data'] as $day_data) {
+            foreach ($day_data as $key => $temp) {
+                $keys[$key] = 1;
+            }
+        }
+        $keys = array_keys($keys);
+        foreach ($data['data'] as $k => $day_data) {
+            foreach ($keys as $key) {
+                if (!isset($day_data[$key])) {
+                    $data['data'][$k][$key] = 0;
+                }
+            }
+            foreach ($day_data as $key => $temp) {
+                $keys[$key] = 1;
+            }
+        }
+        return $data;
+    }
+
+    private function getData($time, $options)
+    {
+        $dates = [$time];
+        $this->Benchmark = new BenchmarkTool($this->User);
+        $result = $this->Benchmark->getTopList(
+            isset($options['scope']) ? $options['scope'] : 'endpoint',
+            isset($options['field']) ? $options['field'] : 'memory',
+            $dates,
+            isset($options['limit']) ? $options['limit'] : 5,
+            isset($options['average']) ? $options['average'] : false,
+        );
+        if (!empty($result)) {
+            return $result[$time];
+        }
+        return false;
+    }
+
+    public function checkPermissions($user)
+    {
+        if (empty($user['Role']['perm_site_admin'])) {
+            return false;
+        }
+        return true;
+    }
+}

app/Lib/Dashboard/EventEvolutionLineWidget.php

@@ -58,9 +58,11 @@ class EventEvolutionLineWidget
             'recursive' => -1
         ];
         $eparams = [];
+        $filteringOnOrg = false;
         if (!empty($options['filter']) && is_array($options['filter'])) {
             foreach ($this->validFilterKeys as $filterKey) {
                 if (!empty($options['filter'][$filterKey])) {
+                    $filteringOnOrg = true;
                     if (!is_array($options['filter'][$filterKey])) {
                         $options['filter'][$filterKey] = [$options['filter'][$filterKey]];
                     }
@@ -87,6 +89,9 @@ class EventEvolutionLineWidget
             'conditions' => $oparams['conditions'],
             'fields' => ['id']
         ]);
+        if ($filteringOnOrg) {
+            $eparams['conditions']['AND']['Event.orgc_id IN'] = !empty($org_ids) ? $org_ids : [-1];
+        }
         $this->Event->virtualFields = [
             'published_date' => null
         ];

app/Lib/Dashboard/OrgContributionToplistWidget.php

@@ -137,4 +137,3 @@ class OrgContributionToplistWidget
         return ['data' => $results];
     }
 }
-?>

app/Lib/Dashboard/OrganisationListWidget.php

@@ -78,4 +78,3 @@ class OrganisationListWidget
         return ['data' => $results];
     }
 }
-?>

app/Lib/Dashboard/OrganisationMapWidget.php

@@ -85,4 +85,3 @@ class OrganisationMapWidget
         return $results;
     }
 }
-?>

app/Lib/Dashboard/OrgsContributorLastMonthWidget.php

@@ -19,4 +19,3 @@ class OrgsContributorLastMonthWidget extends OrgsContributorsGeneric
         return count($results) > 0;
     }
 }
- ?>

app/Lib/Dashboard/OrgsContributorsGeneric.php

@@ -45,4 +45,3 @@ class OrgsContributorsGeneric
         return $result;
     }
 }
-?>

app/Lib/Dashboard/OrgsUsingMitreWidget.php

@@ -32,4 +32,3 @@ class OrgsUsingMitreWidget extends OrgsContributorsGeneric
         return count($events) > 0;
     }
 }
-?>

app/Lib/Dashboard/OrgsUsingObjectsWidget.php

@@ -25,4 +25,3 @@ class OrgsUsingObjectsWidget extends OrgsContributorsGeneric
         return count($eventsIds) > 0;
     }
 }
-?>

app/Lib/Dashboard/UsageDataWidget.php

@@ -243,7 +243,8 @@ class UsageDataWidget
             $count = $this->Event->Attribute->find('count', [
                 'conditions' => $conditions,
                 'contain' => ['Event'],
-                'recursive' => -1
+                'recursive' => -1,
+                'ignoreIndexHint' => 'deleted'
             ]);
             $this->redis->setEx('misp:dashboard:attribute_count:' . $hash, 3600, $count);
         }

app/Lib/Dashboard/UserContributionToplistWidget.php

@@ -145,4 +145,3 @@ class UserContributionToplistWidget
         return true;
     }
 }
-?>

app/Lib/Tools/AttributeValidationTool.php

@@ -556,7 +556,12 @@ class AttributeValidationTool
                   if (!is_numeric($value) || $value < 0 || $value > 10) {
                       return __('The value has to be a number between 0 and 10.');
                   }
-                  return true;*/
+                return true;*/
+            case 'integer':
+                if (is_int($value)) {
+                    return true;
+                }
+                return __('The value has to be an integer value.');
             case 'iban':
             case 'bic':
             case 'btc':

app/Lib/Tools/BenchmarkTool.php

@@ -0,0 +1,178 @@
+<?php
+
+/**
+ * Get filter parameters from index searches
+ */
+
+class BenchmarkTool
+{
+    /** @var Model */
+    public $Model;
+    
+    /** @var redis */
+    public $redis;
+
+    /** @var retention */
+    private $retention = 0;
+
+    const BENCHMARK_SCOPES = ['user', 'endpoint', 'user_agent'];
+    const BENCHMARK_FIELDS = ['time', 'sql_time', 'sql_queries', 'memory'];
+    const BENCHMARK_UNITS = [
+        'time' => 's',
+        'sql_time' => 'ms',
+        'sql_queries' => '',
+        'memory' => 'MB'
+    ];
+
+    public $namespace = 'misp:benchmark:';
+
+    function __construct(Model $model) {
+        $this->Model = $model;
+    }
+
+    public function getSettings()
+    {
+        return [
+            'scope' => self::BENCHMARK_SCOPES,
+            'field' => self::BENCHMARK_FIELDS,
+            'average' => [0, 1],
+            'aggregate' => [0, 1]
+        ];
+    }
+
+    public function getUnits()
+    {
+        return self::BENCHMARK_UNITS;
+    }
+
+    public function startBenchmark()
+    {
+        $start_time = microtime(true);
+        $this->redis = $this->Model->setupRedis();
+        $this->retention = Configure::check('Plugin.benchmark_retention') ? Configure::read('Plugin.benchmark_retention') : 0;
+        return $start_time;
+    }
+
+    public function stopBenchmark(array $options)
+    {
+        $start_time = $options['start_time'];
+        if (!empty($options['user'])) {
+            $sql = $this->Model->getDataSource()->getLog(false, false);
+            $benchmarkData = [
+                'user' => $options['user'],
+                'endpoint' => $options['controller'] . '/' . $options['action'],
+                'user_agent' => $_SERVER['HTTP_USER_AGENT'],
+                'sql_queries' => $sql['count'],
+                'sql_time' => $sql['time'],
+                'time' => (microtime(true) - $start_time),
+                'memory' => (int)(memory_get_peak_usage(true) / 1024 / 1024),
+                //'date' => date('Y-m-d', strtotime("-3 days"))
+                'date' => date('Y-m-d')
+            ];
+            $this->pushBenchmarkDataToRedis($benchmarkData);
+        } else {
+            $sql = $this->Model->getDataSource()->getLog(false, false);
+            $benchmarkData = [
+                'user' => 'SYSTEM',
+                'endpoint' => $options['controller'] . '/' . $options['action'],
+                'user_agent' => 'CLI',
+                'sql_queries' => $sql['count'],
+                'sql_time' => $sql['time'],
+                'time' => (microtime(true) - $start_time),
+                'memory' => (int)(memory_get_peak_usage(true) / 1024 / 1024),
+                //'date' => date('Y-m-d', strtotime("-3 days"))
+                'date' => date('Y-m-d')
+            ];
+            $this->pushBenchmarkDataToRedis($benchmarkData);
+        }
+    }
+
+    private function pushBenchmarkDataToRedis($benchmarkData)
+    {
+        $this->redis = $this->Model->setupRedis();
+        $this->redis->pipeline();
+        $this->redis->sAdd(
+            $this->namespace . 'days',
+            $benchmarkData['date']
+        );
+        foreach (self::BENCHMARK_SCOPES as $scope) {
+            $this->redis->sAdd(
+                $this->namespace . $scope . ':list',
+                $benchmarkData[$scope]
+            );
+            $this->redis->zIncrBy(
+                $this->namespace . $scope . ':count:' . $benchmarkData['date'],
+                1,
+                $benchmarkData[$scope]
+            );
+            foreach (self::BENCHMARK_FIELDS as $field) {
+                $this->redis->zIncrBy(
+                    $this->namespace . $scope . ':' . $field . ':' . $benchmarkData['date'],
+                    $benchmarkData[$field],
+                    $benchmarkData[$scope]
+                );
+            }
+            $this->redis->zIncrBy(
+                $this->namespace . $scope . ':endpoint:' . $benchmarkData['date'] . ':' . $benchmarkData['user'],
+                1,
+                $benchmarkData['endpoint']
+            );
+        }
+        $this->redis->exec();
+    }
+
+    public function getTopList(string $scope, string $field, array $days = [], $limit = 10, $average = false, $aggregate = false)
+    {
+        if (empty($this->redis)) {
+            $this->redis = $this->Model->setupRedis();
+        }
+        $results = [];
+        if (is_string($days)) {
+            $days = [$days];
+        }
+        foreach ($days as $day) {
+            $temp = $this->redis->zrevrange($this->namespace . $scope . ':' . $field . ':' . $day, 0, $limit, true);
+            foreach ($temp as $k => $v) {
+                if ($average) {
+                    $divisor = $this->redis->zscore($this->namespace . $scope . ':count:' . $day, $k);
+                    if ($aggregate) {
+                        $results['aggregate'][$k] = empty($results['aggregate'][$k]) ? ($v / $divisor) : ($results['aggregate'][$k] + ($v / $divisor));
+                    } else {
+                        $results[$day][$k] = (int)($v / $divisor);
+                    }
+                } else {
+                    if ($aggregate) {
+                        $results['aggregate'][$k] = empty($results['aggregate'][$k]) ? $v : ($results['aggregate'][$k] + $v);
+                    } else {
+                        $results[$day][$k] = $v;
+                    }
+                }
+            }
+        }
+        if ($aggregate && $average) {
+            $count_days = count($days);
+            foreach ($results['aggregate'] as $k => $result) {
+                $results['aggregate'][$k] = (int)($result / $count_days);
+            }
+        }
+        return $results;
+    }
+
+    public function getAllTopLists(array $days = null, $limit = 10, $average = false, $aggregate = false, $scope_filter = [])
+    {
+        if (empty($this->redis)) {
+            $this->redis = $this->Model->setupRedis();
+        }
+        if ($days === null) {
+            $days = $this->redis->smembers($this->namespace . 'days');
+        }
+        foreach (self::BENCHMARK_SCOPES as $scope) {
+            if (empty($scope_filter) || in_array($scope, $scope_filter)) {
+                foreach (self::BENCHMARK_FIELDS as $field) {
+                    $results[$scope][$field] = $this->getTopList($scope, $field, $days, $limit, $average, $aggregate);
+                }
+            }
+        }
+        return $results;
+    }
+}

app/Lib/Tools/CorrelationGraphTool.php

@@ -1,4 +1,6 @@
 <?php
+App::uses('OrgImgHelper', 'View/Helper');
+
   class CorrelationGraphTool
   {
       private $__lookupTables = array();
@@ -9,20 +11,24 @@
       /** @var Taxonomy */
       private $__taxonomyModel;
       private $__galaxyClusterModel = false;
-      private $__user = false;
-      private $__json = array();
+      /** @var User */
+      private $__user;
+      /** @var array */
+      private $data;
+      private $orgImgHelper;
 
-      public function construct(Event $eventModel, $taxonomyModel, $galaxyClusterModel, $user, $json)
+      public function __construct(Event $eventModel, $taxonomyModel, $galaxyClusterModel, array $user, array $data)
       {
           $this->__eventModel = $eventModel;
           $this->__taxonomyModel = $taxonomyModel;
           $this->__galaxyClusterModel = $galaxyClusterModel;
           $this->__user = $user;
-          $this->__json = $json;
+          $this->data = $data;
           $this->__lookupTables = array(
-        'analysisLevels' => $this->__eventModel->analysisLevels,
-        'distributionLevels' => $this->__eventModel->Attribute->distributionLevels
-      );
+            'analysisLevels' => $eventModel->analysisLevels,
+            'distributionLevels' => $eventModel->Attribute->distributionLevels
+          );
+          $this->orgImgHelper = new OrgImgHelper(new View());
           return true;
       }
 
@@ -38,7 +44,7 @@
               'sgReferenceOnly' => true,
           ));
           if (empty($event)) {
-              return $this->__json;
+              return $this->data;
           }
           $this->cleanLinks();
           $event[0]['Event']['Orgc'] = $event[0]['Orgc'];
@@ -75,7 +81,7 @@
       public function buildGraphJson($id, $type = 'event', $action = 'create')
       {
           if ($action == 'delete') {
-              return $this->__json;
+              return $this->data;
           }
           switch ($type) {
         case 'event':
@@ -88,13 +94,13 @@
           $this->__expandTag($id);
           break;
       }
-          return $this->__json;
+          return $this->data;
       }
 
       private function __deleteObject($id)
       {
           $this->cleanLinks();
-          return $this->__json;
+          return $this->data;
       }
 
       private function __handleObjects($objects, $anchor_id, $full = false)
@@ -193,8 +199,8 @@
 
       private function __expandGalaxy($id)
       {
-          if (!empty($this->__json['nodes'])) {
-              foreach ($this->__json['nodes'] as $k => $node) {
+          if (!empty($this->data['nodes'])) {
+              foreach ($this->data['nodes'] as $k => $node) {
                   if ($node['type'] == 'galaxy' && $node['id'] == $id) {
                       $current_galaxy_id = $k;
                       $tag_name = $node['tag_name'];
@@ -205,7 +211,7 @@
               $current_galaxy_id = $this->__addGalaxy($id);
           }
           $this->cleanLinks();
-          $events = $this->__eventModel->EventTag->Tag->fetchSimpleEventsForTag($this->__json['nodes'][$current_galaxy_id]['tag_name'], $this->__user, true);
+          $events = $this->__eventModel->EventTag->Tag->fetchSimpleEventsForTag($this->data['nodes'][$current_galaxy_id]['tag_name'], $this->__user, true);
           foreach ($events as $event) {
               $current_event_id = $this->__createNode('event', $event);
               $this->__addLink($current_event_id, $current_galaxy_id);
@@ -229,7 +235,7 @@
       {
           $link = $this->graphJsonContainsLink($from_id, $to_id);
           if ($link === false) {
-              $this->__json['links'][] = array('source' => $from_id, 'target' => $to_id, 'linkDistance' => $linkDistance);
+              $this->data['links'][] = array('source' => $from_id, 'target' => $to_id, 'linkDistance' => $linkDistance);
           }
       }
 
@@ -240,7 +246,7 @@
           if ($from_uuid == $to_uuid) {
               return false;
           }
-          foreach ($this->__json['nodes'] as $k => $node) {
+          foreach ($this->data['nodes'] as $k => $node) {
               if ($node['uuid'] === $from_uuid) {
                   $from_id = $k;
               }
@@ -277,10 +283,9 @@
             );
             break;
           case 'event':
-            if ($this->orgImgExists($data['Orgc']['name'])) {
-                $image = Configure::read('MISP.baseurl') . '/img/orgs/' . h($data['Orgc']['name']) . '.png';
-            } else {
-                $image = Configure::read('MISP.baseurl') . '/img/orgs/MISP.png';
+            $orgImage = $this->orgImgHelper->getOrgLogoAsBase64($data['Orgc']);
+            if ($orgImage === null) {
+                $orgImage = Configure::read('MISP.baseurl') . '/img/misp-org.png';
             }
             $node = array(
               'unique_id' => 'event-' . $data['id'],
@@ -289,7 +294,7 @@
               'id' => $data['id'],
               'expanded' => $expand,
               'uuid' => $data['uuid'],
-              'image' => $image,
+              'image' => $orgImage,
               'info' => $data['info'],
               'org' => $data['Orgc']['name'],
               'analysis' => $this->__lookupTables['analysisLevels'][$data['analysis']],
@@ -345,11 +350,11 @@
             );
             break;
         }
-              $this->__json['nodes'][] = $node;
-              $current_id = count($this->__json['nodes'])-1;
+              $this->data['nodes'][] = $node;
+              $current_id = count($this->data['nodes'])-1;
           } else {
               if ($expand) {
-                  $this->__json['nodes'][$current_id]['expanded'] = 1;
+                  $this->data['nodes'][$current_id]['expanded'] = 1;
               }
           }
           return $current_id;
@@ -357,11 +362,11 @@
 
       public function cleanLinks()
       {
-          if (isset($this->__json['nodes']) && isset($this->__json['links'])) {
+          if (isset($this->data['nodes']) && isset($this->data['links'])) {
               $links = array();
-              foreach ($this->__json['links'] as $link) {
+              foreach ($this->data['links'] as $link) {
                   $temp = array();
-                  foreach ($this->__json['nodes'] as $k => $node) {
+                  foreach ($this->data['nodes'] as $k => $node) {
                       if ($link['source'] == $node) {
                           $temp['source'] = $k;
                       }
@@ -372,32 +377,24 @@
                   $temp['linkDistance'] = $link['linkDistance'];
                   $links[] = $temp;
               }
-              $this->__json['links'] = $links;
+              $this->data['links'] = $links;
           } else {
-              if (!isset($this->__json['links'])) {
-                  $this->__json['links'] = array();
+              if (!isset($this->data['links'])) {
+                  $this->data['links'] = array();
               }
-              if (!isset($this->__json['nodes'])) {
-                  $this->__json['nodes'] = array();
+              if (!isset($this->data['nodes'])) {
+                  $this->data['nodes'] = array();
               }
           }
           return true;
       }
 
-      public function orgImgExists($org)
-      {
-          if (file_exists(APP . 'webroot' . DS . 'img' . DS . 'orgs' . DS . $org . '.png')) {
-              return true;
-          }
-          return false;
-      }
-
       public function graphJsonContains($type, $element)
       {
-          if (!isset($this->__json['nodes'])) {
+          if (!isset($this->data['nodes'])) {
               return false;
           }
-          foreach ($this->__json['nodes'] as $k => $node) {
+          foreach ($this->data['nodes'] as $k => $node) {
               if ($type == 'event' && $node['type'] == 'event' && $node['id'] == $element['id']) {
                   return $k;
               }
@@ -418,10 +415,10 @@
       }
       public function graphJsonContainsLink($id1, $id2)
       {
-          if (!isset($this->__json['links'])) {
+          if (!isset($this->data['links'])) {
               return false;
           }
-          foreach ($this->__json['links'] as $k => $link) {
+          foreach ($this->data['links'] as $k => $link) {
               if (($link['source'] == $id1 && $link['target'] == $id2) || ($link['source'] == $id2 && $link['target'] == $id1)) {
                   return $k;
               }

app/Lib/Tools/CurlClient.php

@@ -6,8 +6,12 @@ class CurlClient extends HttpSocketExtended
     /** @var resource */
     private $ch;
 
-    /** @var int */
-    private $timeout = 30;
+    /**
+     * Maximum time the transfer is allowed to complete in seconds
+     * 300 seconds is recommended timeout for MISP servers
+     * @var int
+     */
+    private $timeout = 300;
 
     /** @var string|null */
     private $caFile;
@@ -30,6 +34,9 @@ class CurlClient extends HttpSocketExtended
     /** @var array */
     private $proxy = [];
 
+    /** @var array */
+    private $defaultOptions;
+
     /**
      * @param array $params
      * @noinspection PhpMissingParentConstructorInspection
@@ -57,6 +64,7 @@ class CurlClient extends HttpSocketExtended
         if (isset($params['ssl_verify_peer'])) {
             $this->verifyPeer = $params['ssl_verify_peer'];
         }
+        $this->defaultOptions = $this->generateDefaultOptions();
     }
 
     /**
@@ -164,6 +172,7 @@ class CurlClient extends HttpSocketExtended
             return;
         }
         $this->proxy = compact('host', 'port', 'method', 'user', 'pass');
+        $this->defaultOptions = $this->generateDefaultOptions(); // regenerate default options in case proxy setting is changed
     }
 
     /**
@@ -194,7 +203,7 @@ class CurlClient extends HttpSocketExtended
             $url .= '?' . http_build_query($query, '', '&', PHP_QUERY_RFC3986);
         }
 
-        $options = $this->generateOptions();
+        $options = $this->defaultOptions; // this will copy default options
         $options[CURLOPT_URL] = $url;
         $options[CURLOPT_CUSTOMREQUEST] = $method;
 
@@ -202,6 +211,10 @@ class CurlClient extends HttpSocketExtended
             $options[CURLOPT_POSTFIELDS] = $query;
         }
 
+        if ($method === 'HEAD') {
+            $options[CURLOPT_NOBODY] = true;
+        }
+
         if (!empty($request['header'])) {
             $headers = [];
             foreach ($request['header'] as $key => $value) {
@@ -231,7 +244,6 @@ class CurlClient extends HttpSocketExtended
             }
             return $len;
         };
-
         if (!curl_setopt_array($this->ch, $options)) {
             throw new \RuntimeException('curl error: Could not set options');
         }
@@ -268,16 +280,6 @@ class CurlClient extends HttpSocketExtended
      */
     private function constructResponse($body, array $headers, $code)
     {
-        if (isset($responseHeaders['content-encoding']) && $responseHeaders['content-encoding'] === 'zstd') {
-            if (!function_exists('zstd_uncompress')) {
-                throw new SocketException('Response is zstd encoded, but PHP do not support zstd decoding.');
-            }
-            $body = zstd_uncompress($body);
-            if ($body === false) {
-                throw new SocketException('Could not decode zstd encoded response.');
-            }
-        }
-
         $response = new HttpSocketResponseExtended();
         $response->code = $code;
         $response->body = $body;
@@ -308,7 +310,7 @@ class CurlClient extends HttpSocketExtended
     /**
      * @return array
      */
-    private function generateOptions()
+    private function generateDefaultOptions()
     {
         $options = [
             CURLOPT_FOLLOWLOCATION => true, // Allows to follow redirect
@@ -316,7 +318,7 @@ class CurlClient extends HttpSocketExtended
             CURLOPT_RETURNTRANSFER => true, // Should cURL return or print out the data? (true = return, false = print)
             CURLOPT_HEADER => false, // Include header in result?
             CURLOPT_TIMEOUT => $this->timeout, // Timeout in seconds
-            CURLOPT_PROTOCOLS => CURLPROTO_HTTPS | CURLPROTO_HTTP, // be sure that only HTTP and HTTPS protocols are enabled,
+            CURLOPT_PROTOCOLS => CURLPROTO_HTTPS | CURLPROTO_HTTP, // be sure that only HTTP and HTTPS protocols are enabled
         ];
 
         if ($this->caFile) {
@@ -332,7 +334,7 @@ class CurlClient extends HttpSocketExtended
         }
 
         if ($this->compress) {
-            $options[CURLOPT_ACCEPT_ENCODING] = $this->supportedEncodings();
+            $options[CURLOPT_ACCEPT_ENCODING] = ''; // empty string means all encodings supported by curl
         }
 
         if ($this->allowSelfSigned) {
@@ -349,25 +351,4 @@ class CurlClient extends HttpSocketExtended
 
         return $options;
     }
-
-    /**
-     * @return string
-     */
-    private function supportedEncodings()
-    {
-        $encodings = [];
-        // zstd is not supported by curl itself, but add support if PHP zstd extension is installed
-        if (function_exists('zstd_uncompress')) {
-            $encodings[] = 'zstd';
-        }
-        // brotli and gzip is supported by curl itself if it is compiled with these features
-        $info = curl_version();
-        if (defined('CURL_VERSION_BROTLI') && $info['features'] & CURL_VERSION_BROTLI) {
-            $encodings[] = 'br';
-        }
-        if ($info['features'] & CURL_VERSION_LIBZ) {
-            $encodings[] = 'gzip, deflate';
-        }
-        return implode(', ', $encodings);
-    }
 }

app/Lib/Tools/HttpSocketExtended.php

@@ -24,7 +24,7 @@ class HttpSocketHttpException extends Exception
             $message .= " for URL $url";
         }
         if ($response->body) {
-            $message .= ': ' . substr($response->body, 0, 100);
+            $message .= ': ' . substr(ltrim($response->body), 0, 100);
         }
 
         parent::__construct($message, (int)$response->code);
@@ -114,10 +114,15 @@ class HttpSocketResponseExtended extends HttpSocketResponse
      */
     public function json()
     {
+        if (strlen($this->body) === 0) {
+            throw new HttpSocketJsonException('Could not parse empty response as JSON.', $this);
+        }
+
         try {
             return JsonTool::decode($this->body);
         } catch (Exception $e) {
-            throw new HttpSocketJsonException('Could not parse response as JSON.', $this, $e);
+            $contentType = $this->getHeader('content-type');
+            throw new HttpSocketJsonException("Could not parse HTTP response as JSON. Received Content-Type $contentType.", $this, $e);
         }
     }
 }

app/Lib/Tools/ServerSyncTool.php

@@ -217,7 +217,7 @@ class ServerSyncTool
     }
 
     /**
-     * @param array $rules
+     * @param array $candidates
      * @return HttpSocketResponseExtended
      * @throws HttpSocketHttpException
      * @throws HttpSocketJsonException
@@ -225,7 +225,7 @@ class ServerSyncTool
     public function filterAnalystDataForPush(array $candidates)
     {
         if (!$this->isSupported(self::PERM_ANALYST_DATA)) {
-            return [];
+            throw new RuntimeException("Remote server do not support analyst data");
         }
 
         return $this->post('/analyst_data/filterAnalystDataForPush', $candidates);
@@ -240,20 +240,23 @@ class ServerSyncTool
     public function fetchIndexMinimal(array $rules)
     {
         if (!$this->isSupported(self::PERM_ANALYST_DATA)) {
-            return [];
+            throw new RuntimeException("Remote server do not support analyst data");
         }
 
         return $this->post('/analyst_data/indexMinimal', $rules);
     }
 
     /**
+     * @param string $type
+     * @param array $uuids
+     * @return HttpSocketResponseExtended
      * @throws HttpSocketJsonException
      * @throws HttpSocketHttpException
      */
     public function fetchAnalystData($type, array $uuids)
     {
         if (!$this->isSupported(self::PERM_ANALYST_DATA)) {
-            return [];
+            throw new RuntimeException("Remote server do not support analyst data");
         }
 
         $params = [
@@ -264,12 +267,10 @@ class ServerSyncTool
         $url .= $this->createParams($params);
         $url .= '.json';
         return $this->get($url);
-
-        // $response = $this->post('/analyst_data/restSearch' , $params);
-        // return $response->json();
     }
 
-        /**
+    /**
+     * @param string $type
      * @param array $analystData
      * @return HttpSocketResponseExtended
      * @throws HttpSocketHttpException
@@ -296,19 +297,26 @@ class ServerSyncTool
 
     /**
      * @param array $eventUuids
+     * @param array $blockedOrgs Blocked organisation UUIDs
      * @return array
      * @throws HttpSocketHttpException
      * @throws HttpSocketJsonException
      * @throws JsonException
      */
-    public function fetchSightingsForEvents(array $eventUuids)
+    public function fetchSightingsForEvents(array $eventUuids, array $blockedOrgs = [])
     {
-        return $this->post('/sightings/restSearch/event', [
+        $postParams = [
             'returnFormat' => 'json',
             'last' => 0, // fetch all
             'includeUuid' => true,
             'uuid' => $eventUuids,
-        ])->json()['response'];
+        ];
+        if (!empty($blockedOrgs)) {
+            $postParams['org_id'] = array_map(function ($uuid) {
+                return "!$uuid";
+            }, $blockedOrgs);
+        }
+        return $this->post('/sightings/restSearch/event', $postParams)->json()['response'];
     }
 
     /**
@@ -498,6 +506,16 @@ class ServerSyncTool
         return $this->socket->getMetaData();
     }
 
+    /**
+     * @param string $message
+     * @return void
+     */
+    public function debug($message)
+    {
+        $memoryUsage = round(memory_get_usage() / 1024 / 1024, 2);
+        CakeLog::debug("[Server sync #{$this->serverId()}]: $message. Memory: $memoryUsage MB");
+    }
+
     /**
      * @params string $url Relative URL
      * @return HttpSocketResponseExtended
@@ -548,6 +566,7 @@ class ServerSyncTool
 
         if ($etag) {
             // Remove compression marks that adds Apache for compressed content
+            // This can be removed in future as this is already checked by MISP itself since 2024-03
             $etagWithoutQuotes = trim($etag, '"');
             $dashPos = strrpos($etagWithoutQuotes, '-');
             if ($dashPos && in_array(substr($etagWithoutQuotes, $dashPos + 1), ['br', 'gzip'], true)) {

app/Lib/Tools/SyncTool.php

@@ -1,7 +1,6 @@
 <?php
 class SyncTool
 {
-
     const ALLOWED_CERT_FILE_EXTENSIONS = ['pem', 'crt'];
 
     /**
@@ -50,7 +49,7 @@ class SyncTool
      * @return HttpSocketExtended
      * @throws Exception
      */
-    public function createHttpSocket($params = array())
+    public function createHttpSocket(array $params = [])
     {
         // Use own CA PEM file
         $caPath = Configure::read('MISP.ca_path');
@@ -84,6 +83,9 @@ class SyncTool
         }
 
         if (function_exists('curl_init')) {
+            if (!isset($params['timeout']) && Configure::check('MISP.curl_request_timeout')) {
+                $params['timeout'] = (int)Configure::read('MISP.curl_request_timeout');
+            }
             App::uses('CurlClient', 'Tools');
             $HttpSocket = new CurlClient($params);
         } else {

app/Model/AnalystData.php

@@ -43,7 +43,7 @@ class AnalystData extends AppModel
         'distribution',
         'sharing_group_id',
     ];
-    protected $EDITABLE_FIELDS = [];
+    public const EDITABLE_FIELDS = [];
 
     /** @var object|null */
     protected $Note;
@@ -180,12 +180,21 @@ class AnalystData extends AppModel
         }
         $this->data[$this->current_type]['modified'] = (new DateTime($this->data[$this->current_type]['modified'], new DateTimeZone('UTC')))->format('Y-m-d H:i:s');
         $this->data[$this->current_type]['created'] = (new DateTime($this->data[$this->current_type]['created'], new DateTimeZone('UTC')))->format('Y-m-d H:i:s');
+
+        if (empty($this->data[$this->current_type]['id'])) {
+            if (!isset($this->data[$this->current_type]['distribution'])) {
+                $this->data[$this->current_type]['distribution'] = Configure::read('MISP.default_event_distribution'); // use default event distribution
+            }
+            if ($this->data[$this->current_type]['distribution'] != 4) {
+                $this->data[$this->current_type]['sharing_group_id'] = null;
+            }
+        }
         return true;
     }
 
     public function getEditableFields(): array
     {
-        return array_merge(self::BASE_EDITABLE_FIELDS, static::EDITABLE_FIELDS);
+        return array_merge(static::BASE_EDITABLE_FIELDS, static::EDITABLE_FIELDS);
     }
 
     /**
@@ -377,6 +386,8 @@ class AnalystData extends AppModel
             return $analystData;
         }
         $this->fetchedUUIDFromRecursion[$analystData['uuid']] = true;
+        $this->Note = ClassRegistry::init('Note');
+        $this->Opinion = ClassRegistry::init('Opinion');
 
         $paramsNote = [
             'recursive' => -1,
@@ -514,6 +525,10 @@ class AnalystData extends AppModel
             $analystData[$type]['org_uuid'] = $user['Organisation']['uuid'];
         }
 
+        if (!isset($analystData[$type]['uuid'])) {
+            $analystData[$type]['uuid'] = CakeText::uuid();
+        }
+
         $this->AnalystDataBlocklist = ClassRegistry::init('AnalystDataBlocklist');
         if ($this->AnalystDataBlocklist->checkIfBlocked($analystData[$type]['uuid'])) {
             $results['errors'][] = __('Blocked by blocklist');
@@ -540,8 +555,8 @@ class AnalystData extends AppModel
 
         if (!Configure::check('MISP.enableOrgBlocklisting') || Configure::read('MISP.enableOrgBlocklisting') !== false) {
             $analystModel->OrgBlocklist = ClassRegistry::init('OrgBlocklist');
-            $orgcUUID = $analystData[$type]['Orgc']['uuid'];
-            if ($analystData[$type]['orgc_uuid'] != 0 && $analystModel->OrgBlocklist->hasAny(array('OrgBlocklist.org_uuid' => $orgcUUID))) {
+            $orgcUUID = $analystData[$type]['orgc_uuid'];
+            if ($orgcUUID != 0 && $analystModel->OrgBlocklist->hasAny(array('OrgBlocklist.org_uuid' => $orgcUUID))) {
                 $results['errors'][] = __('Organisation blocklisted (%s)', $orgcUUID);
                 $results['ignored']++;
                 return $results;
@@ -549,12 +564,6 @@ class AnalystData extends AppModel
         }
 
         $analystData = $analystModel->captureOrganisationAndSG($analystData, $type, $user);
-        if (!isset($analystData[$type]['distribution'])) {
-            $analystData[$type]['distribution'] = Configure::read('MISP.default_event_distribution'); // use default event distribution
-        }
-        if ($analystData[$type]['distribution'] != 4) {
-            $analystData[$type]['sharing_group_id'] = null;
-        }
 
         // Start saving from the leaf since to make sure child elements get saved even if the parent should not be saved (or updated due to locked or timestamp)
         foreach (self::ANALYST_DATA_TYPES as $childType) {
@@ -639,9 +648,8 @@ class AnalystData extends AppModel
             return [];
         }
         $this->Server = ClassRegistry::init('Server');
-        $this->AnalystData = ClassRegistry::init('AnalystData');
 
-        $this->log("Starting Analyst Data sync with server #{$server['Server']['id']}", LOG_INFO);
+        $serverSync->debug("Starting Analyst Data sync");
 
         $analystData = $this->collectDataForPush($serverSync->server());
         $keyedAnalystData = [];
@@ -1007,11 +1015,15 @@ class AnalystData extends AppModel
      *
      * @param array $user
      * @param ServerSyncTool $serverSync
+     * @return int Number of saved analysis
      */
     public function pull(array $user, ServerSyncTool $serverSync)
     {
+        if (!$serverSync->isSupported(ServerSyncTool::PERM_ANALYST_DATA)) {
+            return 0;
+        }
+
         $this->Server = ClassRegistry::init('Server');
-        $this->AnalystData = ClassRegistry::init('AnalystData');
         try {
             $filterRules = $this->buildPullFilterRules($serverSync->server());
             $remoteData = $serverSync->fetchIndexMinimal($filterRules)->json();
@@ -1051,14 +1063,11 @@ class AnalystData extends AppModel
             return 0;
         }
 
-        if ($serverSync->isSupported(ServerSyncTool::PERM_ANALYST_DATA)) {
-            return $this->pullInChunks($user, $remoteUUIDsToFetch, $serverSync);
-        }
+        return $this->pullInChunks($user, $remoteUUIDsToFetch, $serverSync);
     }
 
-    public function pullInChunks(array $user, array $analystDataUuids, ServerSyncTool $serverSync)
+    private function pullInChunks(array $user, array $analystDataUuids, ServerSyncTool $serverSync)
     {
-        $uuids = array_keys($analystDataUuids);
         $saved = 0;
         $serverOrgUUID = $this->Org->find('first', [
             'recursive' => -1,

app/Model/AppModel.php

@@ -91,7 +91,7 @@ class AppModel extends Model
         105 => false, 106 => false, 107 => false, 108 => false, 109 => false, 110 => false,
         111 => false, 112 => false, 113 => true, 114 => false, 115 => false, 116 => false,
         117 => false, 118 => false, 119 => false, 120 => false, 121 => false, 122 => false,
-        123 => false,
+        123 => false, 124 => false, 125 => false,
     );
 
     const ADVANCED_UPDATES_DESCRIPTION = array(
@@ -1990,7 +1990,7 @@ class AppModel extends Model
                 $sqlArray[] = "ALTER TABLE `event_reports` modify `content` mediumtext";
                 break;
             case 117:
-                $sqlArray[] = "CREATE TABLE `user_login_profiles` (
+                $sqlArray[] = "CREATE TABLE IF NOT EXISTS `user_login_profiles` (
                     `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
                     `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
                     `user_id` int(11) NOT NULL,
@@ -2018,7 +2018,7 @@ class AppModel extends Model
                 $sqlArray[] = "ALTER TABLE `access_logs` MODIFY `action` varchar(191) NOT NULL";
                 break;
             case 121:
-                $sqlArray[] = "CREATE TABLE `notes` (
+                $sqlArray[] = "CREATE TABLE IF NOT EXISTS `notes` (
                     `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
                     `uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
                     `object_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
@@ -2026,8 +2026,8 @@ class AppModel extends Model
                     `authors` text,
                     `org_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
                     `orgc_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
-                    `created` datetime DEFAULT CURRENT_TIMESTAMP,
-                    `modified` datetime ON UPDATE CURRENT_TIMESTAMP,
+                    `created` datetime NOT NULL,
+                    `modified` datetime NOT NULL,
                     `distribution` tinyint(4) NOT NULL,
                     `sharing_group_id` int(10) unsigned,
                     `locked` tinyint(1) NOT NULL DEFAULT 0,
@@ -2043,7 +2043,7 @@ class AppModel extends Model
                     KEY `sharing_group_id` (`sharing_group_id`)
                   ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;";
 
-                $sqlArray[] = "CREATE TABLE `opinions` (
+                $sqlArray[] = "CREATE TABLE IF NOT EXISTS `opinions` (
                     `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
                     `uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
                     `object_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
@@ -2051,8 +2051,8 @@ class AppModel extends Model
                     `authors` text,
                     `org_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
                     `orgc_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
-                    `created` datetime DEFAULT CURRENT_TIMESTAMP,
-                    `modified` datetime ON UPDATE CURRENT_TIMESTAMP,
+                    `created` datetime NOT NULL,
+                    `modified` datetime NOT NULL,
                     `distribution` tinyint(4) NOT NULL,
                     `sharing_group_id` int(10) unsigned,
                     `locked` tinyint(1) NOT NULL DEFAULT 0,
@@ -2069,7 +2069,7 @@ class AppModel extends Model
                     KEY `opinion` (`opinion`)
                 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;";
 
-                $sqlArray[] = "CREATE TABLE `relationships` (
+                $sqlArray[] = "CREATE TABLE IF NOT EXISTS `relationships` (
                     `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
                     `uuid` varchar(40) CHARACTER SET ascii NOT NULL,
                     `object_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
@@ -2077,8 +2077,8 @@ class AppModel extends Model
                     `authors` text,
                     `org_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
                     `orgc_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
-                    `created` datetime DEFAULT CURRENT_TIMESTAMP,
-                    `modified` datetime ON UPDATE CURRENT_TIMESTAMP,
+                    `created` datetime NOT NULL,
+                    `modified` datetime NOT NULL,
                     `distribution` tinyint(4) NOT NULL,
                     `sharing_group_id` int(10) unsigned,
                     `locked` tinyint(1) NOT NULL DEFAULT 0,
@@ -2117,14 +2117,14 @@ class AppModel extends Model
                 $sqlArray[] = "ALTER TABLE `servers` ADD `pull_analyst_data` tinyint(1) NOT NULL DEFAULT 0 AFTER `push_analyst_data`;";
                 break;
             case 122:
-                $sqlArray[] = "CREATE TABLE `collections` (
+                $sqlArray[] = "CREATE TABLE IF NOT EXISTS `collections` (
                     `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
                     `uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
                     `org_id` int(10) unsigned NOT NULL,
                     `orgc_id` int(10) unsigned NOT NULL,
                     `user_id` int(10) unsigned NOT NULL,
-                    `created` datetime DEFAULT CURRENT_TIMESTAMP,
-                    `modified` datetime ON UPDATE CURRENT_TIMESTAMP,
+                    `created` datetime NOT NULL,
+                    `modified` datetime NOT NULL,
                     `distribution` tinyint(4) NOT NULL,
                     `sharing_group_id` int(10) unsigned,
                     `name` varchar(191) NOT NULL,
@@ -2141,7 +2141,7 @@ class AppModel extends Model
                     KEY `sharing_group_id` (`sharing_group_id`)
                     ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;";
 
-                $sqlArray[] = "CREATE TABLE `collection_elements` (
+                $sqlArray[] = "CREATE TABLE IF NOT EXISTS `collection_elements` (
                     `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
                     `uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
                     `element_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
@@ -2164,6 +2164,21 @@ class AppModel extends Model
                 $sqlArray[] = 'ALTER TABLE `opinions` MODIFY `modified` datetime NOT NULL;';
                 $sqlArray[] = 'ALTER TABLE `relationships` MODIFY `modified` datetime NOT NULL;';
                 break;
+            case 124:
+                $sqlArray[] = 'CREATE TABLE IF NOT EXISTS `sighting_blocklists` (
+                    `id` int(11) NOT NULL AUTO_INCREMENT,
+                    `org_uuid` varchar(40) COLLATE utf8_bin NOT NULL,
+                    `created` datetime NOT NULL,
+                    `org_name` varchar(255) COLLATE utf8_bin NOT NULL,
+                    `comment` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci,
+                    PRIMARY KEY (`id`),
+                    INDEX `org_uuid` (`org_uuid`),
+                    INDEX `org_name` (`org_name`)
+                  ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;';
+                break;
+            case 125:
+                $sqlArray[] = "ALTER TABLE `feeds` ADD COLUMN `tag_collection_id` INT(11) NOT NULL DEFAULT 0;";
+                break;
             case 'fixNonEmptySharingGroupID':
                 $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
                 $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
@@ -3813,7 +3828,7 @@ class AppModel extends Model
     protected function logException($message, Exception $exception, $type = LOG_ERR)
     {
         // If Sentry is installed, send exception to Sentry
-        if (function_exists('\Sentry\captureException') && $type === LOG_ERR) {
+        if (function_exists('\Sentry\captureException') && $type <= LOG_ERR) {
             \Sentry\captureException(new Exception($message, $type, $exception));
         }
 

app/Model/AttachmentScan.php

@@ -258,7 +258,7 @@ class AttachmentScan extends AppModel
                     $scanned++;
                 }
             } catch (Exception $e) {
-                $this->logException("Could not scan attachment for $type {$attribute['Attribute']['id']}", $e, LOG_WARNING);
+                $this->logException("Could not scan attachment for $type {$attribute[$type]['id']}", $e, LOG_WARNING);
                 $fails++;
             }
 

app/Model/Attribute.php

@@ -118,6 +118,7 @@ class Attribute extends AppModel
         'aba-rtn',
         'gender',
         'counter',
+        'integer',
         'float',
         'port',
         'nationality',
@@ -969,7 +970,7 @@ class Attribute extends AppModel
         $maxWidth = $maxWidth ?: $defaultMaxSize;
         $maxHeight = $maxHeight ?: $defaultMaxSize;
         $suffix = null;
-        
+
         if ($maxWidth == $defaultMaxSize && $maxHeight == $defaultMaxSize) {
             $thumbnailInRedis = Configure::read('MISP.thumbnail_in_redis');
             if ($thumbnailInRedis) {
@@ -1141,29 +1142,36 @@ class Attribute extends AppModel
         }
         $temp = array();
         if (!empty($tagArray[1])) {
-            if ($options['scope'] == 'all' || $options['scope'] == 'Event') {
-                $subquery_options = array(
-                    'conditions' => array(
-                        'tag_id' => $tagArray[1]
-                    ),
-                    'fields' => array(
-                        'event_id'
-                    )
-                );
-                $lookup_field = ($options['scope'] === 'Event') ? 'Event.id' : 'Attribute.event_id';
-                $conditions['AND'][] = array_merge($temp, $this->subQueryGenerator($tag->EventTag, $subquery_options, $lookup_field, 1));
-            }
-            if ($options['scope'] == 'all' || $options['scope'] == 'Attribute') {
-                $subquery_options = array(
-                    'conditions' => array(
-                        'tag_id' => $tagArray[1]
-                    ),
-                    'fields' => array(
-                        $options['scope'] === 'Event' ? 'event.id' : 'attribute_id'
-                    )
-                );
-                $lookup_field = $options['scope'] === 'Event' ? 'Event.id' : 'Attribute.id';
-                $conditions['AND'][] = array_merge($temp, $this->subQueryGenerator($tag->AttributeTag, $subquery_options, $lookup_field, 1));
+            /* 
+             * If we didn't find the given negation tag, no need to use the -1 trick,
+             * it is basically a hack to block the search from finding anything if no positive lookup was valid.
+             * However, if none of the negated tags exist, there's nothing to filter here
+             */
+            if (count($tagArray[1]) !== 1 || $tagArray[1][0] != -1) {
+                if ($options['scope'] == 'all' || $options['scope'] == 'Event') {
+                    $subquery_options = array(
+                        'conditions' => array(
+                            'tag_id' => $tagArray[1]
+                        ),
+                        'fields' => array(
+                            'event_id'
+                        )
+                    );
+                    $lookup_field = ($options['scope'] === 'Event') ? 'Event.id' : 'Attribute.event_id';
+                    $conditions['AND'][] = array_merge($temp, $this->subQueryGenerator($tag->EventTag, $subquery_options, $lookup_field, 1));
+                }
+                if ($options['scope'] == 'all' || $options['scope'] == 'Attribute') {
+                    $subquery_options = array(
+                        'conditions' => array(
+                            'tag_id' => $tagArray[1]
+                        ),
+                        'fields' => array(
+                            $options['scope'] === 'Event' ? 'event.id' : 'attribute_id'
+                        )
+                    );
+                    $lookup_field = $options['scope'] === 'Event' ? 'Event.id' : 'Attribute.id';
+                    $conditions['AND'][] = array_merge($temp, $this->subQueryGenerator($tag->AttributeTag, $subquery_options, $lookup_field, 1));
+                }
             }
         }
         $temp = array();
@@ -1548,33 +1556,57 @@ class Attribute extends AppModel
         $conditions = array();
         if (!$user['Role']['perm_site_admin']) {
             $sgids = $this->SharingGroup->authorizedIds($user);
-            $eventConditions = $this->Event->createEventConditions($user);
-            $conditions = array(
-                'AND' => array(
-                    $eventConditions['AND'],
-                    array(
-                        'OR' => array(
-                            'Event.org_id' => $user['org_id'],
-                            'Attribute.distribution' => array(1, 2, 3, 5),
-                            'AND '=> array(
-                                'Attribute.distribution' => 4,
-                                'Attribute.sharing_group_id' => $sgids,
-                            )
-                        )
-                    ),
-                    array(
-                        'OR' => array(
-                            'Attribute.object_id' => 0,
-                            'Event.org_id' => $user['org_id'],
-                            'Object.distribution' => array(1, 2, 3, 5),
-                            'AND' => array(
-                                'Object.distribution' => 4,
-                                'Object.sharing_group_id' => $sgids,
-                            )
-                        )
-                    )
-                )
-            );
+            $subQuery1 = [
+                'conditions' => ['org_id' => $user['org_id']],
+                'fields' => ['id']
+            ];
+            $subQuery2 = [
+                'conditions' => [
+                    'distribution IN' => [1, 2, 3]
+                ],
+                'fields' => ['id']
+            ];
+            $subQuery3 = [
+                'conditions' => [
+                    'Event.distribution' => 4,
+                    'Event.sharing_group_id IN' => $sgids
+                ],
+                'fields' => ['id']
+            ];
+            if (Configure::read('MISP.unpublishedprivate')) {
+                $subQuery2['conditions']['Event.published'] = 1;
+                $subQuery3['conditions']['Event.published'] = 1;
+            }
+            $conditions = [
+                'OR' => [
+                    $this->subQueryGenerator($this->Event, $subQuery1, 'Attribute.event_id'),
+                    'AND' => [
+                        'OR' => [
+                            $this->subQueryGenerator($this->Event, $subQuery2, 'Attribute.event_id'),
+                            $this->subQueryGenerator($this->Event, $subQuery3, 'Attribute.event_id')
+                        ],
+                        [
+                            'OR' => [
+                                'Attribute.distribution' => [1, 2, 3, 5],
+                                'AND '=> [
+                                    'Attribute.distribution' => 4,
+                                    'Attribute.sharing_group_id' => $sgids,
+                                ]
+                            ]
+                        ],
+                        [
+                            'OR' => [
+                                'Attribute.object_id' => 0,
+                                'Object.distribution' => [1, 2, 3, 5],
+                                'AND' => [
+                                    'Object.distribution' => 4,
+                                    'Object.sharing_group_id' => $sgids,
+                                ]
+                            ]
+                        ]
+                    ]
+                ]
+            ];
         }
         return $conditions;
     }
@@ -1787,7 +1819,7 @@ class Attribute extends AppModel
         if (isset($options['fields'])) {
             $params['fields'] = $options['fields'];
         }
-        if (isset($options['conditions'])) {
+        if (!empty($options['conditions'])) {
             $params['conditions']['AND'][] = $options['conditions'];
         }
         if (empty($options['flatten'])) {
@@ -1883,11 +1915,10 @@ class Attribute extends AppModel
                 return [];
             }
         }
-
         $eventTags = []; // tag cache
         $attributes = [];
+        $params['ignoreIndexHint'] = 'deleted';
         do {
-            $continue = true;
             $results = $this->find('all', $params);
             if (empty($results)) {
                 break;
@@ -2621,7 +2652,8 @@ class Attribute extends AppModel
                 $this->id = $attribute['id'];
             }
         }
-        if (!$this->save(['Attribute' => $attribute], $params)) {
+        $savedAttribute = $this->save(['Attribute' => $attribute], $params);
+        if (!$savedAttribute) {
             $this->logDropped($user, $attribute);
         } else {
             if (!empty($attribute['AttributeTag'])) {
@@ -2659,7 +2691,7 @@ class Attribute extends AppModel
             if (!empty($attribute['Sighting'])) {
                 $this->Sighting->captureSightings($attribute['Sighting'], $this->id, $eventId, $user);
             }
-            $this->Event->captureAnalystData($user, $attribute);
+            $this->Event->captureAnalystData($user, $attribute, 'Attribute', $savedAttribute['Attribute']['uuid']);
         }
         if (!empty($this->validationErrors)) {
             $validationErrors = $this->validationErrors;
@@ -2800,7 +2832,7 @@ class Attribute extends AppModel
             if (!empty($attribute['Sighting'])) {
                 $this->Sighting->captureSightings($attribute['Sighting'], $attributeId, $eventId, $user);
             }
-            $this->Event->captureAnalystData($user, $attribute);
+            $this->Event->captureAnalystData($user, $attribute, 'Attribute', $attribute['uuid']);
             if ($user['Role']['perm_tagger']) {
                 /*
                     We should unwrap the line below and remove the server option in the future once we have tag soft-delete
@@ -2865,7 +2897,7 @@ class Attribute extends AppModel
             if (!empty($tagActions['attach'])) {
                 $this->AttributeTag->saveMany($tagActions['attach']);
             }
-            
+
         }
         if (!empty($tagActions['detach'])) {
             foreach ($tagActions['detach'] as $detach) {
@@ -2883,7 +2915,7 @@ class Attribute extends AppModel
                     $this->Correlation->generateCorrelation(false, $event['Event']['id'], $attributeId);
                 }
             }
-            // Instead of incrementing / decrementing the event 
+            // Instead of incrementing / decrementing the event
             $attribute_count = $this->find('count', [
                 'conditions' => [
                     'Attribute.event_id' => $event['Event']['id'],
@@ -2900,7 +2932,7 @@ class Attribute extends AppModel
         }
         return true;
     }
-    
+
 
     /**
      * @param int $id Attribute ID
@@ -2980,9 +3012,15 @@ class Attribute extends AppModel
         return $adata;
     }
 
-    public function buildFilterConditions(array $user, array &$params)
+    public function buildFilterConditions(array $user, array &$params, $skipBuildConditions = false)
     {
-        $conditions = $this->buildConditions($user);
+        // in some cases we'll build the user ACL conditions elsewhere,
+        // for example when calling this function via restsearch
+        if ($skipBuildConditions) {
+            $conditions = [];
+        } else {
+            $conditions = $this->buildConditions($user);
+        }
         if (isset($params['wildcard'])) {
             $temp = array();
             $options = array(
@@ -3106,7 +3144,7 @@ class Attribute extends AppModel
         $subqueryElements = $this->Event->harvestSubqueryElements($filters);
         $filters = $this->Event->addFiltersFromSubqueryElements($filters, $subqueryElements, $user);
         $filters = $this->Event->addFiltersFromUserSettings($user, $filters);
-        $conditions = $this->buildFilterConditions($user, $filters);
+        $conditions = $this->buildFilterConditions($user, $filters, true);
         $params = array(
             'conditions' => $conditions,
             'fields' => array('Attribute.*', 'Event.org_id', 'Event.distribution'),
@@ -3126,6 +3164,7 @@ class Attribute extends AppModel
             'includeFullModel' => !empty($filters['includeFullModel']) ? $filters['includeFullModel'] : 0,
             'allow_proposal_blocking' => !empty($filters['allow_proposal_blocking']) ? $filters['allow_proposal_blocking'] : 0
         );
+
         if (!empty($filters['attackGalaxy'])) {
             $params['attackGalaxy'] = $filters['attackGalaxy'];
         }
@@ -3351,20 +3390,60 @@ class Attribute extends AppModel
         if (!empty($params['uuid'])) {
             $params['uuid'] = $this->convert_filters($params['uuid']);
             if (!empty($params['uuid']['OR'])) {
-                $conditions['AND'][] = array(
-                    'OR' => array(
-                        'Event.uuid' => $params['uuid']['OR'],
-                        'Attribute.uuid' => $params['uuid']['OR']
-                    )
-                );
+                if ($options['scope'] == 'Attribute') {
+                    $subQuery = [
+                        'conditions' => ['uuid' => $params['uuid']['OR']],
+                        'fields' => ['id']
+                    ];
+                    $pre_lookup = $this->Event->find('first', [
+                        'conditions' => ['Event.uuid' => $params['uuid']['OR']],
+                        'recursive' => -1,
+                        'fields' => ['Event.id']
+                    ]);
+                    if (empty($pre_lookup)) {
+                        $conditions['AND'][] = array(
+                            'OR' => array(
+                                'Attribute.uuid' => $params['uuid']['OR']
+                            )
+                        );
+                    } else {
+                        $conditions['AND'][] = array(
+                            'OR' => array(
+                                $this->subQueryGenerator($this->Event, $subQuery, 'Attribute.event_id'),
+                                'Attribute.uuid' => $params['uuid']['OR']
+                            )
+                        );
+                    }
+                    
+                } else {
+                    $conditions['AND'][] = array(
+                        'OR' => array(
+                            'Event.uuid' => $params['uuid']['OR'],
+                            'Attribute.uuid' => $params['uuid']['OR']
+                        )
+                    );
+                }
             }
             if (!empty($params['uuid']['NOT'])) {
-                $conditions['AND'][] = array(
-                    'NOT' => array(
-                        'Event.uuid' => $params['uuid']['NOT'],
-                        'Attribute.uuid' =>  $params['uuid']['NOT']
-                    )
-                );
+                if ($options['scope'] == 'Attribute') {
+                    $subQuery = [
+                        'conditions' => ['uuid' => $params['uuid']['OR']],
+                        'fields' => ['id']
+                    ];
+                    $conditions['AND'][] = [
+                        'NOT' => [
+                            $this->subQueryGenerator($this->Event, $subQuery, 'Attribute.event_id'),
+                            'Attribute.uuid' =>  $params['uuid']['NOT']
+                        ]
+                    ];
+                } else {
+                    $conditions['AND'][] = array(
+                        'NOT' => array(
+                            'Event.uuid' => $params['uuid']['NOT'],
+                            'Attribute.uuid' =>  $params['uuid']['NOT']
+                        )
+                    );
+                }
             }
         }
         return $conditions;
@@ -3546,7 +3625,7 @@ class Attribute extends AppModel
             ),
             'Other' => array(
                 'desc' => __('Attributes that are not part of any other category or are meant to be used as a component in MISP objects in the future'),
-                'types' => array('comment', 'text', 'other', 'size-in-bytes', 'counter', 'datetime', 'cpe', 'port', 'float', 'hex', 'phone-number', 'boolean', 'anonymised', 'pgp-public-key', 'pgp-private-key')
+                'types' => array('comment', 'text', 'other', 'size-in-bytes', 'counter', 'integer', 'datetime', 'cpe', 'port', 'float', 'hex', 'phone-number', 'boolean', 'anonymised', 'pgp-public-key', 'pgp-private-key')
             )
         );
     }
@@ -3699,6 +3778,7 @@ class Attribute extends AppModel
             'dns-soa-email' => array('desc' => __('RFC 1035 mandates that DNS zones should have a SOA (Statement Of Authority) record that contains an email address where a PoC for the domain could be contacted. This can sometimes be used for attribution/linkage between different domains even if protected by whois privacy'), 'default_category' => 'Attribution', 'to_ids' => 0),
             'size-in-bytes' => array('desc' => __('Size expressed in bytes'), 'default_category' => 'Other', 'to_ids' => 0),
             'counter' => array('desc' => __('An integer counter, generally to be used in objects'), 'default_category' => 'Other', 'to_ids' => 0),
+            'integer' => array('desc' => __('A generic integer generally to be used in objects'), 'default_category' => 'Other', 'to_ids' => 0),
             'datetime' => array('desc' => __('Datetime in the ISO 8601 format'), 'default_category' => 'Other', 'to_ids' => 0),
             'port' => array('desc' => __('Port number'), 'default_category' => 'Network activity', 'to_ids' => 0),
             'ip-dst|port' => array('desc' => __('IP destination and port number separated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1),
@@ -3784,7 +3864,7 @@ class Attribute extends AppModel
         if (isset($attribute['id'])) {
             $conditions['Attribute.id !='] = $attribute['id'];
         }
-        
+
         return $this->find('first', [
             'recursive' => -1,
             'conditions' => $conditions,

app/Model/Behavior/AnalystDataBehavior.php

@@ -9,6 +9,8 @@ class AnalystDataBehavior extends ModelBehavior
 
     private $__current_type = null;
 
+    protected $__valid_sharing_groups = null;
+
     public function setup(Model $Model, $settings = array()) {
         // We want to know whether we're a Note, Opinion or Relationship
         $this->__current_type = $Model->alias;
@@ -22,7 +24,9 @@ class AnalystDataBehavior extends ModelBehavior
         ];
         $type = $Model->current_type;
         if (empty($user['Role']['perm_site_admin'])) {
-            $validSharingGroups = $Model->SharingGroup->authorizedIds($user, true);
+            if ($this->__valid_sharing_groups === null) {
+                $this->__valid_sharing_groups = $Model->SharingGroup->authorizedIds($user, true);
+            }
             $conditions['AND'][] = [
                 'OR' => [
                     $type . '.orgc_uuid' => $user['Organisation']['uuid'],
@@ -30,7 +34,7 @@ class AnalystDataBehavior extends ModelBehavior
                     $type . '.distribution IN' => [1, 2, 3],
                     'AND' => [
                         $type . '.distribution' => 4,
-                        $type . '.sharing_group_id IN' => $validSharingGroups
+                        $type . '.sharing_group_id IN' => $this->__valid_sharing_groups
                     ]
                 ]
             ];
@@ -42,6 +46,41 @@ class AnalystDataBehavior extends ModelBehavior
         ]);
     }
 
+        // Return the analystData of the current type for a given UUID (this only checks the ACL of the analystData, NOT of the parent.)
+        public function fetchForUuids(Model $Model, $uuids, $user = null)
+        {
+            $conditions = [
+                'object_uuid' => $uuids
+            ];
+            $type = $Model->current_type;
+            if (empty($user['Role']['perm_site_admin'])) {
+                if ($this->__valid_sharing_groups === null) {
+                    $this->__valid_sharing_groups = $Model->SharingGroup->authorizedIds($user, true);
+                }
+                $conditions['AND'][] = [
+                    'OR' => [
+                        $type . '.orgc_uuid' => $user['Organisation']['uuid'],
+                        $type . '.org_uuid' => $user['Organisation']['uuid'],
+                        $type . '.distribution IN' => [1, 2, 3],
+                        'AND' => [
+                            $type . '.distribution' => 4,
+                            $type . '.sharing_group_id IN' => $this->__valid_sharing_groups
+                        ]
+                    ]
+                ];
+            }
+            $temp = $Model->find('all', [
+                'recursive' => -1,
+                'conditions' => $conditions,
+                'contain' => ['Org', 'Orgc', 'SharingGroup'],
+            ]);
+            $results = [];
+            foreach ($temp as $result) {
+                $results[$result[$type]['object_uuid']][$type][] = $result[$type];
+            }
+            return $results;
+        }
+
     public function checkACL()
     {
 

app/Model/Behavior/AnalystDataParentBehavior.php

@@ -31,13 +31,79 @@ class AnalystDataParentBehavior extends ModelBehavior
             $temp = $this->{$type}->fetchForUuid($object['uuid'], $this->__currentUser);
             if (!empty($temp)) {
                 foreach ($temp as $k => $temp_element) {
+                    if (in_array($type, ['Note', 'Opinion', 'Relationship'])) {
+                        $temp_element[$type] = $this->{$type}->fetchChildNotesAndOpinions($this->__currentUser, $temp_element[$type], 1);
+                    }
                     $data[$type][] = $temp_element[$type];
                 }
             }
         }
+
+        // include inbound relationship
+        $data['RelationshipInbound'] = Hash::extract($this->Relationship->getInboundRelationships($this->__currentUser, $model->alias, $object['uuid']), '{n}.Relationship');
         return $data;
     }
 
+    public function fetchAnalystDataBulk(Model $model, array $uuids, array $types = ['Note', 'Opinion', 'Relationship']) {
+        $uuids = array_chunk($uuids, 100000);
+        if (empty($this->__currentUser)) {
+            $user_id = Configure::read('CurrentUserId');
+            $this->User = ClassRegistry::init('User');
+            if ($user_id) {
+                $this->__currentUser = $this->User->getAuthUser($user_id);
+            }
+        }
+        $results = [];
+        foreach ($uuids as $uuid_chunk) {
+            foreach ($types as $type) {
+                $this->{$type} = ClassRegistry::init($type);
+                $this->{$type}->fetchRecursive = !empty($model->includeAnalystDataRecursive);
+                $temp = $this->{$type}->fetchForUuids($uuid_chunk, $this->__currentUser);
+                $results = array_merge_recursive($results, $temp);
+            }
+        }
+        return $results;
+    }
+
+    public function attachAnalystDataBulk(Model $model, array $objects, array $types = ['Note', 'Opinion', 'Relationship'])
+    {
+        $uuids = [];
+        $objects = array_chunk($objects, 100000, true);
+        if (empty($this->__currentUser)) {
+            $user_id = Configure::read('CurrentUserId');
+            $this->User = ClassRegistry::init('User');
+            if ($user_id) {
+                $this->__currentUser = $this->User->getAuthUser($user_id);
+            }
+        }
+        foreach ($objects as $chunk => $chunked_objects) {
+            foreach ($chunked_objects as $k => $object) {
+                if (!empty($object['uuid'])) {
+                    $uuids[] = $object['uuid'];
+                }
+            }
+            // No uuids, nothing to attach
+            if (empty($uuids)) {
+                continue;
+            }
+            foreach ($types as $type) {
+                $this->{$type} = ClassRegistry::init($type);
+                $this->{$type}->fetchRecursive = !empty($model->includeAnalystDataRecursive);
+                $temp = $this->{$type}->fetchForUuids($uuids, $this->__currentUser);
+                if (!empty($temp)) {
+                    foreach ($chunked_objects as $k => $object) {
+                        if (!empty($temp[$object['uuid']])) {
+                            $objects[$chunk][$k][$type] = !empty($objects[$chunk][$k][$type]) ? $objects[$chunk][$k][$type] : [];
+                            $objects[$chunk][$k][$type] = array_merge($objects[$chunk][$k][$type], $temp[$object['uuid']][$type]);
+                        }
+                    }
+                }
+            }
+        }
+        $objects = call_user_func_array('array_merge', $objects);
+        return $objects;
+    }
+
     public function afterFind(Model $model, $results, $primary = false)
     {
         if (!empty($model->includeAnalystData)) {

app/Model/Benchmark.php

@@ -0,0 +1,6 @@
+<?php
+App::uses('AppModel', 'Model');
+
+class Benchmark extends AppModel
+{
+}

app/Model/Datasource/Database/MysqlExtended.php

@@ -58,6 +58,7 @@ class MysqlExtended extends Mysql
             'having' => $this->having($query['having'], true, $Model),
             'lock' => $this->getLockingHint($query['lock']),
             'indexHint' => $this->__buildIndexHint($query['forceIndexHint'] ?? null),
+            'ignoreIndexHint' => $this->__buildIgnoreIndexHint($query['ignoreIndexHint'] ?? null)
         ));
     }
 
@@ -90,6 +91,7 @@ class MysqlExtended extends Mysql
                 'having' => $queryData['having'],
                 'lock' => $queryData['lock'],
                 'forceIndexHint' => $queryData['forceIndexHint'] ?? null,
+                'ignoreIndexHint' => $queryData['ignoreIndexHint'] ?? null,
             ),
             $Model
         );
@@ -117,14 +119,25 @@ class MysqlExtended extends Mysql
     }
 
     /**
-     * Builds the index hint for the query
+     * Builds the force index hint for the query
      * 
-     * @param string|null $forceIndexHint FORCE INDEX hint
+     * @param string|null $forceIndexHint INDEX hint
      * @return string
      */
     private function __buildIndexHint($forceIndexHint = null): ?string
     {
-        return isset($forceIndexHint) ? ('FORCE INDEX ' . $forceIndexHint) : null;
+        return isset($forceIndexHint) ? ('FORCE INDEX (' . $forceIndexHint . ')') : null;
+    }
+
+        /**
+     * Builds the ignore index hint for the query
+     * 
+     * @param string|null $ignoreIndexHint INDEX hint
+     * @return string
+     */
+    private function __buildIgnoreIndexHint($ignoreIndexHint = null): ?string
+    {
+        return isset($ignoreIndexHint) ? ('IGNORE INDEX (' . $ignoreIndexHint . ')') : null;
     }
 
     /**
@@ -139,7 +152,9 @@ class MysqlExtended extends Mysql
     public function execute($sql, $options = [], $params = [])
     {
         $log = $options['log'] ?? $this->fullDebug;
-
+        if (Configure::read('Plugin.Benchmarking_enable')) {
+            $log = true;
+        }
         if ($log) {
             $t = microtime(true);
             $this->_result = $this->_execute($sql, $params);
@@ -164,6 +179,10 @@ class MysqlExtended extends Mysql
      */
     public function insertMulti($table, $fields, $values)
     {
+        if (empty($values)) {
+            return true;
+        }
+
         $table = $this->fullTableName($table);
         $holder = substr(str_repeat('?,', count($fields)), 0, -1);
         $fields = implode(',', array_map([$this, 'name'], $fields));

app/Model/Datasource/Database/MysqlObserverExtended.php

@@ -50,6 +50,7 @@ class MysqlObserverExtended extends Mysql
             'having' => $this->having($query['having'], true, $Model),
             'lock' => $this->getLockingHint($query['lock']),
             'indexHint' => $this->__buildIndexHint($query['forceIndexHint'] ?? null),
+            'ignoreIndexHint' => $this->__buildIgnoreIndexHint($query['ignoreIndexHint'] ?? null),
         ));
     }
 
@@ -82,6 +83,7 @@ class MysqlObserverExtended extends Mysql
                 'having' => $queryData['having'],
                 'lock' => $queryData['lock'],
                 'forceIndexHint' => $queryData['forceIndexHint'] ?? null,
+                'ignoreIndexHint' => $queryData['ignoreIndexHint'] ?? null,
             ),
             $Model
         );
@@ -103,20 +105,31 @@ class MysqlObserverExtended extends Mysql
             extract($data);
             $having = !empty($having) ? " $having" : '';
             $lock = !empty($lock) ? " $lock" : '';
-            return rtrim("SELECT {$fields} FROM {$table} {$alias} {$indexHint} {$joins} {$conditions} {$group}{$having} {$order} {$limit}{$lock}");
+            return rtrim("SELECT {$fields} FROM {$table} {$alias} {$indexHint} {$ignoreIndexHint} {$joins} {$conditions} {$group}{$having} {$order} {$limit}{$lock}");
         }
         return parent::renderStatement($type, $data);
     }
 
     /**
-     * Builds the index hint for the query
+     * Builds the force index hint for the query
      * 
-     * @param string|null $forceIndexHint FORCE INDEX hint
+     * @param string|null $forceIndexHint INDEX hint
      * @return string
      */
     private function __buildIndexHint($forceIndexHint = null): ?string
     {
-        return isset($forceIndexHint) ? ('FORCE INDEX ' . $forceIndexHint) : null;
+        return isset($forceIndexHint) ? ('FORCE INDEX (' . $forceIndexHint . ')') : null;
+    }
+
+        /**
+     * Builds the ignore index hint for the query
+     * 
+     * @param string|null $ignoreIndexHint INDEX hint
+     * @return string
+     */
+    private function __buildIgnoreIndexHint($ignoreIndexHint = null): ?string
+    {
+        return isset($ignoreIndexHint) ? ('IGNORE INDEX (' . $ignoreIndexHint . ')') : null;
     }
 
     /**
@@ -131,6 +144,9 @@ class MysqlObserverExtended extends Mysql
     public function execute($sql, $options = [], $params = [])
     {
         $log = $options['log'] ?? $this->fullDebug;
+        if (Configure::read('Plugin.Benchmarking_enable')) {
+            $log = true;
+        }
         $comment = sprintf(
             '%s%s%s',
             empty(Configure::read('CurrentUserId')) ? '' : sprintf(

app/Model/DecayingModel.php

@@ -16,6 +16,9 @@ class DecayingModel extends AppModel
         )
     );
 
+    private $modelCache = [];
+    private $modelCacheForType = [];
+
     private $__registered_model_classes = array(); // Proxy for already instantiated classes
     public $allowed_overrides = array('threshold' => 1, 'lifetime' => 1, 'decay_speed' => 1);
 
@@ -263,6 +266,10 @@ class DecayingModel extends AppModel
     // - full attach Attribute types associated to the requested model
     public function fetchModel($user, $id, $full=true, $conditions=array(), $attach_editable=0)
     {
+        $cacheKey = sprintf('%s', $id);
+        if (isset($this->modelCache[$cacheKey])) {
+            return $this->modelCache[$cacheKey];
+        }
         $conditions['id'] = $id;
         $searchOptions = array(
             'conditions' => $conditions,
@@ -290,6 +297,7 @@ class DecayingModel extends AppModel
             $decayingModel['DecayingModel']['attribute_types'] = $this->DecayingModelMapping->getAssociatedTypes($user, $decayingModel);
         }
         $decayingModel = $this->attachIsEditableByCurrentUser($user, $decayingModel);
+        $this->modelCache[$cacheKey] = $decayingModel;
         return $decayingModel;
     }
 
@@ -612,11 +620,15 @@ class DecayingModel extends AppModel
         if ($model_id === false) { // fetch all allowed and associated models
             $associated_model_ids = $this->DecayingModelMapping->getAssociatedModels($user, $attribute['type'], true);
             $associated_model_ids = isset($associated_model_ids[$attribute['type']]) ? array_values($associated_model_ids[$attribute['type']]) : array();
-            if (!empty($associated_model_ids)) {
+            if (isset($this->modelCacheForType[$attribute['type']])) {
+                $models = $this->modelCacheForType[$attribute['type']];
+            } else if (!empty($associated_model_ids)) {
                 $models = $this->fetchModels($user, $associated_model_ids, false, array('enabled' => true));
+                $this->modelCacheForType[$attribute['type']] = $models;
             }
         } else {
             $models = $this->fetchModels($user, $model_id, false, array());
+            $this->modelCacheForType[$attribute['type']] = $models;
         }
         foreach ($models as $model) {
             if (!empty($model_overrides)) {

app/Model/DecayingModelMapping.php

@@ -25,6 +25,8 @@ class DecayingModelMapping extends AppModel
         )
     );
 
+    private $modelCache = [];
+
     public function resetMappingForModel($new_model, $user) {
         if (empty($new_model['model_id'])) {
             throw new NotFoundException(__('No Decaying Model with the provided ID exists'));
@@ -76,6 +78,10 @@ class DecayingModelMapping extends AppModel
     }
 
     public function getAssociatedModels($user, $attribute_type = false) {
+        $cacheKey = sprintf('%s', $attribute_type);
+        if (isset($this->modelCache[$cacheKey])) {
+            return $this->modelCache[$cacheKey];
+        }
         $conditions = array(
             'OR' => array(
                 'DecayingModel.org_id' => $user['org_id'],
@@ -111,6 +117,7 @@ class DecayingModelMapping extends AppModel
         }
         $associated_models = Hash::combine($associated_models, '{n}.DecayingModelMapping.model_id', '{n}.DecayingModelMapping.model_id', '{n}.DecayingModelMapping.attribute_type');
         $models = array_merge_recursive($associated_default_models, $associated_models);
+        $this->modelCache[$cacheKey] = $models;
         return $models;
     }
 

app/Model/DecayingModelsFormulas/Base.php

@@ -133,9 +133,9 @@ abstract class DecayingModelBase
         }
         if ($last_sighting_timestamp === false) {
             $this->Sighting = ClassRegistry::init('Sighting');
-            $all_sightings = $this->Sighting->listSightings($user, $attribute['id'], 'attribute', false, 0, true);
-            if (!empty($all_sightings)) {
-                $last_sighting_timestamp = $all_sightings[0]['Sighting']['date_sighting'];
+            $last_sighting = $this->Sighting->getLastSightingForAttribute($user, $attribute['id']);
+            if (!empty($last_sighting)) {
+                $last_sighting_timestamp = $last_sighting['Sighting']['date_sighting'];
             } elseif (!is_null($attribute['last_seen'])) {
                 $last_sighting_timestamp = (new DateTime($attribute['last_seen']))->format('U');
             } else {

app/Model/Event.php

@@ -1058,7 +1058,11 @@ class Event extends AppModel
         // prepare attribute for sync
         if (!empty($data['Attribute'])) {
             foreach ($data['Attribute'] as $key => $attribute) {
-                if (!empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) && in_array($attribute['type'], $pushRules['type_attributes']['NOT'])) {
+                if (
+                    !empty(Configure::read('MISP.enable_synchronisation_filtering_on_type')) &&
+                    !empty($pushRules['type_attributes']['NOT']) &&
+                    in_array($attribute['type'], $pushRules['type_attributes']['NOT'])
+                ) {
                     unset($data['Attribute'][$key]);
                     continue;
                 }
@@ -1406,14 +1410,13 @@ class Event extends AppModel
         return $this->delete(null, false);
     }
 
-    public function createEventConditions($user)
+    public function createEventConditions($user, $skip_own_event_rule = false)
     {
         $conditions = array();
         if (!$user['Role']['perm_site_admin']) {
             $sgids = $this->SharingGroup->authorizedIds($user);
             $unpublishedPrivate = Configure::read('MISP.unpublishedprivate');
             $conditions['AND']['OR'] = [
-                'Event.org_id' => $user['org_id'],
                 [
                     'AND' => [
                         'Event.distribution >' => 0,
@@ -1429,6 +1432,9 @@ class Event extends AppModel
                     ]
                 ]
             ];
+            if (!$skip_own_event_rule) {
+                $conditions['AND']['OR'][] = ['Event.org_id' => $user['org_id']];
+            }
         }
         return $conditions;
     }
@@ -2210,11 +2216,7 @@ class Event extends AppModel
                     $event['Attribute'] = $this->Attribute->Correlation->attachCorrelationExclusion($event['Attribute']);
                 }
                 if (!empty($options['includeAnalystData'])) {
-                    foreach ($event['Attribute'] as $k => $attribute) {
-                        $this->Attribute->includeAnalystDataRecursive = true;
-                        $analyst_data = $this->Attribute->attachAnalystData($attribute);
-                        $event['Attribute'][$k] = array_merge($event['Attribute'][$k], $analyst_data);
-                    }
+                    $event['Attribute'] = $this->Attribute->attachAnalystDataBulk($event['Attribute']);
                 }
 
                 // move all object attributes to a temporary container
@@ -2257,6 +2259,7 @@ class Event extends AppModel
                     }
                 }
                 $event['Attribute'] = array_values($event['Attribute']);
+                unset($attribute);
             }
             if (!empty($event['Object'])) {
                 if (!$sharingGroupReferenceOnly) {
@@ -2266,11 +2269,9 @@ class Event extends AppModel
                     if (isset($tempObjectAttributeContainer[$objectValue['id']])) {
                         $objectValue['Attribute'] = $tempObjectAttributeContainer[$objectValue['id']];
                     }
-                    if (!empty($options['includeAnalystData'])) {
-                        $this->Object->includeAnalystDataRecursive = true;
-                        $analyst_data = $this->Object->attachAnalystData($objectValue);
-                        $objectValue = array_merge($objectValue, $analyst_data);
-                    }
+                }
+                if (!empty($options['includeAnalystData'])) {
+                    $event['Object'] = $this->Object->attachAnalystDataBulk($event['Object']);
                 }
                 unset($tempObjectAttributeContainer);
             }
@@ -2278,7 +2279,9 @@ class Event extends AppModel
                 $event['EventReport'] = $this->__attachSharingGroups($event['EventReport'], $sharingGroupData);
             }
             if (empty($options['metadata']) && empty($options['noSightings'])) {
-                $event['Sighting'] = $this->Sighting->attachToEvent($event, $user);
+                if (empty(Configure::read('MISP.disable_sighting_loading'))) {
+                    $event['Sighting'] = $this->Sighting->attachToEvent($event, $user);
+                }
             }
             if ($options['includeSightingdb']) {
                 $this->Sightingdb = ClassRegistry::init('Sightingdb');
@@ -3281,6 +3284,7 @@ class Event extends AppModel
         $template->set('distributionLevels', $this->distributionLevels);
         $template->set('analysisLevels', $this->analysisLevels);
         $template->set('tlp', $subjMarkingString);
+        $template->set('title', Configure::read('MISP.title_text'));
         $template->subject($subject);
         $template->referenceId("event-alert|{$event['Event']['id']}");
 
@@ -3465,7 +3469,7 @@ class Event extends AppModel
                 if ($tagId && !in_array($tagId, $event_tag_ids)) {
                     $eventTags[] = array(
                         'tag_id' => $tagId,
-                        'local' => isset($tag['local']) ? $tag['local'] : 0,
+                        'local' => isset($tag['local']) ? $tag['local'] : false,
                         'relationship_type' => isset($tag['relationship_type']) ? $tag['relationship_type'] : '',
                     );
                     $event_tag_ids[] = $tagId;
@@ -3481,7 +3485,7 @@ class Event extends AppModel
                 if ($tag_id && !in_array($tag_id, $event_tag_ids)) {
                     $eventTags[] = [
                         'tag_id' => $tag_id,
-                        'local' => isset($tag['local']) ? $tag['local'] : 0,
+                        'local' => isset($tag['local']) ? $tag['local'] : false,
                         'relationship_type' => isset($tag['relationship_type']) ? $tag['relationship_type'] : '',
                     ];
                     $event_tag_ids[] = $tag_id;
@@ -3559,7 +3563,7 @@ class Event extends AppModel
                     if ($tagId) {
                         $attributeTags[] = [
                             'tag_id' => $tagId,
-                            'local' => isset($tag['local']) ? $tag['local'] : 0,
+                            'local' => isset($tag['local']) ? $tag['local'] : false,
                             'relationship_type' => isset($tag['relationship_type']) ? $tag['relationship_type'] : '',
                         ];
                     }
@@ -3679,7 +3683,9 @@ class Event extends AppModel
         }
         if (!empty($event['Event']['Object'])) {
             for ($i=0; $i < count($event['Event']['Object']); $i++) { 
-                $event['Event']['Object'][$i] = $this->updatedLockedFieldForAnalystData($event['Event']['Object'][$i]);
+                 if (isset($event['Event']['Object'][$i])) {
+                    $event['Event']['Object'][$i] = $this->updatedLockedFieldForAnalystData($event['Event']['Object'][$i]);
+                }
                 if (!empty($event['Event']['Object'][$i])) {
                     for ($j=0; $j < count($event['Event']['Object'][$i]['Attribute']); $j++) { 
                         $event['Event']['Object'][$i]['Attribute'][$j] = $this->updatedLockedFieldForAnalystData($event['Event']['Object'][$i]['Attribute'][$j]);
@@ -3960,7 +3966,7 @@ class Event extends AppModel
                 $this->Sighting->captureSightings($data['Sighting'], null, $this->id, $user);
             }
 
-            $this->captureAnalystData($user, $data['Event']);
+            $this->captureAnalystData($user, $data['Event'], 'Event', $saveResult['Event']['uuid']);
             if ($fromXml) {
                 $created_id = $this->id;
             }
@@ -4263,7 +4269,7 @@ class Event extends AppModel
                 $this->Sighting->captureSightings($data['Sighting'], null, $this->id, $user);
             }
 
-            $this->captureAnalystData($user, $data['Event']);
+            $this->captureAnalystData($user, $data['Event'], 'Event', $saveResult['Event']['uuid']);
             // if published -> do the actual publishing
             if ($changed && (!empty($data['Event']['published']) && 1 == $data['Event']['published'])) {
                 // The edited event is from a remote server ?
@@ -5000,6 +5006,9 @@ class Event extends AppModel
                 $include = $include && ($filterType['correlation'] == 1);
             } else { // `exclude`
                 $include = $include && ($filterType['correlation'] == 2);
+                if (!empty($attribute['over_correlation'])) {
+                    $include = false;
+                }
             }
 
             if ($filterType['correlationId'] && $include) {
@@ -5581,6 +5590,7 @@ class Event extends AppModel
             $passedArgs['page'] = 0;
         }
         $params = $customPagination->applyRulesOnArray($objects, $passedArgs, 'events', 'category');
+        $objects = $this->attachAnalystDataToViewObjects($objects);
         foreach ($objects as $k => $object) {
             if (isset($referencedByArray[$object['uuid']])) {
                 foreach ($referencedByArray[$object['uuid']] as $objectType => $references) {
@@ -5593,6 +5603,44 @@ class Event extends AppModel
         return $params;
     }
 
+    // take a list of paginated, rearranged objects from the event view generation's viewUI() function
+    // collect all attribute and object uuids from the object list
+    // fetch the related analyst data and inject them back into the object list
+    public function attachAnalystDataToViewObjects($objects)
+    {
+        $attribute_notes = [];
+        $object_notes = [];
+        foreach ($objects as $k => $object) {
+            if ($object['objectType'] === 'object') {
+                $object_notes[] = $object['uuid'];
+                foreach ($object['Attribute'] as $a) {
+                    $attribute_notes[] = $a['uuid'];
+                }
+            } else if ($object['objectType'] === 'attribute') {
+                $attribute_notes[] = $object['uuid'];
+            }
+        }
+        $attribute_notes = $this->Attribute->fetchAnalystDataBulk($attribute_notes);
+        $object_notes = $this->Object->fetchAnalystDataBulk($object_notes);
+        foreach ($objects as $k => $object) {
+            if ($object['objectType'] === 'object') {
+                if (!empty($object_notes[$object['uuid']])) {
+                    $objects[$k] = array_merge($object, $object_notes[$object['uuid']]);
+                }
+                foreach ($object['Attribute'] as $k2 => $a) {
+                    if (!empty($attribute_notes[$a['uuid']])) {
+                        $objects[$k]['Attribute'][$k2] = array_merge($a, $attribute_notes[$a['uuid']]);
+                    }
+                }
+            } else if ($object['objectType'] === 'attribute') {
+                if (!empty($attribute_notes[$object['uuid']])) {
+                    $objects[$k] = array_merge($object, $attribute_notes[$object['uuid']]);
+                }
+            }
+        }
+        return $objects;
+    }
+
     // pass along a json from the server filter rules
     // returns a conditions set to be merged into pagination / event fetch / etc
     public function filterRulesToConditions($rules)
@@ -5968,8 +6016,8 @@ class Event extends AppModel
         } else {
             $event = $this->find('first', array(
                 'recursive' => -1,
-                'conditions' => array('Event.id' => $eventOrEventId),
-                'fields' => ['id', 'info'], // info is required because of SysLogLogableBehavior
+                'conditions' => array('Event.id' => $eventOrEventId)
+                //'fields' => ['id', 'info'], // info is required because of SysLogLogableBehavior
             ));
             if (empty($event)) {
                 return false;
@@ -6066,7 +6114,7 @@ class Event extends AppModel
 
     /**
      * @param string $stixVersion
-     * @param string $file
+     * @param string $file Path to STIX file
      * @param int $distribution
      * @param int|null $sharingGroupId
      * @param bool $galaxiesAsTags
@@ -6124,6 +6172,7 @@ class Event extends AppModel
         try {
             $stdout = ProcessTool::execute($shellCommand, null, true);
         } catch (ProcessException $e) {
+            $this->logException("Could not import $stixVersion file $file", $e);
             $stdout = $e->stdout();
         }
 
@@ -6374,7 +6423,7 @@ class Event extends AppModel
                     unset($data[$dataType . 'Tag'][$k]);
                     continue;
                 }
-                $dataTag['Tag']['local'] = empty($dataTag['local']) ? 0 : 1;
+                $dataTag['Tag']['local'] = empty($dataTag['local']) ? false : true;
                 if (!isset($excludeGalaxy) || !$excludeGalaxy) {
                     if (substr($dataTag['Tag']['name'], 0, strlen('misp-galaxy:')) === 'misp-galaxy:') {
                         $cluster = $this->GalaxyCluster->getCluster($dataTag['Tag']['name'], $user);
@@ -7320,7 +7369,7 @@ class Event extends AppModel
             foreach ($event['EventTag'] as $etk => $eventTag) {
                 $tag = $this->__getCachedTag($eventTag['tag_id'], $justExportable);
                 if ($tag !== null) {
-                    $tag['local'] = empty($eventTag['local']) ? 0 : 1;
+                    $tag['local'] = empty($eventTag['local']) ? false : true;
                     $tag['relationship_type'] = empty($eventTag['relationship_type']) ? null : $eventTag['relationship_type'];
                     $event['EventTag'][$etk]['Tag'] = $tag;
                 } else {
@@ -7335,7 +7384,7 @@ class Event extends AppModel
                     foreach ($attribute['AttributeTag'] as $atk => $attributeTag) {
                         $tag = $this->__getCachedTag($attributeTag['tag_id'], $justExportable);
                         if ($tag !== null) {
-                            $tag['local'] = empty($attributeTag['local']) ? 0 : 1;
+                            $tag['local'] = empty($attributeTag['local']) ? false : true;
                             $tag['relationship_type'] = empty($attributeTag['relationship_type']) ? null : $attributeTag['relationship_type'];
                             $event['Attribute'][$ak]['AttributeTag'][$atk]['Tag'] = $tag;
                         } else {
@@ -8041,7 +8090,7 @@ class Event extends AppModel
         }
     }
 
-    public function captureAnalystData($user, $data)
+    public function captureAnalystData($user, $data, $parentObjectType, $parentObjectUUID)
     {
         $this->Note = ClassRegistry::init('Note');
         $this->Opinion = ClassRegistry::init('Opinion');
@@ -8049,6 +8098,9 @@ class Event extends AppModel
         foreach ($this->Note::ANALYST_DATA_TYPES as $type) {
             if (!empty($data[$type])) {
                 foreach ($data[$type] as $analystData) {
+                    $analystData['note_type_name'] = $type;
+                    $analystData['object_type'] = $parentObjectType;
+                    $analystData['object_uuid'] = $parentObjectUUID;
                     $this->{$type}->captureAnalystData($user, $analystData);
                 }
             }

app/Model/EventDelegation.php

@@ -31,7 +31,10 @@ class EventDelegation extends AppModel
 
     public function transferEvent($delegation, $user)
     {
-        $event = $this->Event->fetchEvent($user, array('eventid' => $delegation['EventDelegation']['event_id']));
+        $event = $this->Event->fetchEvent($user, array(
+            'eventid' => $delegation['EventDelegation']['event_id'],
+            'includeAttachments' => 1
+        ));
         if (empty($event)) {
             throw new NotFoundException('Invalid event.');
         }

app/Model/EventReport.php

@@ -120,7 +120,14 @@ class EventReport extends AppModel
                 __('Validation errors: %s.%sFull report: %s', json_encode($errors), PHP_EOL, json_encode($report['EventReport']))
             );
         } else {
-            $this->Event->captureAnalystData($user, $report);
+            $savedReport = $this->find('first', [
+                'recursive' => -1,
+                'fields' => ['id', 'uuid'],
+                'conditions' => ['id' => $this->id],
+            ]);
+            if ($savedReport) {
+                $this->Event->captureAnalystData($user, $report, 'EventReport', $savedReport['EventReport']['uuid']);
+            }
         }
         return $errors;
     }
@@ -191,7 +198,7 @@ class EventReport extends AppModel
         }
         $errors = $this->saveAndReturnErrors($report, ['fieldList' => self::CAPTURE_FIELDS], $errors);
         if (empty($errors)) {
-            $this->Event->captureAnalystData($user, $report['EventReport']);
+            $this->Event->captureAnalystData($user, $report['EventReport'], 'EventReport', $report['EventReport']['uuid']);
             $this->Event->unpublishEvent($eventId);
         }
         return $errors;
@@ -710,7 +717,7 @@ class EventReport extends AppModel
                 'category' => $typeToCategoryMapping[$complexTypeToolEntry['default_type']][0],
                 'type' => $complexTypeToolEntry['default_type'],
                 'value' => $textToBeReplaced,
-                'to_ids' => $complexTypeToolEntry['to_ids'],
+                'to_ids' => $complexTypeToolEntry['to_ids'] ?? 0,
             ];
             $replacedContent = str_replace($complexTypeToolEntry['original_value'], $textToInject, $replacedContent);
         }

app/Model/Feed.php

@@ -1032,7 +1032,7 @@ class Feed extends AppModel
                 }
             }
         }
-        if ($feed['Feed']['tag_id']) {
+        if ($feed['Feed']['tag_id'] || $feed['Feed']['tag_collection_id']) {
             if (empty($feed['Tag']['name'])) {
                 $feed_tag = $this->Tag->find('first', [
                     'conditions' => [
@@ -1041,23 +1041,42 @@ class Feed extends AppModel
                     'recursive' => -1,
                     'fields' => ['Tag.name', 'Tag.colour', 'Tag.id']
                 ]);
-                $feed['Tag'] = $feed_tag['Tag'];
+                if (!empty($feed_tag)) {
+                    $feed['Tag'] = $feed_tag['Tag'];
+                }
             }
             if (!isset($event['Event']['Tag'])) {
                 $event['Event']['Tag'] = array();
             }
 
-            $feedTag = $this->Tag->find('first', array('conditions' => array('Tag.id' => $feed['Feed']['tag_id']), 'recursive' => -1, 'fields' => array('Tag.name', 'Tag.colour', 'Tag.exportable')));
-            if (!empty($feedTag)) {
-                $found = false;
-                foreach ($event['Event']['Tag'] as $tag) {
-                    if (strtolower($tag['name']) === strtolower($feedTag['Tag']['name'])) {
-                        $found = true;
-                        break;
-                    }
+            if (!empty($feed['Feed']['tag_collection_id'])) {
+                $this->TagCollection = ClassRegistry::init('TagCollection');
+                $tagCollectionID = $feed['Feed']['tag_collection_id'];
+                $tagCollection = $this->TagCollection->find('first', [
+                    'recursive' => -1,
+                    'conditions' => [
+                        'TagCollection.id' => $tagCollectionID,
+                    ],
+                    'contain' => [
+                        'TagCollectionTag' => ['Tag'],
+                    ]
+                ]);
+                foreach ($tagCollection['TagCollectionTag'] as $collectionTag) {
+                    $event['Event']['Tag'][] = $collectionTag['Tag'];
                 }
-                if (!$found) {
-                    $event['Event']['Tag'][] = $feedTag['Tag'];
+            } else {
+                $feedTag = $this->Tag->find('first', array('conditions' => array('Tag.id' => $feed['Feed']['tag_id']), 'recursive' => -1, 'fields' => array('Tag.name', 'Tag.colour', 'Tag.exportable')));
+                if (!empty($feedTag)) {
+                    $found = false;
+                    foreach ($event['Event']['Tag'] as $tag) {
+                        if (strtolower($tag['name']) === strtolower($feedTag['Tag']['name'])) {
+                            $found = true;
+                            break;
+                        }
+                    }
+                    if (!$found) {
+                        $event['Event']['Tag'][] = $feedTag['Tag'];
+                    }
                 }
             }
         }
@@ -1068,6 +1087,9 @@ class Feed extends AppModel
             if (!empty($feed['Feed']['settings']['disable_correlation'])) {
                 $event['Event']['disable_correlation'] = (bool) $feed['Feed']['settings']['disable_correlation'];
             }
+            if (!empty($feed['Feed']['settings']['unpublish_event'])) {
+                $event['Event']['published'] = (bool) $feed['Feed']['settings']['unpublish_event'];
+            }
         }
         return $event;
     }
@@ -1128,9 +1150,13 @@ class Feed extends AppModel
      */
     private function __updateEventFromFeed(HttpSocket $HttpSocket = null, $feed, $uuid, $user, $filterRules)
     {
-        $event = $this->downloadAndParseEventFromFeed($feed, $uuid, $HttpSocket);
+         $event = $this->downloadAndParseEventFromFeed($feed, $uuid, $HttpSocket);
         $event = $this->__prepareEvent($event, $feed, $filterRules);
-        return $this->Event->_edit($event, $user, $uuid, $jobId = null);
+        if (is_array($event)) {
+            return $this->Event->_edit($event, $user, $uuid, $jobId = null);
+        } else {
+            return $event;
+        }
     }
 
     public function addDefaultFeeds($newFeeds)
@@ -1374,8 +1400,25 @@ class Feed extends AppModel
         if ($feed['Feed']['publish']) {
             $this->Event->publishRouter($event['Event']['id'], null, $user);
         }
-        if ($feed['Feed']['tag_id']) {
-            $this->Event->EventTag->attachTagToEvent($event['Event']['id'], ['id' => $feed['Feed']['tag_id']]);
+        if ($feed['Feed']['tag_id'] || $feed['Feed']['tag_collection_id']) {
+            if (!empty($feed['Feed']['tag_collection_id'])) {
+                $this->TagCollection = ClassRegistry::init('TagCollection');
+                $tagCollectionID = $feed['Feed']['tag_collection_id'];
+                $tagCollection = $this->TagCollection->find('first', [
+                    'recursive' => -1,
+                    'conditions' => [
+                        'TagCollection.id' => $tagCollectionID,
+                    ],
+                    'contain' => [
+                        'TagCollectionTag',
+                    ]
+                ]);
+                foreach ($tagCollection['TagCollectionTag'] as $collectionTag) {
+                    $this->Event->EventTag->attachTagToEvent($event['Event']['id'], ['id' => $collectionTag['tag_id']]);
+                }
+            } else {
+                $this->Event->EventTag->attachTagToEvent($event['Event']['id'], ['id' => $feed['Feed']['tag_id']]);
+            }
         }
         return true;
     }

app/Model/GalaxyCluster.php

@@ -197,7 +197,7 @@ class GalaxyCluster extends AppModel
      */
     public function arrangeData($cluster)
     {
-        $models = array('Galaxy', 'SharingGroup', 'GalaxyElement', 'GalaxyClusterRelation', 'Org', 'Orgc', 'TargetingClusterRelation', 'Note', 'Opinion', 'Relationship');
+        $models = array('Galaxy', 'SharingGroup', 'GalaxyElement', 'GalaxyClusterRelation', 'Org', 'Orgc', 'TargetingClusterRelation', 'Note', 'Opinion', 'Relationship', 'RelationshipInbound');
         foreach ($models as $model) {
             if (isset($cluster[$model])) {
                 $cluster['GalaxyCluster'][$model] = $cluster[$model];
@@ -1845,6 +1845,9 @@ class GalaxyCluster extends AppModel
         if (!$compatible) {
             return 0;
         }
+
+        $serverSync->debug("Pulling galaxy clusters with technique $technique");
+
         $clusterIds = $this->getClusterIdListBasedOnPullTechnique($user, $technique, $serverSync);
         $successes = 0;
         // now process the $clusterIds to pull each of the events sequentially

app/Model/Log.php

@@ -38,6 +38,7 @@ class Log extends AppModel
                     'add',
                     'admin_email',
                     'attachTags',
+                    'attachTagToObject',
                     'auth',
                     'auth_fail',
                     'auth_alert',
@@ -78,6 +79,7 @@ class Log extends AppModel
                     'registration',
                     'registration_error',
                     'remove_dead_workers',
+                    'removeTagFromObject',
                     'request',
                     'request_delegation',
                     'reset_auth_key',
@@ -257,6 +259,11 @@ class Log extends AppModel
             $change = implode(", ", $output);
         }
 
+        // If Sentry is installed, send log breadcrumb to Sentry
+        if (function_exists('\Sentry\addBreadcrumb')) {
+            \Sentry\addBreadcrumb('log', $title, [], Sentry\Breadcrumb::LEVEL_INFO);
+        }
+
         $this->create();
         $result = $this->save(['Log' => [
             'org' => $user['Organisation']['name'],
@@ -429,15 +436,14 @@ class Log extends AppModel
                 }
             }
 
-            $entry = $data['Log']['action'];
-            if (!empty($data['Log']['title'])) {
-                $entry .= " -- {$data['Log']['title']}";
-            }
-            if (!empty($data['Log']['description'])) {
-                $entry .= " -- {$data['Log']['description']}";
-            } else if (!empty($data['Log']['change'])) {
-                $entry .= " -- " . JsonTool::encode($data['Log']['change']);
-            }
+            $entry = sprintf(
+                '%s -- %s -- %s',
+                $data['Log']['action'],
+                empty($data['Log']['title']) ? '' : $formatted_title = preg_replace('/\s+/', " ", $data['Log']['title']),
+                empty($data['Log']['description']) ? 
+                    (empty($data['Log']['change']) ? '' : preg_replace('/\s+/', " ", $data['Log']['change'])) :
+                    preg_replace('/\s+/', " ", $data['Log']['description'])
+            );
             $this->syslog->write($action, $entry);
         }
     }

app/Model/MispObject.php

@@ -510,7 +510,8 @@ class MispObject extends AppModel
             }
         }
         $this->create();
-        if ($this->save($object)) {
+        $saveResult = $this->save($object);
+        if ($saveResult) {
             $result = $this->id;
             foreach ($object['Attribute'] as $k => $attribute) {
                 $object['Attribute'][$k]['object_id'] = $this->id;
@@ -528,6 +529,7 @@ class MispObject extends AppModel
                 }
             }
             $this->Attribute->saveAttributes($object['Attribute'], $user);
+            $this->Event->captureAnalystData($user, $object['Object'], 'Object', $saveResult['Object']['uuid']);
         } else {
             $result = $this->validationErrors;
         }
@@ -1001,6 +1003,8 @@ class MispObject extends AppModel
             return $this->validationErrors;
         }
 
+        $this->Event->captureAnalystData($user, $objectToSave['Object'], 'Object', $saveResult['Object']['uuid']);
+
         if (!$onlyAddNewAttribute) {
             $checkFields = array('category', 'value', 'to_ids', 'distribution', 'sharing_group_id', 'comment', 'disable_correlation', 'first_seen', 'last_seen');
             if (!empty($objectToSave['Attribute'])) {
@@ -1139,7 +1143,7 @@ class MispObject extends AppModel
                 $this->Attribute->captureAttribute($attribute, $eventId, $user, $objectId, false, $parentEvent);
             }
         }
-        $this->Event->captureAnalystData($user, $object['Object']);
+        $this->Event->captureAnalystData($user, $object['Object'], 'Object', $object['Object']['uuid']);
         return true;
     }
 
@@ -1219,7 +1223,7 @@ class MispObject extends AppModel
             );
             return $this->validationErrors;
         }
-        $this->Event->captureAnalystData($user, $object);
+        $this->Event->captureAnalystData($user, $object, 'Object', $object['uuid']);
         if (!empty($object['Attribute'])) {
             $attributes = [];
             foreach ($object['Attribute'] as $attribute) {

app/Model/Relationship.php

@@ -49,7 +49,7 @@ class Relationship extends AnalystData
             }
         }
         foreach ($results as $i => $v) {
-            if (!empty($v[$this->alias]['related_object_type']) && !empty($v[$this->alias]['related_object_uuid'])) {
+            if (!empty($v[$this->alias]['related_object_type']) && !empty($v[$this->alias]['related_object_uuid']) && empty($results[$i][$this->alias]['related_object'])) {
                 $results[$i][$this->alias]['related_object'] = $this->getRelatedElement($this->__currentUser, $v[$this->alias]['related_object_type'], $v[$this->alias]['related_object_uuid']);
             }
         }
@@ -146,4 +146,41 @@ class Relationship extends AnalystData
         }
         return $data;
     }
+
+    public function getInboundRelationships(array $user, $object_type, $object_uuid): array
+    {
+        $conditions = [
+            'related_object_type' => $object_type,
+            'related_object_uuid' => $object_uuid,
+        ];
+        $type = 'Relationship';
+        if (empty($user['Role']['perm_site_admin'])) {
+            if ($this->__valid_sharing_groups === null) {
+                $this->__valid_sharing_groups = $this->SharingGroup->authorizedIds($user, true);
+            }
+            $conditions['AND'][] = [
+                'OR' => [
+                    $type . '.orgc_uuid' => $user['Organisation']['uuid'],
+                    $type . '.org_uuid' => $user['Organisation']['uuid'],
+                    $type . '.distribution IN' => [1, 2, 3],
+                    'AND' => [
+                        $type . '.distribution' => 4,
+                        $type . '.sharing_group_id IN' => $this->__valid_sharing_groups
+                    ]
+                ]
+            ];
+        }
+        $inboundRelations = $this->find('all', [
+            'recursive' => -1,
+            'conditions' => $conditions,
+            'contain' => ['Org', 'Orgc', 'SharingGroup'],
+        ]);
+
+        foreach ($inboundRelations as $i => $relationship) {
+            $relationship = $relationship['Relationship'];
+            $inboundRelations[$i]['Relationship']['related_object'] = $this->getRelatedElement($this->__currentUser, $relationship['object_type'], $relationship['object_uuid']);
+        }
+
+        return $inboundRelations;
+    }
 }

app/Model/Server.php

@@ -580,7 +580,17 @@ class Server extends AppModel
             }
             return false;
         }
-        $this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $response);
+        try {
+            $this->__checkIfPulledEventExistsAndAddOrUpdate($event, $eventId, $successes, $fails, $eventModel, $serverSync->server(), $user, $jobId, $force, $response);
+        } catch (Exception $e) {
+            $title = __('Pulling an event (#%s) from Server #%s has failed. The sync process was not interrupted.', $eventId, $serverSync->serverId());
+            $this->loadLog()->createLogEntry(
+                $user,
+                'error',
+                'Server',
+                $serverSync->serverId(),
+                $title, $e->getMessage());
+        }
         return true;
     }
 
@@ -594,6 +604,7 @@ class Server extends AppModel
      * @throws HttpSocketHttpException
      * @throws HttpSocketJsonException
      * @throws JsonException
+     * @throws Exception
      */
     public function pull(array $user, $technique, array $server, $jobId = false, $force = false)
     {
@@ -609,7 +620,7 @@ class Server extends AppModel
         try {
             $server['Server']['version'] = $serverSync->info()['version'];
         } catch (Exception $e) {
-            $this->logException("Could not get remote server `{$server['Server']['name']}` version.", $e);
+            $this->logException("Could not get remote server `{$serverSync->serverName()}` version.", $e);
             if ($e instanceof HttpSocketHttpException && $e->getCode() === 403) {
                 $message = __('Not authorised. This is either due to an invalid auth key, or due to the sync user not having authentication permissions enabled on the remote server. Another reason could be an incorrect sync server setting.');
             } else {
@@ -638,6 +649,8 @@ class Server extends AppModel
             }
         }
 
+        $serverSync->debug("Pulling event list with technique $technique");
+
         try {
             $eventIds = $this->__getEventIdListBasedOnPullTechnique($technique, $serverSync, $force);
         } catch (Exception $e) {
@@ -663,26 +676,29 @@ class Server extends AppModel
                 $job->saveProgress($jobId, __n('Pulling %s event.', 'Pulling %s events.', count($eventIds), count($eventIds)));
             }
             foreach ($eventIds as $k => $eventId) {
+                $serverSync->debug("Pulling event $eventId");
                 $this->__pullEvent($eventId, $successes, $fails, $eventModel, $serverSync, $user, $jobId, $force);
                 if ($jobId && $k % 10 === 0) {
                     $job->saveProgress($jobId, null, 10 + 40 * (($k + 1) / count($eventIds)));
                 }
             }
             foreach ($fails as $eventid => $message) {
-                $this->loadLog()->createLogEntry($user, 'pull', 'Server', $server['Server']['id'], "Failed to pull event #$eventid.", 'Reason: ' . $message);
+                $this->loadLog()->createLogEntry($user, 'pull', 'Server', $serverSync->serverId(), "Failed to pull event #$eventid.", 'Reason: ' . $message);
             }
         }
         if ($jobId) {
             $job->saveProgress($jobId, 'Pulling proposals.', 50);
         }
-        $pulledProposals = $pulledSightings = 0;
+        $pulledProposals = $pulledSightings = $pulledAnalystData = 0;
         if ($technique === 'full' || $technique === 'update') {
             $pulledProposals = $eventModel->ShadowAttribute->pullProposals($user, $serverSync);
 
             if ($jobId) {
                 $job->saveProgress($jobId, 'Pulling sightings.', 75);
             }
+
             $pulledSightings = $eventModel->Sighting->pullSightings($user, $serverSync);
+
             $this->AnalystData = ClassRegistry::init('AnalystData');
             $pulledAnalystData = $this->AnalystData->pull($user, $serverSync);
         }
@@ -809,7 +825,7 @@ class Server extends AppModel
      */
     public function getElligibleClusterIdsFromServerForPull(ServerSyncTool $serverSync, $onlyUpdateLocalCluster=true, array $eligibleClusters=array(), array $conditions=array())
     {
-        $this->log("Fetching eligible clusters from server #{$serverSync->serverId()} for pull: " . JsonTool::encode($conditions), LOG_INFO);
+        $serverSync->debug("Fetching eligible clusters for pull: " . JsonTool::encode($conditions));
 
         if ($onlyUpdateLocalCluster && empty($eligibleClusters)) {
             return []; // no clusters for update
@@ -865,7 +881,7 @@ class Server extends AppModel
      */
     private function getElligibleClusterIdsFromServerForPush(ServerSyncTool $serverSync, array $localClusters=array(), array $conditions=array())
     {
-        $this->log("Fetching eligible clusters from server #{$serverSync->serverId()} for push: " . JsonTool::encode($conditions), LOG_INFO);
+        $serverSync->debug("Fetching eligible clusters for push: " . JsonTool::encode($conditions));
         $clusterArray = $this->fetchCustomClusterIdsFromServer($serverSync, $conditions=$conditions);
         $keyedClusterArray = Hash::combine($clusterArray, '{n}.GalaxyCluster.uuid', '{n}.GalaxyCluster.version');
         if (!empty($localClusters)) {
@@ -905,9 +921,14 @@ class Server extends AppModel
 
         // Fetch event index from cache if exists and is not modified
         $redis = RedisTool::init();
-        $indexFromCache = $redis->get("misp:event_index:{$serverSync->serverId()}");
+        $indexFromCache = $redis->get("misp:event_index_cache:{$serverSync->serverId()}");
         if ($indexFromCache) {
-            list($etag, $eventIndex) = RedisTool::deserialize(RedisTool::decompress($indexFromCache));
+            $etagPos = strpos($indexFromCache, "\n");
+            if ($etagPos === false) {
+                throw new RuntimeException("Could not find etag in cache fro server {$serverSync->serverId()}");
+            }
+            $etag = substr($indexFromCache, 0, $etagPos);
+            $serverSync->debug("Event index loaded from Redis cache with etag $etag containing");
         } else {
             $etag = '""';  // Provide empty ETag, so MISP will compute ETag for returned data
         }
@@ -915,9 +936,21 @@ class Server extends AppModel
         $response = $serverSync->eventIndex($filterRules, $etag);
 
         if ($response->isNotModified() && $indexFromCache) {
-            return $eventIndex;
+            return JsonTool::decode(RedisTool::decompress(substr($indexFromCache, $etagPos + 1)));
         }
 
+        // Save to cache for 24 hours if ETag provided
+        $etag = $response->getHeader('etag');
+        if ($etag) {
+            $serverSync->debug("Event index from remote server has different etag $etag, saving to cache");
+            $data = "$etag\n" . RedisTool::compress($response->body);
+            $redis->setex("misp:event_index_cache:{$serverSync->serverId()}", 3600 * 24, $data);
+        } elseif ($indexFromCache) {
+            RedisTool::unlink($redis, "misp:event_index_cache:{$serverSync->serverId()}");
+        }
+
+        unset($indexFromCache); // clean up memory
+
         $eventIndex = $response->json();
 
         // correct $eventArray if just one event, probably this response returns old MISP
@@ -925,15 +958,6 @@ class Server extends AppModel
             $eventIndex = [$eventIndex];
         }
 
-        // Save to cache for 24 hours if ETag provided
-        $etag = $response->getHeader('etag');
-        if ($etag) {
-            $data = RedisTool::compress(RedisTool::serialize([$etag, $eventIndex]));
-            $redis->setex("misp:event_index:{$serverSync->serverId()}", 3600 * 24, $data);
-        } elseif ($indexFromCache) {
-            RedisTool::unlink($redis, "misp:event_index:{$serverSync->serverId()}");
-        }
-
         return $eventIndex;
     }
 
@@ -1362,7 +1386,7 @@ class Server extends AppModel
             return []; // pushing clusters is not enabled
         }
 
-        $this->log("Starting $technique clusters sync with server #{$serverSync->serverId()}", LOG_INFO);
+        $serverSync->debug("Starting $technique clusters sync");
 
         $this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
         $this->Event = ClassRegistry::init('Event');
@@ -4722,15 +4746,18 @@ class Server extends AppModel
         return $servers;
     }
 
+    /**
+     * @return Generator[string, array]
+     */
     public function updateJSON()
     {
-        $results = array();
         foreach (['Galaxy', 'Noticelist', 'Warninglist', 'Taxonomy', 'ObjectTemplate', 'ObjectRelationship'] as $target) {
             $model = ClassRegistry::init($target);
+            $start = microtime(true);
             $result = $model->update();
-            $results[$target] = $result === false ? false : true;
+            $duration = microtime(true) - $start;
+            yield $target => ['success' => $result !== false, 'result' => $result, 'duration' => $duration];
         }
-        return $results;
     }
 
     public function resetRemoteAuthKey($id)
@@ -5113,6 +5140,22 @@ class Server extends AppModel
                     'type' => 'numeric',
                     'null' => true
                 ),
+                'curl_request_timeout' => [
+                    'level' => 1,
+                    'description' => __('Control the default timeout in seconds of curl HTTP requests issued by MISP (during synchronisation, feed fetching, etc.)'),
+                    'value' => 300,
+                    'test' => 'testForNumeric',
+                    'type' => 'numeric',
+                    'null' => true
+                ],
+                'disable_sighting_loading' => [
+                    'level' => 1,
+                    'description' => __('If an instance has an extremely high number of sightings, including the sightings in the search algorithms can bring an instance to a grinding halt. Enable this setting to temporarily disable the search until the issue is remedied. This setting will also disable sightings from being attached via /events/view API calls.'),
+                    'value' => false,
+                    'test' => 'testBoolFalse',
+                    'type' => 'boolean',
+                    'null' => true
+                ],
                 'disable_event_locks' => [
                     'level' => 1,
                     'description' => __('Disable the event locks that are executed periodically when a user browses an event view. It can be useful to leave event locks enabled to warn users that someone else is editing the same event, but generally it\'s extremely verbose and can cause issues in certain setups, so it\'s recommended to disable this.'),
@@ -5390,6 +5433,13 @@ class Server extends AppModel
                     'test' => 'testForEmpty',
                     'type' => 'string',
                 ],
+                'email_reply_to' => [
+                    'level' => 2,
+                    'description' => __('Reply to e-mail address for e-mails send from MISP instance.'),
+                    'value' => '',
+                    'test' => 'testForEmpty',
+                    'type' => 'string',
+                ],
                 'taxii_sync' => array(
                     'level' => 3,
                     'description' => __('This setting is deprecated and can be safely removed.'),
@@ -5689,6 +5739,13 @@ class Server extends AppModel
                     'type' => 'boolean',
                     'test' => 'testBool'
                 ),
+                'enableSightingBlocklisting' => array(
+                    'level' => 1,
+                    'description' => __('Blocklisting organisation UUIDs to prevent the creation of any sightings created by the blocklisted organisation.'),
+                    'value' => true,
+                    'type' => 'boolean',
+                    'test' => 'testBool'
+                ),
                 'log_client_ip' => array(
                     'level' => 1,
                     'description' => __('If enabled, all log entries will include the IP address of the user.'),
@@ -6430,6 +6487,22 @@ class Server extends AppModel
                     'editable' => false,
                     'redacted' => true
                 ),
+                'api_key_quick_lookup' => [
+                    'level' => self::SETTING_CRITICAL,
+                    'description' => __('Allow for the temporary storing of hashed API keys in redis for a short period of time, to allow for faster authentication of consecutive API requests. This is a massive speed-up for tools unable to continue the session, querying fast endpoints of MISP in rapid succession, at the cost of storing HMAC hashed api keys in redis.'),
+                    'value' => false,
+                    'null' => true,
+                    'test' => 'testBool',
+                    'type' => 'boolean',
+                ],
+                'api_key_quick_lookup_expiration' => [
+                    'level' => self::SETTING_CRITICAL,
+                    'description' => __('If the api key quick lookup is enabled, this setting will allow you to tune the expiration of the keys in redis. A longer expiration means fewer costly lookups during high frequency queries, at the cost of longer persistence. When an API key is revoked, they stay valid until this expiration kicks in. The value is specified in seconds.'),
+                    'value' => 180,
+                    'null' => true,
+                    'test' => 'testForNumeric',
+                    'type' => 'numeric',
+                ],
                 'alert_on_suspicious_logins' => [
                     'level' => 1,
                     'description' => __('When enabled, MISP will alert users of logins from new devices / suspicious logins. Please make sure that your logs table has additional indexes (on the user_id and action fields) for this not to be a performance bottleneck for now (expected to be resolved soon).'),
@@ -6563,6 +6636,14 @@ class Server extends AppModel
                     'type' => 'boolean',
                     'null' => true,
                 ],
+                'otp_disabled' => [
+                    'level' => self::SETTING_OPTIONAL,
+                    'description' => __('Disable TOTP on this instance.'),
+                    'value' => false,
+                    'test' => 'testBool',
+                    'type' => 'boolean',
+                    'null' => true,
+                ],
                 'otp_required' => array(
                     'level' => 2,
                     'description' => __('Require authentication with OTP. Users that do not have (T/H)OTP configured will be forced to create a token at first login. You cannot use it in combination with external authentication plugins.'),
@@ -7496,6 +7577,13 @@ class Server extends AppModel
                     'test' => 'testBool',
                     'type' => 'boolean'
                 ),
+                'Benchmarking_enable' => [
+                    'level' => 2,
+                    'description' => __('Enable the benchmarking functionalities to capture information about execution times, SQL query loads and more per user and per endpoint.'),
+                    'value' => false,
+                    'test' => 'testBool',
+                    'type' => 'boolean'
+                ],
                 'Enrichment_services_enable' => array(
                     'level' => 0,
                     'description' => __('Enable/disable the enrichment services'),

app/Model/ShadowAttribute.php

@@ -196,6 +196,24 @@ class ShadowAttribute extends AppModel
 
         // convert into utc and micro sec
         $this->data = $this->Attribute->ISODatetimeToUTC($this->data, $this->alias);
+
+        $trigger_id = 'shadow-attribute-before-save';
+        $isTriggerCallable = $this->isTriggerCallable($trigger_id);
+        if ($isTriggerCallable) {
+            $triggerData = $this->data;
+            $shadowAttribute_id = $triggerData['ShadowAttribute']['id'] ?? 0;
+            $workflowErrors = [];
+            $logging = [
+                'model' => 'ShadowAttribute',
+                'action' => 'add',
+                'id' => $shadowAttribute_id,
+                'message' => __('The workflow `%s` prevented the saving of this proposal.', $trigger_id)
+            ];
+            $workflowSuccess = $this->executeTrigger($trigger_id, $triggerData, $workflowErrors, $logging);
+            if (!$workflowSuccess) {
+                return false;
+            }
+        }
         return true;
     }
 
@@ -688,6 +706,8 @@ class ShadowAttribute extends AppModel
             return 0;
         }
 
+        $serverSync->debug("Pulling proposals");
+
         $i = 1;
         $fetchedCount = 0;
         $chunkSize = 1000;

app/Model/Sighting.php

@@ -25,6 +25,8 @@ class Sighting extends AppModel
 
     public $recursive = -1;
 
+    private $__blockedOrgs = null;
+
     public $actsAs = array(
             'Containable',
     );
@@ -72,6 +74,16 @@ class Sighting extends AppModel
         } else {
             $this->data['Sighting']['uuid'] = strtolower($this->data['Sighting']['uuid']);
         }
+        if ($this->__blockedOrgs === null) {
+            $SightingBlocklist = ClassRegistry::init('SightingBlocklist');
+            $this->__blockedOrgs = $SightingBlocklist->find('column', [
+                'recursive' => -1,
+                'fields' => ['org_uuid']
+            ]);
+        }
+        if (!empty($this->data['Sighting']['org_uuid']) && in_array($this->data['Sighting']['org_uuid'], $this->__blockedOrgs)) {
+            return false;
+        }
         return true;
     }
 
@@ -428,18 +440,33 @@ class Sighting extends AppModel
             return ['Sighting.attribute_id' => array_column(array_column($attributes, 'Attribute'), 'id')];
         }
 
-        $hostOrgId = Configure::read('MISP.host_org_id');
+        // Merge attributes by Event ID
         $userOrgId = $user['org_id'];
-        $conditions = [];
+        $attributesByEventId = [];
         foreach ($attributes as $attribute) {
-            $attributeConditions = ['Sighting.attribute_id' => $attribute['Attribute']['id']];
-            $ownEvent = $attribute['Event']['org_id'] == $userOrgId;
-            if (!$ownEvent) {
+            $eventId = $attribute['Event']['id'];
+            if (isset($attributesByEventId[$eventId])) {
+                $attributesByEventId[$eventId]['ids'][] = $attribute['Attribute']['id'];
+            } else {
+                $ownEvent = $attribute['Event']['org_id'] == $userOrgId;
+                $attributesByEventId[$eventId] = [
+                    'ids' => [$attribute['Attribute']['id']],
+                    'ownEvent' => $ownEvent,
+                ];
+            }
+        }
+
+        // Create conditions for merged attributes
+        $hostOrgId = Configure::read('MISP.host_org_id');
+        $conditions = [];
+        foreach ($attributesByEventId as $eventId => $eventAttributes) {
+            $attributeConditions = ['Sighting.attribute_id' => $eventAttributes['ids']];
+            if (!$eventAttributes['ownEvent']) {
                 if ($sightingsPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
                     $attributeConditions['Sighting.org_id'] = $userOrgId;
                 } else if ($sightingsPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) {
-                    if (!$this->isReporter($attribute['Event']['id'], $userOrgId)) {
-                        continue; // skip attribute
+                    if (!$this->isReporter($eventId, $userOrgId)) {
+                        continue; // skip event
                     }
                 } else if ($sightingsPolicy === self::SIGHTING_POLICY_HOST_ORG) {
                     $attributeConditions['Sighting.org_id'] = [$userOrgId, $hostOrgId];
@@ -1010,6 +1037,33 @@ class Sighting extends AppModel
         return $sightings;
     }
 
+     /**
+     * @param int $id
+     * @return array
+     */
+    public function getLastSightingForAttribute(array $user, $id): array
+    {
+        $conditions = [
+            'Sighting.attribute_id' => $id,
+            'Sighting.type' => 0,
+        ];
+
+        $sightingsPolicy = $this->sightingsPolicy();
+        if ($sightingsPolicy === self::SIGHTING_POLICY_EVENT_OWNER || $sightingsPolicy === self::SIGHTING_POLICY_HOST_ORG) {
+            $conditions['Sighting.org_id'] = [$user['org_id'], Configure::read('MISP.host_org_id')];
+        } else if ($sightingsPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) {
+            $all_sightings = $this->listSightings($user, [$id], 'attribute', false, 0, true);
+            $sighting = $all_sightings[0]['Sighting']['date_sighting'];
+            return $sighting;
+        }
+        $sighting = $this->find('first', [
+            'conditions' => $conditions,
+            'recursive' => -1,
+            'order' => ['Sighting.date_sighting DESC']
+        ]);
+        return empty($sighting) ? [] : $sighting;
+    }
+
     /**
      * @param array $user
      * @param string $returnFormat
@@ -1059,20 +1113,30 @@ class Sighting extends AppModel
                 $filters['org_id'] = array($filters['org_id']);
             }
             foreach ($filters['org_id'] as $k => $org_id) {
+                $negation = false;
+                if (is_string($org_id) && $org_id[0] === '!') {
+                    $negation = true;
+                    $org_id = substr($org_id, 1);
+                }
                 if (Validation::uuid($org_id)) {
                     $org = $this->Organisation->find('first', array(
                         'conditions' => array('Organisation.uuid' => $org_id),
                         'recursive' => -1,
                         'fields' => array('Organisation.id'),
                     ));
-                    if (empty($org)) {
-                        $filters['org_id'][$k] = -1;
-                    } else {
-                        $filters['org_id'][$k] = $org['Organisation']['id'];
+                    if (!empty($org)) {
+                        $temp = $org['Organisation']['id'];
                     }
                 }
+                if ($negation) {
+                    $conditions['Sighting.org_id'][] = $temp;
+                } else {
+                    $conditions['Sighting.org_id NOT IN'][] = $temp;
+                }
+                if (empty($conditions['Sighting.org_id']) && empty($conditions['Sighting.org_id NOT IN'])) {
+                    $conditions['Sighting.org_id'] = -1;
+                }
             }
-            $conditions['Sighting.org_id'] = $filters['org_id'];
         }
 
         if (isset($filters['source'])) {
@@ -1136,34 +1200,37 @@ class Sighting extends AppModel
         $tmpfile->write($exportTool->header($exportToolParams));
         $separator = $exportTool->separator($exportToolParams);
         
-        // fetch sightings matching the query without ACL checks
-        if (!empty($conditions['Sighting.event_id']) && is_array($conditions['Sighting.event_id'])) {
-            $conditions_copy = $conditions;
-            $sightingIds = [];
-            foreach ($conditions['Sighting.event_id'] as $e_id) {
-                $conditions_copy['Sighting.event_id'] = $e_id;
-                $tempIds = $this->find('column', [
+
+        if (empty(Configure::read('MISP.disable_sighting_loading'))) {
+            // fetch sightings matching the query without ACL checks
+            if (!empty($conditions['Sighting.event_id']) && is_array($conditions['Sighting.event_id'])) {
+                $conditions_copy = $conditions;
+                $sightingIds = [];
+                foreach ($conditions['Sighting.event_id'] as $e_id) {
+                    $conditions_copy['Sighting.event_id'] = $e_id;
+                    $tempIds = $this->find('column', [
+                        'conditions' => $conditions,
+                        'fields' => ['Sighting.id'],
+                        'contain' => $contain
+                    ]);
+                    if (!empty($tempIds)) {
+                        $sightingIds = array_merge($sightingIds, $tempIds);
+                    }
+                }
+            } else {
+                $sightingIds = $this->find('column', [
                     'conditions' => $conditions,
                     'fields' => ['Sighting.id'],
                     'contain' => $contain
                 ]);
-                if (!empty($tempIds)) {
-                    $sightingIds = array_merge($sightingIds, $tempIds);
-                }
             }
-        } else {
-            $sightingIds = $this->find('column', [
-                'conditions' => $conditions,
-                'fields' => ['Sighting.id'],
-                'contain' => $contain
-            ]);
-        }
-        
-        foreach (array_chunk($sightingIds, 500) as $chunk) {
-            // fetch sightings with ACL checks and sighting policies
-            $sightings = $this->getSightings($user, $chunk, $includeEvent, $includeAttribute, $includeUuid);
-            foreach ($sightings as $sighting) {
-                $tmpfile->writeWithSeparator($exportTool->handler($sighting, $exportToolParams), $separator);
+            
+            foreach (array_chunk($sightingIds, 500) as $chunk) {
+                // fetch sightings with ACL checks and sighting policies
+                $sightings = $this->getSightings($user, $chunk, $includeEvent, $includeAttribute, $includeUuid);
+                foreach ($sightings as $sighting) {
+                    $tmpfile->writeWithSeparator($exportTool->handler($sighting, $exportToolParams), $separator);
+                }
             }
         }
 
@@ -1211,6 +1278,8 @@ class Sighting extends AppModel
             if (!isset($attributes[$s['attribute_uuid']])) {
                 continue; // attribute doesn't exists or user don't have permission to access it
             }
+            $existingSighting[$s['uuid']] = true; // just to be sure that there are no sigthings with duplicated UUID
+
             list($attributeId, $eventId) = $attributes[$s['attribute_uuid']];
 
             if ($s['type'] === '2') {
@@ -1226,11 +1295,8 @@ class Sighting extends AppModel
             if ($user['Role']['perm_sync']) {
                 if (isset($s['org_id'])) {
                     if ($s['org_id'] != 0 && !empty($s['Organisation'])) {
-                        if (isset($existingOrganisations[$s['Organisation']['uuid']])) {
-                            $saveOnBehalfOf = $existingOrganisations[$s['Organisation']['uuid']];
-                        } else {
-                            $saveOnBehalfOf = $this->Organisation->captureOrg($s['Organisation'], $user);
-                        }
+                        $saveOnBehalfOf = $existingOrganisations[$s['Organisation']['uuid']] ??
+                            $this->Organisation->captureOrg($s['Organisation'], $user);
                     } else {
                         $saveOnBehalfOf = 0;
                     }
@@ -1252,8 +1318,8 @@ class Sighting extends AppModel
         }
 
         if ($this->saveMany($toSave)) {
-            $existingUuids = array_column($toSave, 'uuid');
-            $this->Event->publishSightingsRouter($event['Event']['id'], $user, $passAlong, $existingUuids);
+            $sightingsUuidsToPush = array_column($toSave, 'uuid');
+            $this->Event->publishSightingsRouter($event['Event']['id'], $user, $passAlong, $sightingsUuidsToPush);
             return count($toSave);
         }
 
@@ -1352,27 +1418,26 @@ class Sighting extends AppModel
      */
     public function pullSightings(array $user, ServerSyncTool $serverSync)
     {
+        $serverSync->debug("Fetching event index for pulling sightings");
+
         $this->Server = ClassRegistry::init('Server');
         try {
             $remoteEvents = $this->Server->getEventIndexFromServer($serverSync);
         } catch (Exception $e) {
-            $this->logException("Could not fetch event IDs from server {$serverSync->server()['Server']['name']}", $e);
+            $this->logException("Could not fetch event IDs from server {$serverSync->serverName()}", $e);
             return 0;
         }
-
         // Remove events from list that do not have published sightings.
         foreach ($remoteEvents as $k => $remoteEvent) {
             if ($remoteEvent['sighting_timestamp'] == 0) {
                 unset($remoteEvents[$k]);
             }
         }
-
         // Downloads sightings just from events that exists locally and remote sighting_timestamp is newer than local.
         $localEvents = $this->Event->find('list', [
             'fields' => ['Event.uuid', 'Event.sighting_timestamp'],
             'conditions' => (count($remoteEvents) > 10000) ? [] : ['Event.uuid' => array_column($remoteEvents, 'uuid')],
         ]);
-
         $eventUuids = [];
         foreach ($remoteEvents as $remoteEvent) {
             if (isset($localEvents[$remoteEvent['uuid']]) && $localEvents[$remoteEvent['uuid']] < $remoteEvent['sighting_timestamp']) {
@@ -1380,7 +1445,6 @@ class Sighting extends AppModel
             }
         }
         unset($remoteEvents, $localEvents);
-
         if (empty($eventUuids)) {
             return 0;
         }
@@ -1390,6 +1454,8 @@ class Sighting extends AppModel
             return 0;
         }
 
+        $serverSync->debug("Pulling sightings for " . count($eventUuids) . " events");
+
         if ($serverSync->isSupported(ServerSyncTool::FEATURE_SIGHTING_REST_SEARCH)) {
             return $this->pullSightingNewWay($user, $eventUuids, $serverSync);
         } else {
@@ -1408,17 +1474,23 @@ class Sighting extends AppModel
      */
     private function pullSightingNewWay(array $user, array $eventUuids, ServerSyncTool $serverSync)
     {
+        $SightingBlocklist = ClassRegistry::init('SightingBlocklist');
+        $blockedSightingsOrgs = $SightingBlocklist->find('column', [
+            'recursive' => -1,
+            'fields' => ['org_uuid']
+        ]);
+
         $uuids = array_keys($eventUuids);
+        shuffle($uuids); // shuffle array to avoid keeping events with a lof ot sightings in same batch all the time
         $saved = 0;
         $savedEventUuids = [];
-        foreach (array_chunk($uuids, 100) as $chunk) {
+        foreach (array_chunk($uuids, 20) as $chunk) {
             try {
-                $sightings = $serverSync->fetchSightingsForEvents($chunk);
+                $sightings = $serverSync->fetchSightingsForEvents($chunk, $blockedSightingsOrgs);
             } catch (Exception $e) {
-                $this->logException("Failed to download sightings from {$serverSync->server()['Server']['name']}.", $e);
+                $this->logException("Failed to download sightings from remote server {$serverSync->server()['Server']['name']}.", $e);
                 continue;
             }
-
             $sightingsToSave = [];
             foreach ($sightings as $sighting) {
                 $sighting = $sighting['Sighting'];

app/Model/SightingBlocklist.php

@@ -0,0 +1,167 @@
+<?php
+App::uses('AppModel', 'Model');
+
+class SightingBlocklist extends AppModel
+{
+    public $useTable = 'sighting_blocklists';
+
+    public $recursive = -1;
+
+    public $actsAs = [
+        'AuditLog',
+            'SysLogLogable.SysLogLogable' => array( // TODO Audit, logable
+                    'userModel' => 'User',
+                    'userKey' => 'user_id',
+                    'change' => 'full'),
+            'Containable',
+    ];
+    public $blocklistFields = ['org_uuid', 'comment', 'org_name'];
+
+    public $blocklistTarget = 'org';
+
+    private $blockedCache = [];
+
+    public $validate = array(
+        'org_uuid' => array(
+            'unique' => array(
+                'rule' => 'isUnique',
+                'message' => 'Organisation already blocklisted.'
+            ),
+            'uuid' => array(
+                'rule' => 'uuid',
+                'message' => 'Please provide a valid RFC 4122 UUID'
+            ),
+        )
+    );
+
+    public function beforeValidate($options = array())
+    {
+        parent::beforeValidate();
+        if (empty($this->data['OrgBlocklist']['id'])) {
+            $this->data['OrgBlocklist']['date_created'] = date('Y-m-d H:i:s');
+        }
+        return true;
+    }
+
+    public function afterDelete()
+    {
+        parent::afterDelete();
+        if (!empty($this->data['OrgBlocklist']['org_uuid'])) {
+            $this->cleanupBlockedCount($this->data['OrgBlocklist']['org_uuid']);
+        }
+    }
+
+    public function afterFind($results, $primary = false)
+    {
+        foreach ($results as $k => $result) {
+            if (isset($result['OrgBlocklist']['org_uuid'])) {
+                $results[$k]['OrgBlocklist']['blocked_data'] = $this->getBlockedData($result['OrgBlocklist']['org_uuid']);
+            }
+        }
+        return $results;
+    }
+
+    /**
+     * @param array $eventArray
+     */
+    public function removeBlockedEvents(array &$eventArray)
+    {
+        $blocklistHits = $this->find('column', array(
+            'conditions' => array('OrgBlocklist.org_uuid' => array_unique(array_column($eventArray, 'orgc_uuid'))),
+            'fields' => array('OrgBlocklist.org_uuid'),
+        ));
+        if (empty($blocklistHits)) {
+            return;
+        }
+        $blocklistHits = array_flip($blocklistHits);
+        foreach ($eventArray as $k => $event) {
+            if (isset($blocklistHits[$event['orgc_uuid']])) {
+                unset($eventArray[$k]);
+            }
+        }
+    }
+
+    /**
+     * @param int|string $orgIdOrUuid Organisation ID or UUID
+     * @return bool
+     */
+    public function isBlocked($orgIdOrUuid)
+    {
+        if (isset($this->blockedCache[$orgIdOrUuid])) {
+            return $this->blockedCache[$orgIdOrUuid];
+        }
+
+        if (is_numeric($orgIdOrUuid)) {
+            $orgUuid = $this->getUUIDFromID($orgIdOrUuid);
+        } else {
+            $orgUuid = $orgIdOrUuid;
+        }
+
+        $isBlocked = $this->hasAny(['OrgBlocklist.org_uuid' => $orgUuid]);
+        $this->blockedCache[$orgIdOrUuid] = $isBlocked;
+        return $isBlocked;
+    }
+
+    private function getUUIDFromID($orgID)
+    {
+        $this->Organisation = ClassRegistry::init('Organisation');
+        $orgUuid = $this->Organisation->find('first', [
+            'conditions' => ['Organisation.id' => $orgID],
+            'fields' => ['Organisation.uuid'],
+            'recursive' => -1,
+        ]);
+        if (empty($orgUuid)) {
+            return false; // org not found by ID, so it is not blocked
+        }
+        $orgUuid = $orgUuid['Organisation']['uuid'];
+        return $orgUuid;
+    }
+
+    public function saveEventBlocked($orgIdOrUUID)
+    {
+        if (is_numeric($orgIdOrUUID)) {
+            $orgcUUID = $this->getUUIDFromID($orgIdOrUUID);
+        } else {
+            $orgcUUID = $orgIdOrUUID;
+        }
+        $lastBlockTime = time();
+        $redisKeyBlockAmount = "misp:blocklist_blocked_amount:{$orgcUUID}";
+        $redisKeyBlockLastTime = "misp:blocklist_blocked_last_time:{$orgcUUID}";
+        $redis = RedisTool::init();
+        if ($redis !== false) {
+            $pipe = $redis->multi(Redis::PIPELINE)
+                ->incr($redisKeyBlockAmount)
+                ->set($redisKeyBlockLastTime, $lastBlockTime);
+            $pipe->exec();
+        }
+    }
+
+    private function cleanupBlockedCount($orgcUUID)
+    {
+        $redisKeyBlockAmount = "misp:blocklist_blocked_amount:{$orgcUUID}";
+        $redisKeyBlockLastTime = "misp:blocklist_blocked_last_time:{$orgcUUID}";
+        $redis = RedisTool::init();
+        if ($redis !== false) {
+            $pipe = $redis->multi(Redis::PIPELINE)
+                ->del($redisKeyBlockAmount)
+                ->del($redisKeyBlockLastTime);
+            $pipe->exec();
+        }
+    }
+
+    public function getBlockedData($orgcUUID)
+    {
+        $redisKeyBlockAmount = "misp:blocklist_blocked_amount:{$orgcUUID}";
+        $redisKeyBlockLastTime = "misp:blocklist_blocked_last_time:{$orgcUUID}";
+        $blockData = [
+            'blocked_amount' => false,
+            'blocked_last_time' => false,
+        ];
+        $redis = RedisTool::init();
+        if ($redis !== false) {
+            $blockData['blocked_amount'] = $redis->get($redisKeyBlockAmount);
+            $blockData['blocked_last_time'] = $redis->get($redisKeyBlockLastTime);
+        }
+        return $blockData;
+    }
+}

app/Model/User.php

@@ -2124,11 +2124,11 @@ class User extends AppModel
 
             return true;
         } else {
-            return $this->forgot($email);
+            return $this->forgot($email, $ip);
         }
     }
 
-    public function forgot($email, $ip, $jobId = null)
+    public function forgot($email, $ip)
     {
         $user = $this->find('first', [
             'recursive' => -1,
@@ -2140,9 +2140,8 @@ class User extends AppModel
         if (empty($user)) {
             return false;
         }
-        $redis = $this->setupRedis();
         $token = RandomTool::random_str(true, 40, '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ');
-        $redis->set('misp:forgot:' . $token, $user['User']['id'], ['nx', 'ex' => 600]);
+        RedisTool::init()->set('misp:forgot:' . $token, $user['User']['id'], ['nx', 'ex' => 600]);
         $baseurl = Configure::check('MISP.external_baseurl') ? Configure::read('MISP.external_baseurl') : Configure::read('MISP.baseurl');
         $body = __(
             "Dear MISP user,\n\nyou have requested a password reset on the MISP instance at %s. Click the link below to change your password.\n\n%s\n\nThe link above is only valid for 10 minutes, feel free to request a new one if it has expired.\n\nIf you haven't requested a password reset, reach out to your admin team and let them know that someone has attempted it in your stead.\n\nMake sure you keep the contents of this e-mail confidential, do NOT ever forward it as it contains a reset token that is equivalent of a password if acted upon. The IP used to trigger the request was: %s\n\nBest regards,\nYour MISP admin team",

app/Model/UserLoginProfile.php

@@ -46,6 +46,8 @@ class UserLoginProfile extends AppModel
     private function browscapGetBrowser()
     {
         $logger = new \Monolog\Logger('name');
+        $streamHandler = new \Monolog\Handler\StreamHandler('php://stderr', \Monolog\Logger::INFO);
+        $logger->pushHandler($streamHandler);
 
         if (function_exists('apcu_fetch')) {
             App::uses('ApcuCacheTool', 'Tools');

app/Model/Workflow.php

@@ -461,7 +461,8 @@ class Workflow extends AppModel
                     $trigger_id,
                     JsonTool::encode($data),
                     JsonTool::encode($logging),
-                    $jobId
+                    $jobId,
+                    Configure::check('CurrentUserId') ? JsonTool::encode(Configure::read('CurrentUserId')) : null,
                 ],
                 true,
                 $jobId

app/Model/WorkflowModules/WorkflowBaseModule.php

@@ -213,6 +213,8 @@ class WorkflowBaseModule
             if ($operator == 'in_or') {
                 return !empty($matching);
             } elseif ($operator == 'in_and') {
+                sort($matching);
+                sort($value);
                 return array_values($matching) == array_values($value);
             } elseif ($operator == 'not_in_or') {
                 return empty($matching);

app/Model/WorkflowModules/action/Module_stop_execution.php

@@ -6,6 +6,7 @@ class Module_stop_execution extends WorkflowBaseActionModule
     public $blocking = true;
     public $id = 'stop-execution';
     public $name = 'Stop execution';
+    public $version = '0.2';
     public $description = 'Essentially stops the execution for blocking workflows. Do nothing for non-blocking ones';
     public $icon = 'ban';
     public $inputs = 1;
@@ -15,12 +16,25 @@ class Module_stop_execution extends WorkflowBaseActionModule
     public function __construct()
     {
         parent::__construct();
+
+        $this->params = [
+            [
+                'id' => 'message',
+                'label' => 'Stop message',
+                'type' => 'input',
+                'default' => __('Execution stopped'),
+                'placeholder' => __('Execution stopped'),
+                'jinja_supported' => true,
+            ],
+        ];
     }
 
     public function exec(array $node, WorkflowRoamingData $roamingData, array &$errors = []): bool
     {
         parent::exec($node, $roamingData, $errors);
-        $errors[] = __('Execution stopped');
+        $rData = $roamingData->getData();
+        $params = $this->getParamsWithValues($node, $rData);
+        $errors[] = empty($params['message']['value']) ? $params['message']['default'] : $params['message']['value'];
         return false;
     }
 }

app/Model/WorkflowModules/logic/Module_distribution_if.php

@@ -5,7 +5,7 @@ class Module_distribution_if extends WorkflowBaseLogicModule
 {
     public $id = 'distribution-if';
     public $name = 'IF :: Distribution';
-    public $version = '0.2';
+    public $version = '0.3';
     public $description = 'Distribution IF / ELSE condition block. The `then` output will be used if the encoded conditions is satisfied, otherwise the `else` output will be used.';
     public $icon = 'code-branch';
     public $inputs = 1;
@@ -103,12 +103,15 @@ class Module_distribution_if extends WorkflowBaseLogicModule
             $final_sharing_group = $this->__extractSharingGroupIDs(
                 $data['Event'],
                 $data['Event']['Attribute'][0]['Object'] ?? [],
-                $data['Event']['Attribute'][0]
+                $data['Event']['Attribute'][0],
+                $scope
             );
             if ($operator == 'equals') {
-                return !array_diff($final_sharing_group, $selected_sharing_groups); // All sharing groups are in the selection
+                return empty($selected_sharing_groups) ? !empty($final_sharing_group) :
+                    !array_diff($final_sharing_group, $selected_sharing_groups); // All sharing groups are in the selection
             } else if ($operator == 'not_equals') {
-                return count(array_diff($final_sharing_group, $selected_sharing_groups)) == count($final_sharing_group); // All sharing groups are in the selection
+                return empty($selected_sharing_groups) ? empty($final_sharing_group) :
+                    count(array_diff($final_sharing_group, $selected_sharing_groups)) == count($final_sharing_group); // All sharing groups are in the selection
             }
             $errors[] = __('Condition operator not supported for that distribution level');
             return false;
@@ -159,9 +162,15 @@ class Module_distribution_if extends WorkflowBaseLogicModule
         return min($distri1, $distri2);
     }
 
-    private function __extractSharingGroupIDs(array $event, array $object=[], array $attribute=[]): array
+    private function __extractSharingGroupIDs(array $event, array $object=[], array $attribute=[], $scope='event'): array
     {
         $sgIDs = [];
+        if ($scope == 'event') {
+            if (!empty($event) && $event['distribution'] == 4) {
+                $sgIDs[] = $event['sharing_group_id'];
+            }
+            return $sgIDs;
+        }
         if (!empty($event) && $event['distribution'] == 4) {
             $sgIDs[] = $event['sharing_group_id'];
         }

app/Model/WorkflowModules/trigger/Module_shadow_attribute_before_save.php

@@ -0,0 +1,45 @@
+<?php
+include_once APP . 'Model/WorkflowModules/WorkflowBaseModule.php';
+
+class Module_shadow_attribute_before_save extends WorkflowBaseTriggerModule
+{
+    public $id = 'shadow-attribute-before-save';
+    public $scope = 'shadow-attribute';
+    public $name = 'Shadow Attribute Before Save';
+    public $description = 'This trigger is called just before a Shadow Attribute is saved in the database';
+    public $icon = 'comment';
+    public $inputs = 0;
+    public $outputs = 1;
+    public $blocking = true;
+    public $misp_core_format = true;
+    public $trigger_overhead = self::OVERHEAD_MEDIUM;
+
+    public function __construct()
+    {
+        parent::__construct();
+        $this->trigger_overhead_message = __('This trigger is called each time a Shadow Attribute is about to be saved. This means that when a large quantity of Shadow Attributes are being saved (e.g. Feed pulling or synchronisation), the workflow will be run for as many time as there are Shadow Attributes.');  
+    }
+
+    public function normalizeData(array $data)
+    {
+        $this->Event = ClassRegistry::init('Event');
+        $this->Attribute = ClassRegistry::init('Attribute');
+
+        if (empty($data['ShadowAttribute'])) {
+            return false;
+        }
+        
+        // If we're dealing with a proposed edit, we retrieve the data about the attribute 
+        if ($data['ShadowAttribute']['old_id']) {
+            $event = $this->Attribute->fetchAttribute($data['ShadowAttribute']['old_id']);
+            $event['Attribute']['ShadowAttribute'] = array($data['ShadowAttribute']);
+        } else {
+            // If it is a proposal to add a new attribute, we retrieve only the data about the event
+            $event = $this->Event->quickFetchEvent($data['ShadowAttribute']['event_id']);
+            $event['Event']['ShadowAttribute'] = [$data['ShadowAttribute']];
+        }
+
+        $event = parent::normalizeData($event);
+        return $event;
+    }
+}

app/Plugin/EcsLog/Lib/Log/Engine/EcsLog.php

@@ -182,10 +182,10 @@ class EcsLog implements CakeLogInterface
     }
 
     /**
-     * @param Exception $exception
+     * @param Throwable $exception
      * @return void
      */
-    public static function handleException(Exception $exception)
+    public static function handleException(Throwable $exception)
     {
         $code = $exception->getCode();
         $code = ($code && is_int($code)) ? $code : 1;

app/Plugin/OidcAuth/Controller/Component/Auth/OidcAuthenticate.php

@@ -13,10 +13,11 @@ App::uses('Oidc', 'OidcAuth.Lib');
  *  - OidcAuth.organisation_property (default: `organization`)
  *  - OidcAuth.organisation_uuid_property (default: `organization_uuid`)
  *  - OidcAuth.roles_property (default: `roles`)
- *  - OidcAuth.default_org
+ *  - OidcAuth.default_org - organisation ID, UUID or name if organisation is not provided by OIDC
  *  - OidcAuth.unblock (boolean, default: false)
  *  - OidcAuth.offline_access (boolean, default: false)
  *  - OidcAuth.check_user_validity (integer, default `0`)
+ *  - OidcAuth.update_user_role (boolean, default: true) - if disabled, manually modified role in MISP admin interface will be not changed from OIDC
  */
 class OidcAuthenticate extends BaseAuthenticate
 {

app/Plugin/OidcAuth/Lib/Oidc.php

@@ -49,17 +49,22 @@ class Oidc
         }
 
         $organisationProperty = $this->getConfig('organisation_property', 'organization');
-        $organisationName = $claims->{$organisationProperty} ?? $this->getConfig('default_org');
+        $organisationName = $claims->{$organisationProperty} ?? null;
 
         $organisationUuidProperty = $this->getConfig('organisation_uuid_property', 'organization_uuid');
         $organisationUuid = $claims->{$organisationUuidProperty} ?? null;
 
         $organisationId = $this->checkOrganization($organisationName, $organisationUuid, $mispUsername);
         if (!$organisationId) {
-            if ($user) {
-                $this->block($user);
+            $defaultOrganisationId = $this->defaultOrganisationId();
+            if ($defaultOrganisationId) {
+                $organisationId = $defaultOrganisationId;
+            } else {
+                if ($user) {
+                    $this->block($user);
+                }
+                return false;
             }
-            return false;
         }
 
         $roleProperty = $this->getConfig('roles_property', 'roles');
@@ -101,7 +106,7 @@ class Oidc
                 $user['org_id'] = $organisationId;
             }
 
-            if ($user['role_id'] != $roleId) {
+            if ($user['role_id'] != $roleId && $this->getConfig('update_user_role', true)) {
                 $this->User->updateField($user, 'role_id', $roleId);
                 $this->log($mispUsername, "User role changed from {$user['role_id']} to $roleId.");
                 $user['role_id'] = $roleId;
@@ -123,7 +128,7 @@ class Oidc
             return $user;
         }
 
-        $this->log($mispUsername, 'User not found in database.');
+        $this->log($mispUsername, 'User not found in database, creating new one.');
 
         $time = time();
         $userData = [
@@ -222,31 +227,37 @@ class Oidc
         $roleProperty = $this->getConfig('roles_property', 'roles');
         $roles = $claims->{$roleProperty} ?? $oidc->requestUserInfo($roleProperty);
         if ($roles === null) {
-            $this->log($user['email'], "Role property `$roleProperty` is missing in claims.");
+            $this->log($user['email'], "Role property `$roleProperty` is missing in claims.", LOG_ERR);
             return false;
         }
 
         $roleId = $this->getUserRole($roles, $user['email']);
         if ($roleId === null) {
-            $this->log($user['email'], 'No role was assigned.');
+            $this->log($user['email'], 'No role was assigned.', LOG_WARNING);
             return false;
         }
 
-        if ($update && $user['role_id'] != $roleId) {
+        if ($update && $user['role_id'] != $roleId && $this->getConfig('update_user_role', true)) {
             $this->User->updateField($user, 'role_id', $roleId);
             $this->log($user['email'], "User role changed from {$user['role_id']} to $roleId.");
         }
 
         // Check user org
         $organisationProperty = $this->getConfig('organisation_property', 'organization');
-        $organisationName = $claims->{$organisationProperty} ?? $this->getConfig('default_org');
+        $organisationName = $claims->{$organisationProperty} ?? null;
 
         $organisationUuidProperty = $this->getConfig('organisation_uuid_property', 'organization_uuid');
         $organisationUuid = $claims->{$organisationUuidProperty} ?? null;
 
         $organisationId = $this->checkOrganization($organisationName, $organisationUuid, $user['email']);
         if (!$organisationId) {
-            return false;
+            $defaultOrganisationId = $this->defaultOrganisationId();
+            if ($defaultOrganisationId) {
+                $organisationId = $defaultOrganisationId;
+            } else {
+                $this->log($user['email'], 'No organisation was assigned.', LOG_WARNING);
+                return false;
+            }
         }
 
         if ($update && $user['org_id'] != $organisationId) {
@@ -291,9 +302,10 @@ class Oidc
         $providerUrl = $this->getConfig('provider_url');
         $clientId = $this->getConfig('client_id');
         $clientSecret = $this->getConfig('client_secret');
+        $issuer = $this->getConfig('issuer', null, false);
 
         if (class_exists("\JakubOnderka\OpenIDConnectClient")) {
-            $oidc = new \JakubOnderka\OpenIDConnectClient($providerUrl, $clientId, $clientSecret);
+            $oidc = new \JakubOnderka\OpenIDConnectClient($providerUrl, $clientId, $clientSecret, $issuer);
         } else if (class_exists("\Jumbojett\OpenIDConnectClient")) {
             throw new Exception("Jumbojett OIDC implementation is not supported anymore, please use JakubOnderka's client");
         } else {
@@ -320,6 +332,8 @@ class Oidc
     }
 
     /**
+     * Fetch organisation ID from database by provided name and UUID. If organisation is not found, it is created. If
+     * organisation with given UUID has different name, then is renamed.
      * @param string $orgName Organisation name or UUID
      * @param string|null $orgUuid Organisation UUID
      * @param string $mispUsername
@@ -376,6 +390,41 @@ class Oidc
         return $orgId;
     }
 
+    /**
+     * @return false|int Organisation ID or false if org not found
+     */
+    private function defaultOrganisationId()
+    {
+        $defaultOrgName = $this->getConfig('default_org');
+        if (empty($defaultOrgName)) {
+            return false;
+        }
+
+        if (is_numeric($defaultOrgName)) {
+            $conditions = ['id' => $defaultOrgName];
+        } else if (Validation::uuid($defaultOrgName)) {
+            $conditions = ['uuid' => strtolower($defaultOrgName)];
+        } else {
+            $conditions = ['name' => $defaultOrgName];
+        }
+        $orgAux = $this->User->Organisation->find('first', [
+            'fields' => ['Organisation.id'],
+            'conditions' => $conditions,
+        ]);
+        if (empty($orgAux)) {
+            if (is_numeric($defaultOrgName)) {
+                $this->log(null, "Could not find default organisation with ID `$defaultOrgName`.", LOG_ERR);
+            } else if (Validation::uuid($defaultOrgName)) {
+                $this->log(null, "Could not find default organisation with UUID `$defaultOrgName`.", LOG_ERR);
+            } else {
+                $this->log(null, "Could not find default organisation with name `$defaultOrgName`.", LOG_ERR);
+            }
+            return false;
+        }
+
+        return $orgAux['Organisation']['id'];
+    }
+
     /**
      * @param int $orgId
      * @param string $newName
@@ -394,12 +443,13 @@ class Oidc
     }
 
     /**
-     * @param array $roles Role list provided by OIDC
+     * @param array|string $roles Role list provided by OIDC
      * @param string $mispUsername
      * @return int|null Role ID or null if no role matches
      */
-    private function getUserRole(array $roles, $mispUsername)
+    private function getUserRole($roles, $mispUsername)
     {
+        $roles = is_string($roles) ? explode($this->getConfig('roles_delimiter', ','), $roles) : $roles;
         $this->log($mispUsername, 'Provided roles: ' . implode(', ', $roles));
         $roleMapper = $this->getConfig('role_mapper');
         if (!is_array($roleMapper)) {
@@ -453,13 +503,15 @@ class Oidc
     /**
      * @param string $config
      * @param mixed|null $default
+     * @param bool $required When true and variable is not set, RuntimeException will be thrown
      * @return mixed
+     * @throws RuntimeException when config option is not set
      */
-    private function getConfig($config, $default = null)
+    private function getConfig($config, $default = null, $required = true)
     {
         $value = Configure::read("OidcAuth.$config");
-        if (empty($value)) {
-            if ($default === null) {
+        if ($value === null) {
+            if ($default === null && $required) {
                 throw new RuntimeException("Config option `OidcAuth.$config` is not set.");
             }
             return $default;

app/Plugin/OidcAuth/README.md

@@ -32,6 +32,7 @@ $config = array(
     ...
     'OidcAuth' = [
         'provider_url' => '{{ OIDC_PROVIDER }}',
+        'issuer' => '{{ OIDC_ISSUER }}', // If omitted, it defaults to provider_url
         'client_id' => '{{ OIDC_CLIENT_ID }}',
         'client_secret' => '{{ OIDC_CLIENT_SECRET }}',
         'role_mapper' => [ // if user has multiple roles, first role that match will be assigned to user

app/View/AnalystData/add.ctp

@@ -214,6 +214,7 @@ if (!$ajax) {
                 $('#RelationshipRelationshipType').val($('#pickerRelationshipTypeSelect').val());
                 $(that).popover('hide')
             });
+            $('#genericModal').attr('tabindex', '')
         });
     }
 <?php endif; ?>

app/View/AnalystData/view.ctp

@@ -9,10 +9,6 @@ $fields = [
         'path' => $modelSelection . '.uuid',
         'class' => '',
         'type' => 'uuid',
-        'object_type' => $modelSelection,
-        'notes_path' => $modelSelection . '.Note',
-        'opinions_path' => $modelSelection . '.Opinion',
-        'relationships_path' => $modelSelection . '.Relationship',
     ],
     [
         'key' => __('Note Type'),
@@ -119,14 +115,57 @@ echo $this->element(
 );
 
 $object_uuid = Hash::get($data, $modelSelection . '.uuid');
+
+$notes = $data[$modelSelection]['Note'] ?? [];
+$opinions = $data[$modelSelection]['Opinion'] ?? [];
+$relationships_outbound = $data[$modelSelection]['Relationship'] ?? [];
+$relationships_inbound = $data[$modelSelection]['RelationshipInbound'] ?? [];
+$notesOpinions = array_merge($notes, $opinions);
+if(!function_exists("countNotes")) {
+    function countNotes($notesOpinions) {
+        $notesTotalCount = count($notesOpinions);
+        $notesCount = 0;
+        $relationsCount = 0;
+        foreach ($notesOpinions as $notesOpinion) {
+            if ($notesOpinion['note_type'] == 2) { // relationship
+                $relationsCount += 1;
+            } else {
+                $notesCount += 1;
+            }
+            if (!empty($notesOpinion['Note'])) {
+                $nestedCounts = countNotes($notesOpinion['Note']);
+                $notesTotalCount += $nestedCounts['total'];
+                $notesCount += $nestedCounts['notesOpinions'];
+                $relationsCount += $nestedCounts['relations'];
+            }
+            if (!empty($notesOpinion['Opinion'])) {
+                $nestedCounts = countNotes($notesOpinion['Opinion']);
+                $notesTotalCount += $nestedCounts['total'];
+                $notesCount += $nestedCounts['notesOpinions'];
+                $relationsCount += $nestedCounts['relations'];
+            }
+        }
+        return ['total' => $notesTotalCount, 'notesOpinions' => $notesCount, 'relations' => $relationsCount];
+    }
+}
+$counts = countNotes($notesOpinions);
+$notesOpinionCount = $counts['notesOpinions'];
+$allCounts = [
+    'notesOpinions' => $counts['notesOpinions'],
+    'relationships_outbound' => count($relationships_outbound),
+    'relationships_inbound' => count($relationships_inbound),
+];
+
 $options = [
     'container_id' => 'analyst_data_thread',
     'object_type' => $modelSelection,
     'object_uuid' => $object_uuid,
     'shortDist' => $shortDist,
-    'notes' => $data[$modelSelection]['Note'] ?? [],
-    'opinions' => $data[$modelSelection]['Opinion'] ?? [],
-    'relationships' => $data[$modelSelection]['Relationship'] ?? [],
+    'notes' => $notes,
+    'opinions' => $opinions,
+    'relationships_outbound' => $relationships_outbound,
+    'relationships_inbound' => $relationships_inbound,
+    'allCounts' => $allCounts,
 ];
 
 echo $this->element('genericElements/assetLoader', [

app/View/Attributes/view_analyst_data.ctp

@@ -0,0 +1,11 @@
+<?php
+    $notes = !empty($object['Note']) ? $object['Note'] : [];
+    $opinions = !empty($object['Opinion']) ? $object['Opinion'] : [];
+    $relationships = !empty($object['Relationship']) ? $object['Relationship'] : [];
+    $relationshipsInbound = !empty($object['RelationshipInbound']) ? $object['RelationshipInbound'] : [];
+    echo $this->element('genericElements/Analyst_data/generic_simple', [
+        'analyst_data' => ['notes' => $notes, 'opinions' => $opinions, 'relationships_outbound' => $relationships, 'relationships_inbound' => $relationshipsInbound],
+        'object_uuid' => $object['uuid'],
+        'object_type' => 'Attribute'
+    ]);
+?>

app/View/Benchmarks/index.ctp

@@ -0,0 +1,110 @@
+<?php
+    $passedArgsArray = json_decode($passedArgs, true);
+    $fields = [
+        [
+            'name' => __('Date'),
+            'sort' => 'date',
+            'data_path' => 'date'
+        ],
+        [
+            'name' => __('scope'),
+            'sort' => 'scope',
+            'data_path' => 'scope'
+        ],
+        [
+            'name' => __('Key'),
+            'sort' => 'key',
+            'data_path' => 'text'
+        ],
+        [
+            'name' => __('field'),
+            'sort' => 'field',
+            'data_path' => 'field'
+        ],
+        [
+            'name' => __('Value'),
+            'element' => 'custom',
+            'function' => function($row) {
+                return empty($row['unit']) ? h($row['value']) : h($row['value'] . ' ' . $row['unit']);
+            },
+            'sort' => 'value'
+        ]
+    ];
+    $quick_filters = [];
+    foreach ($settings as $key => $setting_data) {
+        $temp = $filters;
+        $url = $baseurl . '/benchmarks/index';
+        foreach ($temp as $s => $v) {
+            if ($v && $s != $key) {
+                if (is_array($v)) {
+                    foreach ($v as $multi_v) {
+                        $url .= '/' . $s . '[]:' . $multi_v;
+                    }
+                } else {
+                    $url .= '/' . $s . ':' . $v;
+                }
+                
+            }
+        }
+        if ($key != 'average' && $key != 'aggregate') {
+            $quick_filters[$key]['all'] = [
+                'url' => h($url),
+                'text' => __('All'),
+                'active' => !$filters[$key],
+                'style' => 'display:inline;'
+            ];
+        }
+        foreach ($setting_data as $setting_element) {
+            $text = $setting_element;
+            if ($key == 'average') {
+                $text = $setting_element ? 'average / request' : 'total';
+            }
+            if ($key == 'aggregate') {
+                $text = $setting_element ? 'aggregate' : 'daily';
+            }
+            $quick_filters[$key][] = [
+                'url' => h($url . '/' . $key . ':' . $setting_element),
+                'text' => $text,
+                'active' => $filters[$key] == $setting_element,
+                'style' => 'display:inline;'
+            ];
+        }
+    }
+    echo $this->element('genericElements/IndexTable/scaffold', [
+        'scaffold_data' => [
+            'passedArgsArray' => $passedArgsArray,
+            'data' => [
+                'persistUrlParams' => array_keys($settings),
+                'data' => $data,
+                'top_bar' => [
+                    'pull' => 'right',
+                    'children' => [
+                        [
+                            'children' => $quick_filters['scope']
+                        ],
+                        [
+                            'children' => $quick_filters['field']
+                        ],
+                        [
+                            'children' => $quick_filters['average']
+                        ],
+                        [
+                            'children' => $quick_filters['aggregate']
+                        ],
+                        [
+                            'type' => 'search',
+                            'button' => __('Filter'),
+                            'placeholder' => __('Enter value to search'),
+                            'data' => '',
+                            'searchKey' => 'quickFilter'
+                        ]
+                    ]
+                ],
+                'fields' => $fields,
+                'title' => empty($ajax) ? __('Benchmark results') : false,
+                'description' => empty($ajax) ? __('Results of the collected benchmarks. You can filter it further by passing the limit, scope, field parameters.') : false,
+            ]
+        ]
+    ]);
+
+?>

app/View/Dashboards/index.ctp

@@ -23,7 +23,7 @@ $(function () {
         saveDashboardState();
     });
     grid.on('added', function(event, items) {
-        resetDashboardGrid(grid);
+        resetDashboardGrid(grid, false);
     });
     grid.on('gsresizestop', function(event, element) {
         $(element).find('.widgetContentInner').trigger('widget-resized')

app/View/Elements/Events/View/row_attribute.ctp

@@ -70,16 +70,6 @@
       <td class="short context hidden"><?= $objectId ?></td>
       <td class="short context hidden uuid">
         <span class="quickSelect"><?php echo h($object['uuid']); ?></span>
-        <?php
-          $notes = !empty($object['Note']) ? $object['Note'] : [];
-          $opinions = !empty($object['Opinion']) ? $object['Opinion'] : [];
-          $relationships = !empty($object['Relationship']) ? $object['Relationship'] : [];
-          echo $this->element('genericElements/Analyst_data/generic', [
-              'analyst_data' => ['notes' => $notes, 'opinions' => $opinions, 'relationships' => $relationships],
-              'object_uuid' => $object['uuid'],
-              'object_type' => 'Attribute'
-          ]);
-        ?>
       </td>
       <td class="short context hidden">
           <?php echo $this->element('/Events/View/seen_field', array('object' => $object)); ?>
@@ -90,12 +80,14 @@
           $notes = !empty($object['Note']) ? $object['Note'] : [];
           $opinions = !empty($object['Opinion']) ? $object['Opinion'] : [];
           $relationships = !empty($object['Relationship']) ? $object['Relationship'] : [];
-          echo $this->element('genericElements/shortUuidWithNotes', [
+          $relationshipsInbound = !empty($object['RelationshipInbound']) ? $object['RelationshipInbound'] : [];
+          echo $this->element('genericElements/shortUuidWithNotesAjax', [
               'uuid' => $object['uuid'],
               'object_type' => 'Attribute',
               'notes' => $notes,
               'opinions' => $opinions,
               'relationships' => $relationships,
+              'relationshipsInbound' => $relationshipsInbound,
           ]);
         ?>
       </td>
@@ -141,9 +133,9 @@
           <?php
               $value = $this->element('/Events/View/value_field', array('object' => $object));
               if (Configure::read('Plugin.Enrichment_hover_enable') && isset($modules) && isset($modules['hover_type'][$object['type']])) {
-                  $commonDataFields = sprintf('data-object-type="Attribute" data-object-id="%s"', $objectId);
+                  $commonDataFields = sprintf('data-object-type="attributes" data-object-id="%s"', $objectId);
                   $spanExtra = Configure::read('Plugin.Enrichment_hover_popover_only') ? '' : sprintf(' class="eventViewAttributeHover" %s', $commonDataFields);
-                  $popupButton = sprintf('<i class="fa fa-search-plus useCursorPointer eventViewAttributePopup noPrint" title="%s" %s></i>', __('Show hover enrichment'), $commonDataFields);
+                  $popupButton = sprintf('<i class="fa fa-search-plus useCursorPointer eventViewAttributePopup noPrint" role="button" tabindex="0" title="%s" %s></i>', __('Show hover enrichment'), $commonDataFields);
                   echo sprintf(
                       '<span%s>%s</span> %s',
                       $spanExtra,
@@ -239,8 +231,13 @@
                             $relatedData[__('Event UUIDs')] = implode('<br>', array_map('h', $feed['event_uuids']));
                         }
                         $popover = '';
-                        foreach ($relatedData as $k => $v) {
-                            $popover .= '<span class="bold black">' . $k . '</span>: <span class="blue">' . $v . '</span><br>';
+                        $event_count = count($relatedData);
+                        if ($event_count > 20) {
+                                $popover = '<span class="bold black">' . __('Events') . '</span>: <span class="blue">' . __('Zounds... of events (%d)', $event_count) . '</span><br>';
+                        } else {
+                            foreach ($relatedData as $k => $v) {
+                                $popover .= '<span class="bold black">' . h($k) . '</span>: <span class="blue">' . $v . '</span><br>';
+                            }
                         }
                         if ($isSiteAdmin || $hostOrgUser) {
                             if ($feed['source_format'] === 'misp') {
@@ -279,7 +276,6 @@
                 }
                 if (isset($object['Server'])) {
                     foreach ($object['Server'] as $server) {
-                        $popover = '';
                         foreach ($server as $k => $v) {
                             if ($k == 'id') continue;
                             if (is_array($v)) {
@@ -290,21 +286,37 @@
                             } else {
                                 $v = h($v);
                             }
-                            $popover .= '<span class=\'bold black\'>' . Inflector::humanize(h($k)) . '</span>: <span class="blue">' . $v . '</span><br />';
                         }
                         if (empty($server['event_uuids'])) {
                             $server['event_uuids'] = [0 => 1]; // Make sure to print the content once
                         }
-                        foreach ($server['event_uuids'] as $k => $event_uuid) {
+                        $event_count = count($server['event_uuids']);
+                        $popover = '';
+                        if ($event_count > 20) {
                             $liContents = '';
-                            $url = $isSiteAdmin ? sprintf('%s/servers/previewEvent/%s/%s', $baseurl, h($server['id']), h($event_uuid)) : '#';
+                            $message = __('Zounds... of events (%d)', $event_count);
+                            $url = $isSiteAdmin ? sprintf('%s/servers/previewIndex/%s', $baseurl, h($server['id'])) : '#';
+                            $popover = '<span class=\'bold black\'>' . __('Event uuid') . '</span>: <span class="blue">' . $message . '</span><br />';
                             $liContents .= sprintf(
                                 '<a href="%s" data-toggle="popover" data-content="%s" data-trigger="hover">%s</a>&nbsp;',
                                 $url,
                                 h($popover),
-                                'S' . h($server['id']) . ':' . ($k + 1)
+                                'S' . h($server['id']) . ':' . $message
                             );
                             echo "<li>$liContents</li>";
+                        } else {
+                            foreach ($server['event_uuids'] as $k => $event_uuid) {
+                                $popover = '<span class=\'bold black\'>' . __('Event uuid') . '</span>: <span class="blue">' . h($event_uuid) . '</span><br />';
+                                $liContents = '';
+                                $url = $isSiteAdmin ? sprintf('%s/servers/previewEvent/%s/%s', $baseurl, h($server['id']), h($event_uuid)) : '#';
+                                $liContents .= sprintf(
+                                    '<a href="%s" data-toggle="popover" data-content="%s" data-trigger="hover">%s</a>&nbsp;',
+                                    $url,
+                                    h($popover),
+                                    'S' . h($server['id']) . ':' . ($k + 1)
+                                );
+                                echo "<li>$liContents</li>";
+                            }
                         }
                     }
                 }