Merge branch '2.4' of github.com:MISP/MISP into 2.4

- helps administrators to easily see what went wrong in terms of timeouts / oom issues

.github/workflows/main.yml

@@ -184,20 +184,8 @@ jobs:
           app/Console/cake Admin setSetting "Plugin.ZeroMQ_enable" 1
           app/Console/cake Admin setSetting "Plugin.ZeroMQ_audit_notifications_enable" 1
 
-      - name: Update Galaxies
-        run: app/Console/cake Admin updateGalaxies
-
-      - name: Update Taxonomies
-        run: app/Console/cake Admin updateTaxonomies
-
-      - name: Update Warninglists
-        run: app/Console/cake Admin updateWarningLists --verbose
-
-      - name: Update Noticelists
-        run: app/Console/cake Admin updateNoticeLists
-
-      - name: Update Object Templates
-        run: app/Console/cake Admin updateObjectTemplates 1
+      - name: Update JSON
+        run: app/Console/cake Admin updateJSON
 
       - name: Turn MISP live
         run: app/Console/cake Admin live 1
@@ -269,13 +257,16 @@ jobs:
       - name: Check requirements.txt
         run: python tests/check_requirements.py
 
-      - name: Logs
+      - name: System logs
         if: ${{ always() }}
         # update logs_test.sh when adding more logsources here
         run: |
           tail -n +1 `pwd`/app/tmp/logs/*
           tail -n +1 /var/log/apache2/*.log
 
+      - name: Application logs
+        if: ${{ always() }}
+        run: |
           app/Console/cake Log export /tmp/logs.json.gz --without-changes
           zcat /tmp/logs.json.gz
 

.gitignore

@@ -82,10 +82,20 @@ app/Lib/EventWarning/Custom/*
 !/app/webroot/img/orgs/MISP.png
 !/app/webroot/img/orgs/NATO.png
 !/app/webroot/img/orgs/NCIRC.png
+/app/files/img/custom/*
+!/app/files/img/custom/empty
+!/app/files/img/orgs
+/app/files/img/orgs/*
+!/app/files/img/orgs/ADMIN.png
+!/app/files/img/orgs/MIL.be.png
+!/app/files/img/orgs/MISP.png
+!/app/files/img/orgs/NATO.png
+!/app/files/img/orgs/NCIRC.png
 /app/Config/bootstrap.php
 /app/Config/database.php
 /app/Config/core.php
 /app/Config/config.php
+/app/Config/hmac_key.php
 /app/Console/Command/training.json
 /app/Lib/cakephp
 /app/webroot/gpg.asc

INSTALL/MYSQL.sql

@@ -1428,7 +1428,7 @@ CREATE TABLE IF NOT EXISTS `users` (
 -- Table structure for table `user_login_profiles`
 --
 
-CREATE TABLE `user_login_profiles` (
+CREATE TABLE IF NOT EXISTS `user_login_profiles` (
   `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
   `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
   `user_id` int(11) NOT NULL,
@@ -1698,4 +1698,4 @@ INSERT IGNORE INTO `org_blocklists` (`org_uuid`, `created`, `org_name`, `comment
 
 INSERT IGNORE INTO `admin_settings` (`setting`, `value`) VALUES 
 ('fix_login', NOW()),
-('default_role', 3);
+('default_role', 3);

PyMISP

@@ -1 +1 @@
-Subproject commit c0077c19cf3843a65d5b26076dd26deca0405033
+Subproject commit 8b4f98ac4c2e6c8cc1dba064f937dac816b67d0f

README.md

@@ -46,6 +46,8 @@ The objective of MISP is to foster the sharing of structured information within
 </tr>
 </table>
 
+[![CLA FREE initiative](https://raw.githubusercontent.com/ossbase-org/ossbase.org/main/logos/cla-free-small.png)](https://ossbase.org/initiatives/cla-free/)
+
 Core functions
 ------------------
 - An **efficient IOC and indicators** database, allowing to store technical and non-technical information about malware samples, incidents, attackers and intelligence.

VERSION.json

@@ -1 +1 @@
-{"major":2, "minor":4, "hotfix":184}
+{"major":2, "minor":4, "hotfix":192}

app/Config/bootstrap.default.php

@@ -180,3 +180,7 @@ CakeLog::config('error', array(
 CakePlugin::loadAll(array(
 	'CakeResque' => array('bootstrap' => true)
 ));
+
+
+// Enable the additional exception logging for certain failures (timeouts, out of memory, etc)
+Configure::write('Exception.renderer', 'AppExceptionRenderer');

app/Console/Command/AdminShell.php

@@ -302,16 +302,22 @@ class AdminShell extends AppShell
     public function updateJSON()
     {
         $this->out('Updating all JSON structures.');
-        $results = $this->Server->updateJSON();
-        foreach ($results as $type => $result) {
+        $overallSuccess = true;
+        foreach ($this->Server->updateJSON() as $type => $result) {
             $type = Inflector::pluralize(Inflector::humanize($type));
-            if ($result !== false) {
-                $this->out(__('%s updated.', $type));
+            if ($result['success']) {
+                $this->out(__('%s updated in %.2f seconds.', $type, $result['duration']));
             } else {
                 $this->out(__('Could not update %s.', $type));
+                $this->out($result['result']);
+                $overallSuccess = false;
             }
         }
-        $this->out('All JSON structures updated. Thank you and have a very safe and productive day.');
+        if ($overallSuccess) {
+            $this->out('All JSON structures updated. Thank you and have a very safe and productive day.');
+        } else {
+            $this->error('Some structure could no be updated');
+        }
     }
 
     public function updateGalaxies()
@@ -616,9 +622,9 @@ class AdminShell extends AppShell
         try {
             $redis = RedisTool::init();
             for ($i = 0; $i < 10; $i++) {
-                $persistence = $redis->info('persistence');
-                if (isset($persistence['loading']) && $persistence['loading']) {
-                    $this->out('Redis is still loading...');
+                $pong = $redis->ping();
+                if ($pong !== true) {
+                    $this->out('Redis is still loading... ' . $pong);
                     sleep(1);
                 } else {
                     break;

app/Console/Command/AppShell.php

@@ -18,6 +18,7 @@
 
 App::uses('AppModel', 'Model');
 App::uses('BackgroundJobsTool', 'Tools');
+App::uses('BenchmarkTool', 'Tools');
 
 require_once dirname(__DIR__) . '/../Model/Attribute.php';   // FIXME workaround bug where Vendor/symfony/polyfill-php80/Resources/stubs/Attribute.php is loaded instead
 
@@ -38,7 +39,18 @@ abstract class AppShell extends Shell
     {
         $configLoad = $this->Tasks->load('ConfigLoad');
         $configLoad->execute();
-
+        if (Configure::read('Plugin.Benchmarking_enable')) {
+            $Benchmark = new BenchmarkTool(ClassRegistry::init('User'));
+            $start_time = $Benchmark->startBenchmark();
+            register_shutdown_function(function () use ($start_time, $Benchmark) {
+                $Benchmark->stopBenchmark([
+                    'user' => 0,
+                    'controller' => 'Shell::' . $this->modelClass,
+                    'action' => $this->command,
+                    'start_time' => $start_time
+                ]);
+            });
+        }
         parent::initialize();
     }
 

app/Console/Command/Ls22Shell.php

@@ -20,7 +20,8 @@ class Ls22Shell extends AppShell
                 $this->__servers[] = [
                     'Server' => [
                         'url' => trim($fields[0]),
-                        'authkey' => trim($fields[2])
+                        'authkey' => trim($fields[2]),
+                        'self_signed' => true,
                     ]
                 ];
             }
@@ -30,6 +31,7 @@ class Ls22Shell extends AppShell
     public function getOptionParser()
     {
         $this->stdout->styles('green', array('text' => 'green'));
+        $this->stdout->styles('black', array('text' => 'black'));
 
         $parser = parent::getOptionParser();
         $parser->addSubcommand('enableTaxonomy', [
@@ -414,6 +416,13 @@ class Ls22Shell extends AppShell
 
     public function scores()
     {
+        $MITIGATION_DETECTION_OBJECT_UUIDs = [
+            'b5acf82e-ecca-4868-82fe-9dbdf4d808c3', # yara
+            '3c177337-fb80-405a-a6c1-1b2ddea8684a', # suricata
+            'aa21a3cd-ab2c-442a-9999-a5e6626591ec', # sigma
+            '6bce7d01-dbec-4054-b3c2-3655a19382e2', # script
+            '35b4dd03-4fa9-4e0e-97d8-a2867b11c956', # yabin
+        ];
         $results = [];
         $this->__getInstances($this->param('instances'));
         $server = null;
@@ -464,6 +473,7 @@ class Ls22Shell extends AppShell
             $params = [
                 'org' => $org_id,
                 'includeWarninglistHits' => true,
+                // 'includeAnalystData' => true,
             ];
             if (!empty($time_range)) {
                 $params['publish_timestamp'] = $time_range;
@@ -486,6 +496,8 @@ class Ls22Shell extends AppShell
                 'warnings' => 0,
                 'events_extended' => 0,
                 'extending_events' => 0,
+                'mitigation_detection_rules_count' => 0,
+                'analyst_data_count' => 0,
             ];
             foreach ($events['response'] as $event) {
                 $event_uuid_per_org[$event['Event']['uuid']] = $event['Event']['Orgc']['name'];
@@ -511,6 +523,11 @@ class Ls22Shell extends AppShell
                         }
                     }
                 }
+
+                #  ['Note' => 0, 'Opinion' => 0, 'Relationship' => 0,]
+                $analystDataCount = $this->countAnalystData($event['Event'], $org_name);
+                $results[$org_name]['analyst_data_count'] = $analystDataCount['Note'] + $analystDataCount['Opinion'] + $analystDataCount['Relationship'];
+
                 foreach ($event['Event']['Attribute'] as $attribute) {
                     if (!empty($attribute['referenced_by'])) {
                         $results[$org_name]['connected_elements'] +=1;
@@ -533,6 +550,9 @@ class Ls22Shell extends AppShell
                     foreach ($event['Event']['Object'] as $object) {
                         $results[$org_name]['attribute_count'] += count($object['Attribute']);
                         $results[$org_name]['object_count'] += 1;
+                        if (in_array($object['template_uuid'], $MITIGATION_DETECTION_OBJECT_UUIDs)) {
+                            $results[$org_name]['mitigation_detection_rules_count'] += 1;
+                        }
                         if (!empty($object['ObjectReference'])) {
                             $results[$org_name]['connected_elements'] += 1;
                         }
@@ -547,6 +567,9 @@ class Ls22Shell extends AppShell
                                 }
                             }
                         }
+                        if (!empty($attribute['warnings'])) {
+                            $results[$org_name]['warnings'] += 1;
+                        }
                     }
                 }
 
@@ -579,13 +602,24 @@ class Ls22Shell extends AppShell
                 } else {
                     $results[$k]['metrics']['warnings'] = 0;
                 }
+                $results[$k]['metrics']['mitigation_detection_rules'] = 100 * ($result['mitigation_detection_rules_count'] / ($result['event_count']));
                 $results[$k]['metrics']['connectedness'] = 100 * ($result['connected_elements'] / ($result['attribute_count'] + $result['object_count']));
                 $results[$k]['metrics']['attack_weight'] = 100 * (2*($result['attack']) + $result['attribute_attack']) / ($result['attribute_count'] + $result['object_count']);
                 $results[$k]['metrics']['other_weight'] = 100 * (2*($result['other']) + $result['attribute_other']) / ($result['attribute_count'] + $result['object_count']);
-                $results[$k]['metrics']['collaboration'] = 100 * ((2*$result['events_extended'] + $result['extending_events']) / $result['event_count']);
-                $results[$k]['metrics']['collaboration'] = 100 * (2*(2*$result['events_extended'] + $result['extending_events']) / $result['event_count']);
+                // $results[$k]['metrics']['collaboration'] = 100 * ((2*$result['events_extended'] + $result['extending_events']) / $result['event_count']);
+                // $results[$k]['metrics']['collaboration'] = 100 * (2*(2*$result['events_extended'] + $result['extending_events']) / $result['event_count']);
+                $results[$k]['metrics']['collaboration'] = 100 * (($result['events_extended'] + $result['extending_events']));
+                
+                # Math magic to give lots of points of you extend or have your events extended. You quickly get point, but it slows down very quick
+                if (($result['events_extended'] + $result['extending_events']) == 0) {
+                    $results[$k]['metrics']['collaboration'] = 0;
+                } else {
+                    $results[$k]['metrics']['collaboration'] = min(5*log(($result['events_extended'] + $result['extending_events']), 1.17), 100);
+                }
+
+                $results[$k]['metrics']['collaboration_analyst'] = $result['analyst_data_count'] > 0 ? 100 : 0;
             }
-            foreach (['connectedness', 'attack_weight', 'other_weight', 'warnings', 'collaboration'] as $metric) {
+            foreach (['connectedness',  'mitigation_detection_rules', 'attack_weight', 'other_weight', 'warnings', 'collaboration', 'collaboration_analyst'] as $metric) {
                 if (empty($results[$k]['metrics'][$metric])) {
                     $results[$k]['metrics'][$metric] = 0;
                 }
@@ -594,18 +628,21 @@ class Ls22Shell extends AppShell
                 }
             }
             $results[$k]['score'] = round(
-                    20 * $results[$k]['metrics']['warnings'] +
-                    20 * $results[$k]['metrics']['connectedness'] +
+                    15 * $results[$k]['metrics']['warnings'] +
+                    15 * $results[$k]['metrics']['mitigation_detection_rules'] +
+                    10 * $results[$k]['metrics']['connectedness'] +
                     40 * $results[$k]['metrics']['attack_weight'] +
                     10 * $results[$k]['metrics']['other_weight'] +
+                    // 7 * $results[$k]['metrics']['collaboration'] + 3 * $results[$k]['metrics']['collaboration_analyst']
                     10 * $results[$k]['metrics']['collaboration']
                 ) / 100;
             $scores[$k]['total'] = $results[$k]['score'];
-            $scores[$k]['warnings'] = round(20 * $results[$k]['metrics']['warnings']);
-            $scores[$k]['connectedness'] = round(20 * $results[$k]['metrics']['connectedness']);
+            $scores[$k]['warnings'] = round(15 * $results[$k]['metrics']['warnings']);
+            $scores[$k]['mitigation_detection_rules'] = round(15 * $results[$k]['metrics']['mitigation_detection_rules']);
+            $scores[$k]['connectedness'] = round(10 * $results[$k]['metrics']['connectedness']);
             $scores[$k]['attack_weight'] = round(40 * $results[$k]['metrics']['attack_weight']);
             $scores[$k]['other_weight'] = round(10 * $results[$k]['metrics']['other_weight']);
-            $scores[$k]['collaboration'] = round(10 * $results[$k]['metrics']['collaboration']);
+            $scores[$k]['collaboration'] = round(7 * $results[$k]['metrics']['collaboration']) + round(3 * $results[$k]['metrics']['collaboration_analyst']);
         }
         arsort($scores, SORT_DESC);
         $this->out(str_repeat('=', 128), 1, Shell::NORMAL);
@@ -618,20 +655,22 @@ class Ls22Shell extends AppShell
         $this->out(str_repeat('=', 128), 1, Shell::NORMAL);
         foreach ($scores as $org => $score) {
             $score_string[0] = str_repeat('█', round($score['warnings']/100));
-            $score_string[1] = str_repeat('█', round($score['connectedness']/100));
-            $score_string[2] = str_repeat('█', round($score['attack_weight']/100));
-            $score_string[3] = str_repeat('█', round($score['other_weight']/100));
-            $score_string[4] = str_repeat('█', round($score['collaboration']/100));
+            $score_string[1] = str_repeat('█', round($score['mitigation_detection_rules']/100));
+            $score_string[2] = str_repeat('█', round($score['connectedness']/100));
+            $score_string[3] = str_repeat('█', round($score['attack_weight']/100));
+            $score_string[4] = str_repeat('█', round($score['other_weight']/100));
+            $score_string[5] = str_repeat('█', round($score['collaboration']/100));
             $this->out(sprintf(
                 '| %s | %s | %s |',
                 str_pad($org, 10, ' ', STR_PAD_RIGHT),
                 sprintf(
-                    '<error>%s</error><warning>%s</warning><question>%s</question><info>%s</info><green>%s</green>%s',
+                    '<error>%s</error><black>%s</black><warning>%s</warning><question>%s</question><info>%s</info><green>%s</green>%s',
                     $score_string[0],
                     $score_string[1],
                     $score_string[2],
                     $score_string[3],
                     $score_string[4],
+                    $score_string[5],
                     str_repeat(' ', 100 - mb_strlen(implode('', $score_string)))
                 ),
                 str_pad($score['total'] . '%', 8, ' ', STR_PAD_RIGHT)
@@ -639,8 +678,9 @@ class Ls22Shell extends AppShell
         }
         $this->out(str_repeat('=', 128), 1, Shell::NORMAL);
         $this->out(sprintf(
-            '| Legend: %s %s %s %s %s |',
+            '| Legend: %s %s %s %s %s %s |',
             '<error>█: Warnings</error>',
+            '<black>█: Detection/Mitigation Rules</black>',
             '<warning>█: Connectedness</warning>',
             '<question>█: ATT&CK context</question>',
             '<info>█: Other Context</info>',
@@ -650,4 +690,30 @@ class Ls22Shell extends AppShell
         $this->out(str_repeat('=', 128), 1, Shell::NORMAL);
         file_put_contents(APP . 'tmp/report.json', json_encode($results, JSON_PRETTY_PRINT));
     }
+
+    private function countAnalystData($data, $orgc_name): array {
+        $analystTypes = ['Note', 'Opinion', 'Relationship'];
+        $counts = [
+            'Note' => 0,
+            'Opinion' => 0,
+            'Relationship' => 0,
+        ];
+
+        foreach ($analystTypes as $type) {
+            if (!empty($data[$type])) {
+                foreach ($data[$type] as $entry) {
+                    if ($entry['Orgc']['name'] == $orgc_name) {
+                        $counts[$type] += 1;
+                    }
+                }
+                foreach ($data[$type] as $child) {
+                    $nestedCounts = $this->countAnalystData($child, $orgc_name);
+                    $counts['Note'] += $nestedCounts['Note'];
+                    $counts['Opinion'] += $nestedCounts['Opinion'];
+                    $counts['Relationship'] += $nestedCounts['Relationship'];
+                }
+            }
+        }
+        return $counts;
+    }
 }

app/Console/Command/OrganisationShell.php

@@ -0,0 +1,64 @@
+<?php
+
+/**
+ * @property User $User
+ * @property Log $Log
+ * @property UserLoginProfile $UserLoginProfile
+ */
+class OrganisationShell extends AppShell
+{
+    public $uses = ['Organisation'];
+
+    public function getOptionParser()
+    {
+        $parser = parent::getOptionParser();
+        $parser->addSubcommand('list', [
+            'help' => __('Get list of organisations.'),
+            'parser' => [
+                'arguments' => [
+                    'filter' => ['help' => __('Filter the list by name.'), 'required' => false],
+                    'local' => ['help' => __('Filter the list by local/known organisations.'), 'required' => false],
+                ],
+                'options' => [
+                    'json' => ['help' => __('Output as JSON.'), 'boolean' => true],
+                ],
+            ]
+        ]);
+        return $parser;
+    }
+
+    public function list()
+    {
+        $filter = $this->args[0] ?? null;
+        $local = isset($this->args[1]) ? $this->args[1] : null;
+        $conditions = [];
+        if ($filter) {
+            $conditions = ['OR' => [
+                'Organisation.name LIKE' => "%$filter%",
+                'Organisation.uuid LIKE' => "%$filter%"
+            ]];
+        }
+        if ($local !== null) {
+            $conditions['OR'][] = [
+                'Organisation.local' => $local
+            ];
+        }
+        $organisations = $this->Organisation->find('all', [
+            'recursive' => -1,
+            'conditions' => $conditions
+        ]);
+        if ($this->params['json']) {
+            $this->out($this->json($organisations));
+        } else {
+            foreach ($organisations as $organisation) {
+                $this->out(sprintf(
+                    '%d. [%s] %s',
+                    $organisation['Organisation']['id'],
+                    $organisation['Organisation']['uuid'],
+                    $organisation['Organisation']['name']
+                ));
+            }
+            $this->out(count($organisations) . ' hits.');
+        }
+    }
+}

app/Console/Command/RoleShell.php

@@ -0,0 +1,56 @@
+<?php
+
+/**
+ * @property User $User
+ * @property Log $Log
+ * @property UserLoginProfile $UserLoginProfile
+ */
+class RoleShell extends AppShell
+{
+    public $uses = ['Role'];
+
+    public function getOptionParser()
+    {
+        $parser = parent::getOptionParser();
+        $parser->addSubcommand('list', [
+            'help' => __('Get list of the roles.'),
+            'parser' => [
+                'arguments' => [
+                    'filter' => ['help' => __('Filter list by name.'), 'required' => false],
+                ],
+                'options' => [
+                    'json' => ['help' => __('Output as JSON.'), 'boolean' => true],
+                ],
+            ]
+        ]);
+        return $parser;
+    }
+
+    public function list()
+    {
+        $filter = $this->args[0] ?? null;
+        if ($filter) {
+            $conditions = ['OR' => [
+                'Role.name LIKE' => "%$filter%"
+            ]];
+        } else {
+            $conditions = [];
+        }
+        $roles = $this->Role->find('all', [
+            'recursive' => -1,
+            'conditions' => $conditions
+        ]);
+        if ($this->params['json']) {
+            $this->out($this->json($roles));
+        } else {
+            foreach ($roles as $role) {
+                $this->out(sprintf(
+                    '%d. %s',
+                    $role['Role']['id'],
+                    $role['Role']['name']
+                ));
+            }
+        }
+        $this->out(count($roles) . ' hits.');
+    }
+}

app/Console/Command/ServerShell.php

@@ -85,6 +85,7 @@ class ServerShell extends AppShell
 
         $userId = $this->args[0];
         $user = $this->getUser($userId);
+        Configure::write('CurrentUserId', $userId);
 
         if (!empty($this->args[1])) {
             $technique = $this->args[1];
@@ -124,11 +125,11 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Pull'] . PHP_EOL);
         }
-
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $serverId = $this->args[1];
         $server = $this->getServer($serverId);
+        Configure::write('CurrentUserId', $userId);
         if (!empty($this->args[2])) {
             $technique = $this->args[2];
         } else {
@@ -143,10 +144,14 @@ class ServerShell extends AppShell
         if (!empty($this->args[4]) && $this->args[4] === 'force') {
             $force = true;
         }
+
+        // Try to enable garbage collector as pulling events can use a lot of memory
+        gc_enable();
+
         try {
             $result = $this->Server->pull($user, $technique, $server, $jobId, $force);
             if (is_array($result)) {
-                $message = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4]);
+                $message = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled, %s analyst data pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4], $result[5]);
                 $this->Job->saveStatus($jobId, true, $message);
             } else {
                 $message = __('ERROR: %s', $result);
@@ -164,11 +169,12 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Push'] . PHP_EOL);
         }
-
+        
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $serverId = $this->args[1];
         $server = $this->getServer($serverId);
+        Configure::write('CurrentUserId', $userId);
         $technique = empty($this->args[2]) ? 'full' : $this->args[2];
         if (!empty($this->args[3])) {
             $jobId = $this->args[3];
@@ -203,6 +209,7 @@ class ServerShell extends AppShell
         $userId = $this->args[0];
         $user = $this->getUser($userId);
 
+        Configure::write('CurrentUserId', $userId);
         $technique = isset($this->args[1]) ? $this->args[1] : 'full';
 
         $servers = $this->Server->find('list', array(
@@ -366,7 +373,7 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Fetch feeds as local data'] . PHP_EOL);
         }
-
+        
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $feedId = $this->args[1];
@@ -422,7 +429,7 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Cache server'] . PHP_EOL);
         }
-
+        
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $scope = $this->args[1];
@@ -485,7 +492,7 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Cache feeds for quick lookups'] . PHP_EOL);
         }
-
+        
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $scope = $this->args[1];
@@ -731,6 +738,7 @@ class ServerShell extends AppShell
 
     public function sendPeriodicSummaryToUsers()
     {
+        
         $periods = $this->__getPeriodsForToday();
         $start_time = time();
         echo __n('Started periodic summary generation for the %s period', 'Started periodic summary generation for periods: %s', count($periods), implode(', ', $periods)) . PHP_EOL;
@@ -796,7 +804,7 @@ class ServerShell extends AppShell
         if (empty($this->args[0]) || empty($this->args[1])) {
             die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Push Taxii'] . PHP_EOL);
         }
-
+        
         $userId = $this->args[0];
         $user = $this->getUser($userId);
         $serverId = $this->args[1];

app/Console/Command/StatisticsShell.php

@@ -303,10 +303,10 @@ class StatisticsShell extends AppShell {
         $this->out(json_encode([
             'events' => $this->Event->find('count'),
             'attributes' => $this->Event->Attribute->find('count',
-                ['conditions' => ['Attribute.deleted' => 0], 'recursive' => -1]
+                ['recursive' => -1]
             ),
             'objects' => $this->Event->Object->find('count',
-                ['conditions' => ['Object.deleted' => 0], 'recursive' => -1]
+                ['recursive' => -1]
             ),
             'correlations' => $this->Correlation->find('count') / 2,
             'users' => $this->User->find('count',

app/Console/Command/UserShell.php

@@ -16,7 +16,21 @@ class UserShell extends AppShell
             'help' => __('Get list of user accounts.'),
             'parser' => [
                 'arguments' => [
-                    'userId' => ['help' => __('User ID or e-mail address.'), 'required' => true],
+                    'userId' => ['help' => __('User ID or e-mail address to filter.'), 'required' => false],
+                ],
+                'options' => [
+                    'json' => ['help' => __('Output as JSON.'), 'boolean' => true],
+                ],
+            ]
+        ]);
+        $parser->addSubcommand('create', [
+            'help' => __('Create a new user account.'),
+            'parser' => [
+                'arguments' => [
+                    'email' => ['help' => __('E-mail address (also used as the username.'), 'required' => true],
+                    'role_id' => ['help' => __('Role ID of the user. For a list of available roles, use `cake Roles list`.'), 'required' => true],
+                    'org_id' => ['help' => __('Organisation under which the user should be created'), 'required' => true],
+                    'password' => ['help' => __('Enter a password to assign to the user (optional) - if none is set, the user will receive a temporary password.')]
                 ],
                 'options' => [
                     'json' => ['help' => __('Output as JSON.'), 'boolean' => true],
@@ -82,6 +96,15 @@ class UserShell extends AppShell
                 ],
             ],
         ]);
+        $parser->addSubcommand('change_role', [
+            'help' => __('Change user role.'),
+            'parser' => [
+                'arguments' => [
+                    'userId' => ['help' => __('User ID or e-mail address.'), 'required' => true],
+                    'new_role' => ['help' => __('Role ID or Role name.'), 'required' => true],
+                ]
+            ],
+        ]);
         $parser->addSubcommand('change_authkey', [
             'help' => __('Change authkey. When advanced authkeys are enabled, old authkeys will be disabled.'),
             'parser' => [
@@ -180,6 +203,48 @@ class UserShell extends AppShell
         }
     }
 
+    public function create()
+    {
+        if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
+            $this->err('Invalid input. Usage: `User create [email] [role_id] [org_id] [password:optional]`');
+        }
+        $user = [
+            'email' => $this->args[0],
+            'role_id' => $this->args[1],
+            'org_id' => $this->args[2],
+            'change_pw' => true
+        ];
+        if (!empty($this->args[3])) {
+            $user['password'] = $this->args[3];
+            $user['confirm_password'] = $this->args[3];
+            $user['change_pw'] = true;
+        }
+        $this->User->create();
+        $result = $this->User->save($user);
+        // do not fetch sensitive or big values
+        $schema = $this->User->schema();
+        unset($schema['authkey']);
+        unset($schema['password']);
+        unset($schema['gpgkey']);
+        unset($schema['certif_public']);
+
+        $fields = array_keys($schema);
+        $fields[] = 'Role.*';
+        $fields[] = 'Organisation.*';
+
+        $user = $this->User->find('first', [
+            'recursive' => -1,
+            'fields' => $fields,
+            'conditions' => ['User.id' => $this->User->id],
+            'contain' => ['Organisation', 'Role', 'UserSetting'],
+        ]);
+        if ($this->params['json']) {
+            $this->out($this->json($user));
+        } else {
+            $this->out('User created.');
+        }
+    }
+
     public function init()
     {
         if (!Configure::read('Security.salt')) {
@@ -443,6 +508,35 @@ class UserShell extends AppShell
         }
     }
 
+    public function change_role()
+    {
+        list($userId, $newRole) = $this->args;
+        $user = $this->getUser($userId);
+
+        if (is_numeric($newRole)) {
+            $conditions = ['Role.id' => $newRole];
+        } else {
+            $conditions = ['Role.name' => $newRole];
+        }
+
+        $newRoleFromDb = $this->User->Role->find('first', [
+            'conditions' => $conditions,
+            'fields' => ['Role.id'],
+        ]);
+
+        if (empty($newRoleFromDb)) {
+            $this->error("Role `$newRole` not found.");
+        }
+
+        if ($newRoleFromDb['Role']['id'] == $user['role_id']) {
+            $this->error("Role `$newRole` is already assigned to {$user['email']}.");
+        }
+
+        $this->User->updateField($user, 'role_id', $newRoleFromDb['Role']['id']);
+
+        $this->out("Role changed from `{$user['role_id']}` to `{$newRoleFromDb['Role']['id']}`.");
+    }
+
     public function user_ips()
     {
         list($userId) = $this->args;
@@ -575,7 +669,7 @@ class UserShell extends AppShell
     }
 
     /**
-     * @param string|int $userId
+     * @param string|int $userId User ID or User e-mail
      * @return array
      */
     private function getUser($userId)

app/Console/Command/WorkflowShell.php

@@ -17,6 +17,9 @@ class WorkflowShell extends AppShell {
         $data = JsonTool::decode($this->args[1]);
         $logging = JsonTool::decode($this->args[2]);
         $jobId = $this->args[3];
+        if (!empty($this->args[4])) {
+            Configure::write('CurrentUserId', JsonTool::decode($this->args[4]));
+        }
 
         $blockingErrors = [];
         $executionSuccess = $this->Workflow->executeWorkflowForTrigger($trigger_id, $data, $blockingErrors);

app/Controller/AnalystDataBlocklistsController.php

@@ -0,0 +1,103 @@
+<?php
+App::uses('AppController', 'Controller');
+
+class AnalystDataBlocklistsController extends AppController
+{
+    public $components = array('Session', 'RequestHandler', 'BlockList');
+
+    public $paginate = array(
+            'limit' => 60,
+            'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 entries <- no we won't, this is the max a user van view/page.
+            'order' => array(
+                'AnalystDataBlocklist.created' => 'DESC'
+            ),
+    );
+
+    public function index()
+    {
+        $passedArgsArray = array();
+        $passedArgs = $this->passedArgs;
+        $params = array();
+        $validParams = array('analyst_data_uuid', 'comment', 'analyst_data_info', 'analyst_data_orgc');
+        foreach ($validParams as $validParam) {
+            if (!empty($this->params['named'][$validParam])) {
+                $params[$validParam] = $this->params['named'][$validParam];
+            }
+        }
+        if (!empty($this->params['named']['searchall'])) {
+            $params['AND']['OR'] = array(
+                'analyst_data_uuid' => $this->params['named']['searchall'],
+                'comment' => $this->params['named']['searchall'],
+                'analyst_data_info' => $this->params['named']['searchall'],
+                'analyst_data_orgc' => $this->params['named']['searchall']
+            );
+        }
+        $this->set('passedArgs', json_encode($passedArgs));
+        $this->set('passedArgsArray', $passedArgsArray);
+        return $this->BlockList->index($this->_isRest(), $params);
+    }
+
+    public function add()
+    {
+        $this->set('action', 'add');
+        return $this->BlockList->add($this->_isRest());
+    }
+
+    public function edit($id)
+    {
+        $this->set('action', 'edit');
+        return $this->BlockList->edit($this->_isRest(), $id);
+    }
+
+    public function delete($id)
+    {
+        if (Validation::uuid($id)) {
+            $entry = $this->AnalystDataBlocklist->find('first', array(
+                'conditions' => array('analyst_data_uuid' => $id)
+            ));
+            if (empty($entry)) {
+                throw new NotFoundException(__('Invalid blocklist entry'));
+            }
+            $id = $entry['AnalystDataBlocklist']['id'];
+        }
+        return $this->BlockList->delete($this->_isRest(), $id);
+    }
+
+    public function massDelete()
+    {
+        if ($this->request->is('post') || $this->request->is('put')) {
+            if (!isset($this->request->data['AnalystDataBlocklist'])) {
+                $this->request->data = array('AnalystDataBlocklist' => $this->request->data);
+            }
+            $ids = $this->request->data['AnalystDataBlocklist']['ids'];
+            $analyst_data_ids = json_decode($ids, true);
+            if (empty($analyst_data_ids)) {
+                throw new NotFoundException(__('Invalid Analyst Data IDs.'));
+            }
+            $result = $this->AnalystDataBlocklist->deleteAll(array('AnalystDataBlocklist.id' => $analyst_data_ids));
+            if ($result) {
+                if ($this->_isRest()) {
+                    return $this->RestResponse->saveSuccessResponse('AnalystDataBlocklist', 'Deleted', $ids, $this->response->type());
+                } else {
+                    $this->Flash->success('Blocklist entry removed');
+                    $this->redirect(array('controller' => 'AnalystDataBlocklist', 'action' => 'index'));
+                }
+            } else {
+                $error = __('Failed to delete Analyst Data from AnalystDataBlocklist. Error: ') . PHP_EOL . h($result);
+                if ($this->_isRest()) {
+                    return $this->RestResponse->saveFailResponse('AnalystDataBlocklist', 'Deleted', false, $error, $this->response->type());
+                } else {
+                    $this->Flash->error($error);
+                    $this->redirect(array('controller' => 'AnalystDataBlocklists', 'action' => 'index'));
+                }
+            }
+        } else {
+            $ids = json_decode($this->request->query('ids'), true);
+            if (empty($ids)) {
+                throw new NotFoundException(__('Invalid analyst data IDs.'));
+
+            }
+            $this->set('analyst_data_ids', $ids);
+        }
+    }
+}

app/Controller/AnalystDataController.php

@@ -0,0 +1,333 @@
+<?php
+App::uses('AppController', 'Controller');
+
+class AnalystDataController extends AppController
+{
+
+    public $components = ['Session', 'RequestHandler'];
+
+    public $paginate = [
+        'limit' => 60,
+        'order' => []
+    ];
+
+    public $uses = [
+        'Opinion',
+        'Note',
+        'Relationship'
+    ];
+
+    private $__valid_types = [
+        'Opinion',
+        'Note',
+        'Relationship'
+    ];
+
+    // public $modelSelection = 'Note';
+
+    private function _setViewElements()
+    {
+        $dropdownData = [];
+        $this->loadModel('Event');
+        $dropdownData['distributionLevels'] = $this->Event->distributionLevels;
+        $this->set('initialDistribution', Configure::read('MISP.default_event_distribution'));
+        $dropdownData['sgs'] = $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1);
+        $dropdownData['valid_targets'] = array_combine($this->AnalystData->valid_targets, $this->AnalystData->valid_targets);
+        $this->set(compact('dropdownData'));
+        $this->set('modelSelection', $this->modelSelection);
+        $this->set('distributionLevels', $this->Event->distributionLevels);
+        App::uses('LanguageRFC5646Tool', 'Tools');
+        $this->set('languageRFC5646', ['' => __('- No language -'), LanguageRFC5646Tool::getLanguages()]);
+    }
+    
+    public function add($type = 'Note', $object_uuid = null, $object_type = null)
+    {
+        $this->__typeSelector($type);
+        if (!empty($object_uuid)) {
+            $this->request->data[$this->modelSelection]['object_uuid'] = $object_uuid;
+        }
+        if (!empty($object_type)) {
+            $this->request->data[$this->modelSelection]['object_type'] = $object_type;
+        }
+        
+        if (empty($this->request->data[$this->modelSelection]['object_type']) && !empty($this->request->data[$this->modelSelection]['object_uuid'])) {
+            $this->request->data[$this->modelSelection]['object_type'] = $this->AnalystData->deduceType($object_uuid);
+        }
+        $params = [];
+        $this->CRUD->add($params);
+        if ($this->restResponsePayload) {
+            return $this->restResponsePayload;
+        }
+        $this->_setViewElements();
+        if ($type == 'Relationship') {
+            $this->set('existingRelations', $this->AnalystData->getExistingRelationships());
+        }
+        $this->set('menuData', array('menuList' => 'analyst_data', 'menuItem' => 'add_' . strtolower($type)));
+        $this->render('add');
+    }
+
+    public function edit($type = 'Note', $id)
+    {
+        if ($type === 'all' && Validation::uuid($id)) {
+            $this->loadModel('AnalystData');
+            $type = $this->AnalystData->deduceType($id);
+        }
+        $this->__typeSelector($type);
+        if (!is_numeric($id) && Validation::uuid($id)) {
+            $id = $this->AnalystData->getIDFromUUID($type, $id);
+        }
+
+        $this->set('id', $id);
+        $conditions = $this->AnalystData->buildConditions($this->Auth->user());
+        $params = [
+            'fields' => $this->AnalystData->getEditableFields(),
+            'conditions' => $conditions,
+            'afterFind' => function(array $analystData): array {
+                $canEdit = $this->ACL->canEditAnalystData($this->Auth->user(), $analystData, $this->modelSelection);
+                if (!$canEdit) {
+                    throw new MethodNotAllowedException(__('You are not authorised to do that.'));
+                }
+                return $analystData;
+            },
+            'beforeSave' => function(array $analystData): array {
+                $analystData[$this->modelSelection]['modified'] = date('Y-m-d H:i:s');
+                return $analystData;
+            }
+        ];
+        $this->CRUD->edit($id, $params);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }
+        $this->_setViewElements();
+        if ($type == 'Relationship') {
+            $this->set('existingRelations', $this->AnalystData->getExistingRelationships());
+        }
+        $this->set('menuData', array('menuList' => 'analyst_data', 'menuItem' => 'edit'));
+        $this->render('add');
+    }
+
+    public function delete($type = 'Note', $id, $hard=true)
+    {
+        if ($type === 'all' && Validation::uuid($id)) {
+            $this->loadModel('AnalystData');
+            $type = $this->AnalystData->deduceType($id);
+        }
+        $this->__typeSelector($type);
+        if (!is_numeric($id) && Validation::uuid($id)) {
+            $id = $this->AnalystData->getIDFromUUID($type, $id);
+        }
+
+        $params = [
+            'afterFind' => function(array $analystData) {
+                $canEdit = $this->ACL->canEditAnalystData($this->Auth->user(), $analystData, $this->modelSelection);
+                if (!$canEdit) {
+                    throw new MethodNotAllowedException(__('You are not authorised to do that.'));
+                }
+                return $analystData;
+            },
+            'afterDelete' => function($deletedAnalystData) use ($hard) {
+                if (empty($hard)) {
+                    return;
+                }
+                $type = $this->AnalystData->deduceAnalystDataType($deletedAnalystData);
+                $info = '- Unsupported analyst type -';
+                if ($type === 'Note') {
+                    $info = $deletedAnalystData[$type]['note'];
+                } else if ($type === 'Opinion') {
+                    $info = sprintf('%s/100 :: %s', $deletedAnalystData[$type]['opinion'], $deletedAnalystData[$type]['comment']);
+                } else if ($type === 'Relationship') {
+                    $info = sprintf('-- %s --> %s :: %s', $deletedAnalystData[$type]['relationship_type'] ?? '[undefined]', $deletedAnalystData[$type]['related_object_type'], $deletedAnalystData[$type]['related_object_uuid']);
+                }
+                $this->AnalystDataBlocklist = ClassRegistry::init('AnalystDataBlocklist');
+                $this->AnalystDataBlocklist->create();
+                if (!empty($deletedAnalystData[$type]['orgc_uuid'])) {
+                    if (!empty($deletedAnalystData[$type]['Orgc'])) {
+                        $orgc = $deletedAnalystData[$type];
+                    } else {
+                        $orgc = $this->Orgc->find('first', array(
+                            'conditions' => ['Orgc.uuid' => $deletedAnalystData[$type]['orgc_uuid']],
+                            'recursive' => -1,
+                            'fields' => ['Orgc.name'],
+                        ));
+                    }
+                } else {
+                    $orgc = ['Orgc' => ['name' => 'MISP']];
+                }
+                $this->AnalystDataBlocklist->save(['analyst_data_uuid' => $deletedAnalystData[$type]['uuid'], 'analyst_data_info' => $info, 'analyst_data_orgc' => $orgc['Orgc']['name']]);
+            }
+        ];
+        $this->CRUD->delete($id, $params);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }
+        
+    }
+
+    public function view($type = 'Note', $id)
+    {
+        if ($type === 'all' && Validation::uuid($id)) {
+            $this->loadModel('AnalystData');
+            $type = $this->AnalystData->getAnalystDataTypeFromUUID($id);
+        }
+        $this->__typeSelector($type);
+        if (!is_numeric($id) && Validation::uuid($id)) {
+            $id = $this->AnalystData->getIDFromUUID($type, $id);
+        }
+
+        if (!$this->IndexFilter->isRest()) {
+            $this->AnalystData->fetchRecursive = true;
+        }
+        $conditions = $this->AnalystData->buildConditions($this->Auth->user());
+        $this->CRUD->view($id, [
+            'conditions' => $conditions,
+            'contain' => ['Org', 'Orgc'],
+            'afterFind' => function(array $analystData) {
+                if (!$this->request->is('ajax')) {
+                    unset($analystData[$this->modelSelection]['_canEdit']);
+                }
+                return $analystData;
+            }
+        ]);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }
+        $this->set('id', $id);
+        $this->loadModel('Event');
+        $this->_setViewElements();
+        $this->set('distributionLevels', $this->Event->distributionLevels);
+        $this->set('shortDist', $this->Event->shortDist);
+        $this->set('menuData', array('menuList' => 'analyst_data', 'menuItem' => 'view'));
+        $this->render('view');
+    }
+
+    public function index($type = 'Note')
+    {
+        $this->__typeSelector($type);
+
+        $conditions = $this->AnalystData->buildConditions($this->Auth->user());
+        $params = [
+            'filters' => ['uuid', 'target_object'],
+            'quickFilters' => ['name'],
+            'conditions' => $conditions,
+            'afterFind' => function(array $data) {
+                foreach ($data as $i => $analystData) {
+                    if (!$this->request->is('ajax')) {
+                        unset($analystData[$this->modelSelection]['_canEdit']);
+                    }
+                }
+                return $data;
+            }
+        ];
+        $this->CRUD->index($params);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }
+        $this->_setViewElements();
+        $this->set('menuData', array('menuList' => 'analyst_data', 'menuItem' => 'index'));
+    }
+
+    public function getRelatedElement($type, $uuid)
+    {
+        $this->__typeSelector('Relationship');
+        $data = $this->AnalystData->getRelatedElement($this->Auth->user(), $type, $uuid);
+        return $this->RestResponse->viewData($data, 'json');
+    }
+
+    public function getChildren($type = 'Note', $uuid, $depth=2)
+    {
+        $this->__typeSelector($type);
+        $data = $this->AnalystData->getChildren($this->Auth->user(), $uuid, $depth);
+        return $this->RestResponse->viewData($data, 'json');
+    }
+
+    public function filterAnalystDataForPush()
+    {
+        if (!$this->request->is('post')) {
+            throw new MethodNotAllowedException(__('This function is only accessible via POST requests.'));
+        }
+
+        $this->loadModel('AnalystData');
+
+        $allIncomingAnalystData = $this->request->data;
+        $allData = $this->AnalystData->filterAnalystDataForPush($allIncomingAnalystData);
+
+        return $this->RestResponse->viewData($allData, $this->response->type());
+    }
+
+    public function indexMinimal()
+    {
+        $this->loadModel('AnalystData');
+        $filters = [];
+        if ($this->request->is('post')) {
+            $filters = $this->request->data;
+        }
+        $options = [];
+        if (!empty($filters['orgc_name'])) {
+            $orgcNames = $filters['orgc_name'];
+            if (!is_array($orgcNames)) {
+                $orgcName = [$orgcNames];
+            }
+            $filterName = 'orgc_uuid';
+            foreach ($orgcNames as $orgcName) {
+                if ($orgcName[0] === '!') {
+                    $orgc = $this->AnalystData->Orgc->fetchOrg(substr($orgcName, 1));
+                    if ($orgc === false) {
+                        continue;
+                    }
+                    $options[]['AND'][] = ["{$filterName} !=" => $orgc['uuid']];
+                } else {
+                    $orgc = $this->AnalystData->Orgc->fetchOrg($orgcName);
+                    if ($orgc === false) {
+                        continue;
+                    }
+                    $options['OR'][] = [$filterName => $orgc['uuid']];
+                }
+            }
+        }
+        $allData = $this->AnalystData->indexMinimal($this->Auth->user(), $options);
+
+        return $this->RestResponse->viewData($allData, $this->response->type());
+    }
+
+    public function pushAnalystData()
+    {
+        if (!$this->Auth->user()['Role']['perm_sync'] || !$this->Auth->user()['Role']['perm_analyst_data']) {
+            throw new MethodNotAllowedException(__('You do not have the permission to do that.'));
+        }
+        if (!$this->_isRest()) {
+            throw new MethodNotAllowedException(__('This action is only accessible via a REST request.'));
+        }
+        if ($this->request->is('post')) {
+            $this->loadModel('AnalystData');
+            $analystData = $this->request->data;
+            $saveResult = $this->AnalystData->captureAnalystData($this->Auth->user(), $analystData);
+            $messageInfo = __('%s imported, %s ignored, %s failed. %s', $saveResult['imported'], $saveResult['ignored'], $saveResult['failed'], !empty($saveResult['errors']) ? implode(', ', $saveResult['errors']) : '');
+            if ($saveResult['success']) {
+                $message = __('Analyst Data imported. ') . $messageInfo;
+                return $this->RestResponse->saveSuccessResponse('AnalystData', 'pushAnalystData', false, $this->response->type(), $message);
+            } else {
+                $message = __('Could not import analyst data. ') . $messageInfo;
+                return $this->RestResponse->saveFailResponse('AnalystData', 'pushAnalystData', false, $message);
+            }
+        }
+    }
+
+    private function __typeSelector($type) {
+        foreach ($this->__valid_types as $vt) {
+            if ($type === $vt) {
+                $this->modelSelection = $vt;
+                $this->loadModel($vt);
+                $this->AnalystData = $this->{$vt};
+                $this->modelClass = $vt;
+                $this->{$vt}->current_user = $this->Auth->user();
+                if (!empty($this->request->data)) {
+                    if (!isset($this->request->data[$type])) {
+                        $this->request->data = [$type => $this->request->data];
+                    }
+                }
+                return $vt;
+            }
+        }
+        throw new MethodNotAllowedException(__('Invalid type.'));
+    }
+}

app/Controller/AppController.php

@@ -33,13 +33,19 @@ class AppController extends Controller
 
     public $helpers = array('OrgImg', 'FontAwesome', 'UserName');
 
-    private $__queryVersion = '158';
-    public $pyMispVersion = '2.4.184';
+    private $__queryVersion = '162';
+    public $pyMispVersion = '2.4.190';
     public $phpmin = '7.2';
     public $phprec = '7.4';
     public $phptoonew = '8.0';
     private $isApiAuthed = false;
 
+    /** @var redis */
+    private $redis = null;
+
+    /** @var benchmark_results */
+    private $benchmark_results = null;
+
     public $baseurl = '';
 
     public $restResponsePayload = null;
@@ -57,9 +63,14 @@ class AppController extends Controller
     /** @var ACLComponent */
     public $ACL;
 
+    /** @var BenchmarkComponent */
+    public $Benchmark;
+
     /** @var RestResponseComponent */
     public $RestResponse;
 
+    public $start_time;
+
     public function __construct($request = null, $response = null)
     {
         parent::__construct($request, $response);
@@ -97,13 +108,19 @@ class AppController extends Controller
 
     public function beforeFilter()
     {
-        $controller = $this->request->params['controller'];
-        $action = $this->request->params['action'];
-
         if (Configure::read('MISP.system_setting_db')) {
             App::uses('SystemSetting', 'Model');
             SystemSetting::setGlobalSetting();
         }
+        
+        $this->User = ClassRegistry::init('User');
+        if (Configure::read('Plugin.Benchmarking_enable')) {
+            App::uses('BenchmarkTool', 'Tools');
+            $this->Benchmark = new BenchmarkTool($this->User);
+            $this->start_time = $this->Benchmark->startBenchmark();
+        }
+        $controller = $this->request->params['controller'];
+        $action = $this->request->params['action'];
 
         $this->_setupBaseurl();
         $this->Auth->loginRedirect = $this->baseurl . '/users/routeafterlogin';
@@ -147,8 +164,6 @@ class AppController extends Controller
             Configure::write('Config.language', 'eng');
         }
 
-        $this->User = ClassRegistry::init('User');
-
         if (!empty($this->request->params['named']['disable_background_processing'])) {
             Configure::write('MISP.background_jobs', 0);
         }
@@ -226,6 +241,7 @@ class AppController extends Controller
         ) {
             // REST authentication
             if ($this->_isRest() || $this->_isAutomation()) {
+                
                 // disable CSRF for REST access
                 $this->Security->csrfCheck = false;
                 $loginByAuthKeyResult = $this->__loginByAuthKey();
@@ -306,6 +322,7 @@ class AppController extends Controller
             $this->set('isAclTagger', $role['perm_tagger']);
             $this->set('isAclGalaxyEditor', !empty($role['perm_galaxy_editor']));
             $this->set('isAclSighting', $role['perm_sighting'] ?? false);
+            $this->set('isAclAnalystDataCreator', $role['perm_analyst_data'] ?? false);
             $this->set('aclComponent', $this->ACL);
             $this->userRole = $role;
 
@@ -632,21 +649,21 @@ class AppController extends Controller
         }
 
         // Check if user accepted terms and conditions
-        if (!$user['termsaccepted'] && !empty(Configure::read('MISP.terms_file')) && !$this->_isControllerAction(['users' => ['terms', 'logout', 'login', 'downloadTerms']])) {
+        if (!$user['termsaccepted'] && !empty(Configure::read('MISP.terms_file')) && !$this->_isControllerAction(['users' => ['terms', 'logout', 'login', 'downloadTerms', 'totp_new', 'email_otp']])) {
             //if ($this->_isRest()) throw new MethodNotAllowedException('You have not accepted the terms of use yet, please log in via the web interface and accept them.');
             $this->redirect(array('controller' => 'users', 'action' => 'terms', 'admin' => false));
             return false;
         }
 
         // Check if user must change password
-        if ($user['change_pw'] && !$this->_isControllerAction(['users' => ['terms', 'change_pw', 'logout', 'login']])) {
+        if ($user['change_pw'] && !$this->_isControllerAction(['users' => ['terms', 'change_pw', 'logout', 'login', 'totp_new', 'email_otp']])) {
             //if ($this->_isRest()) throw new MethodNotAllowedException('Your user account is expecting a password change, please log in via the web interface and change it before proceeding.');
             $this->redirect(array('controller' => 'users', 'action' => 'change_pw', 'admin' => false));
             return false;
         }
 
         // Check if user must read news
-        if (!$this->_isControllerAction(['news' => ['index'], 'users' => ['terms', 'change_pw', 'login', 'logout']])) {
+        if (!$this->_isControllerAction(['news' => ['index'], 'users' => ['terms', 'change_pw', 'login', 'logout', 'totp_new', 'email_otp']])) {
             $this->loadModel('News');
             $latestNewsCreated = $this->News->latestNewsTimestamp();
             if ($latestNewsCreated && $user['newsread'] < $latestNewsCreated) {
@@ -862,6 +879,21 @@ class AppController extends Controller
 
     public function afterFilter()
     {
+        // benchmarking
+        if (Configure::read('Plugin.Benchmarking_enable') && isset($this->Benchmark)) {
+            $this->Benchmark->stopBenchmark([
+                'user' => $this->Auth->user('id'),
+                'controller' => $this->request->params['controller'],
+                'action' => $this->request->params['action'],
+                'start_time' => $this->start_time
+            ]);
+
+            //if ($redis && !$redis->exists('misp:auth_fail_throttling:' . $key)) {
+                //$redis->setex('misp:auth_fail_throttling:' . $key, 3600, 1);
+                //return true;
+            //}
+            
+        }
         if ($this->isApiAuthed && $this->_isRest() && !Configure::read('Security.authkey_keep_session')) {
             $this->Session->destroy();
         }
@@ -960,31 +992,6 @@ class AppController extends Controller
         return $this->userRole['perm_site_admin'];
     }
 
-    protected function _getApiAuthUser($key, &$exception)
-    {
-        if (strlen($key) === 40) {
-            // check if the key is valid -> search for users based on key
-            $user = $this->_checkAuthUser($key);
-            if (!$user) {
-                $exception = $this->RestResponse->throwException(
-                    401,
-                    __('This authentication key is not authorized to be used for exports. Contact your administrator.')
-                );
-                return false;
-            }
-        } else {
-            $user = $this->Auth->user();
-            if (!$user) {
-                $exception = $this->RestResponse->throwException(
-                    401,
-                    __('You have to be logged in to do that.')
-                );
-                return false;
-            }
-        }
-        return $user;
-    }
-
     private function __captureParam($data, $param, $value)
     {
         if ($this->modelClass->checkParam($param)) {
@@ -1032,7 +1039,19 @@ class AppController extends Controller
                     $data = array_merge($data, $temp);
                 } else {
                     foreach ($options['paramArray'] as $param) {
-                        if (isset($temp[$param])) {
+                        if (substr($param, -1) == '*') {
+                            $root = substr($param, 0, strlen($param)-1);
+                            foreach ($temp as $existingParamKey => $v) {
+                                $leftover = substr($existingParamKey, strlen($param)-1);
+                                if (
+                                    $root == substr($existingParamKey, 0, strlen($root)) &&
+                                    preg_match('/^[\w_-. ]+$/', $leftover) == 1
+                                ) {
+                                    $data[$existingParamKey] = $temp[$existingParamKey];
+                                    break;
+                                }
+                            }
+                        } else if (isset($temp[$param])) {
                             $data[$param] = $temp[$param];
                         }
                     }
@@ -1100,6 +1119,23 @@ class AppController extends Controller
 
     protected function _checkAuthUser($authkey)
     {
+        if (Configure::read('Security.api_key_quick_lookup')) {
+            $redis = RedisTool::init();
+            if (file_exists(APP . 'Config/hmac_key.php')) {
+                include(APP . 'Config/hmac_key.php');
+                $hashed_authkey = hash_hmac('sha512', $authkey, $hmac_key);
+                if ($redis && $redis->exists('misp:fast_authkey_lookup:' . $hashed_authkey)) {
+                    $user = RedisTool::deserialize($redis->get('misp:fast_authkey_lookup:' . $hashed_authkey));
+                    if ($user) {
+                        return $user;
+                    }
+                }
+            } else {
+                App::uses('RandomTool', 'Tools');
+                $hmac_key = RandomTool::random_str(true, 40);
+                file_put_contents(APP . 'Config/hmac_key.php', sprintf('<?php%s$hmac_key = \'%s\';', PHP_EOL, $hmac_key));
+            }
+        }
         if (Configure::read('Security.advanced_authkeys')) {
             $user = $this->User->AuthKey->getAuthUserByAuthKey($authkey);
         } else {
@@ -1113,6 +1149,13 @@ class AppController extends Controller
             return false;
         }
         $user['logged_by_authkey'] = true;
+        if (Configure::read('Security.api_key_quick_lookup') && !empty($hmac_key) && $redis) {
+            $expiration = Configure::read('Security.api_key_quick_lookup_expiration') ? Configure::read('Security.api_key_quick_lookup_expiration') : 180;
+            if ($redis) {
+                $hashed_authkey = hash_hmac('sha512', $authkey, $hmac_key);
+                $redis->setex('misp:fast_authkey_lookup:' . $hashed_authkey, $expiration, RedisTool::serialize($user));
+            }
+        }
         return $user;
     }
 
@@ -1161,12 +1204,11 @@ class AppController extends Controller
                         null);
                     $this->__preAuthException($authName . ' authentication failed. Contact your MISP support for additional information at: ' . Configure::read('MISP.contact'));
                 }
-                $temp = $this->_checkExternalAuthUser($server[$headerNamespace . $header]);
-                $user['User'] = $temp;
-                if ($user['User']) {
-                    $this->User->updateLoginTimes($user['User']);
+                $user = $this->_checkExternalAuthUser($server[$headerNamespace . $header]);
+                if ($user) {
+                    $this->User->updateLoginTimes($user);
                     //$this->Session->renew();
-                    $this->Session->write(AuthComponent::$sessionKey, $user['User']);
+                    $this->Session->write(AuthComponent::$sessionKey, $user);
                     if (Configure::read('MISP.log_auth')) {
                         $this->Log = ClassRegistry::init('Log');
                         $change = $this->User->UserLoginProfile->_getUserProfile();
@@ -1176,7 +1218,7 @@ class AppController extends Controller
                             $user,
                             'auth',
                             'User',
-                            $user['User']['id'],
+                            $user['id'],
                             'Successful authentication using ' . $authName . ' key',
                             json_encode($change));
                     }
@@ -1326,13 +1368,8 @@ class AppController extends Controller
         if ($filters === false) {
             return $exception;
         }
-        $key = empty($filters['key']) ? $filters['returnFormat'] : $filters['key'];
-        $user = $this->_getApiAuthUser($key, $exception);
-        if ($user === false) {
-            return $exception;
-        }
 
-        session_write_close(); // Rest search can be longer, so close session to allow concurrent requests
+        $user = $this->_closeSession();
 
         if (isset($filters['returnFormat'])) {
             $returnFormat = $filters['returnFormat'];
@@ -1519,10 +1556,20 @@ class AppController extends Controller
      * Close session without writing changes to them and return current user.
      * @return array
      */
-    protected function _closeSession()
+    protected function _closeSession($saveSession = false)
     {
         $user = $this->Auth->user();
-        session_abort();
+
+        // Hack to store user info in static AuthComponent::$_user variable to avoid starting session again by calling
+        // $this->Auth->user()
+        AuthComponent::$sessionKey = null;
+        $this->Auth->login($user);
+
+        if ($saveSession) {
+            @session_write_close();
+        } else {
+            session_abort();
+        }
         return $user;
     }
 

app/Controller/AttributesController.php

@@ -33,38 +33,16 @@ class AttributesController extends AppController
     {
         parent::beforeFilter();
 
-        $this->Auth->allow('restSearch');
-        $this->Auth->allow('returnAttributes');
-        $this->Auth->allow('downloadAttachment');
-        $this->Auth->allow('text');
-        $this->Auth->allow('rpz');
-        $this->Auth->allow('bro');
-
         // permit reuse of CSRF tokens on the search page.
         if ('search' === $this->request->params['action']) {
             $this->Security->csrfCheck = false;
         }
         $this->Security->unlockedActions[] = 'getMassEditForm';
         $this->Security->unlockedActions[] = 'search';
+
         if ($this->request->action === 'add_attachment') {
             $this->Security->unlockedFields = array('values');
-        }
-
-        // convert uuid to id if present in the url and overwrite id field
-        if (isset($this->request->params->query['uuid'])) {
-            $params = array(
-                    'conditions' => array('Attribute.uuid' => $this->request->params->query['uuid']),
-                    'recursive' => 0,
-                    'fields' => 'Attribute.id'
-                    );
-            $result = $this->Attribute->find('first', $params);
-            if (isset($result['Attribute']) && isset($result['Attribute']['id'])) {
-                $id = $result['Attribute']['id'];
-                $this->params->addParams(array('pass' => array($id))); // FIXME find better way to change id variable if uuid is found. params->url and params->here is not modified accordingly now
-            }
-        }
-
-        if ($this->request->action === 'viewPicture') {
+        } elseif ($this->request->action === 'viewPicture') {
             $this->Security->doNotGenerateToken = true;
         }
     }
@@ -793,12 +771,14 @@ class AttributesController extends AppController
                 $result = $this->Attribute->save($this->request->data, array('fieldList' => Attribute::EDITABLE_FIELDS));
                 if ($result) {
                     $this->Attribute->AttributeTag->handleAttributeTags($this->Auth->user(), $this->request->data['Attribute'], $attribute['Event']['id'], $capture=true);
+                    $this->Attribute->Event->captureAnalystData($this->Auth->user(), $this->request->data['Attribute'], 'Attribute', $existingAttribute['Attribute']['uuid']);
                 }
                 $this->Attribute->Object->updateTimestamp($existingAttribute['Attribute']['object_id']);
             } else {
                 $result = $this->Attribute->save($this->request->data, array('fieldList' => Attribute::EDITABLE_FIELDS));
                 if ($result) {
                     $this->Attribute->AttributeTag->handleAttributeTags($this->Auth->user(), $this->request->data['Attribute'], $attribute['Event']['id'], $capture=true);
+                    $this->Attribute->Event->captureAnalystData($this->Auth->user(), $this->request->data['Attribute'], 'Attribute', $existingAttribute['Attribute']['uuid']);
                 }
                 if ($this->request->is('ajax')) {
                     $this->autoRender = false;
@@ -1576,10 +1556,11 @@ class AttributesController extends AppController
 
             // Force index for performance reasons see #3321
             if (isset($filters['value'])) {
-                $this->paginate['forceIndexHint'] = '(value1, value2)';
+                $this->paginate['forceIndexHint'] = 'value1, value2';
             }
 
             $this->paginate['conditions'] = $params['conditions'];
+            $this->paginate['ignoreIndexHint'] = 'deleted';
             $attributes = $this->paginate();
             $this->Attribute->attachTagsToAttributes($attributes, ['includeAllTags' => true]);
 
@@ -3038,4 +3019,35 @@ class AttributesController extends AppController
             ];
         }
     }
+
+    public function viewAnalystData($id, $seed = null)
+    {
+        $this->Attribute->includeAnalystDataRecursive = true;
+        $attribute = $this->Attribute->fetchAttributes(
+            $this->Auth->user(),
+            [
+                'conditions' => $this->__idToConditions($id),
+                'flatten' => true
+            ]
+        );
+        if(empty($attribute)) {
+            throw new NotFoundException(__('Invalid Attribute.'));
+        } else {
+            $attribute[0]['Attribute'] = array_merge_recursive($attribute[0]['Attribute'], $this->Attribute->attachAnalystData($attribute[0]['Attribute']));
+        }
+        if ($this->_isRest()) {
+            $validFields = ['Note', 'Opinion', 'Relationship'];
+            $results = [];
+            foreach ($validFields as $field) {
+                if (!empty($attribute[0]['Attribute'][$field])) {
+                    $results[$field] = $attribute[0]['Attribute'][$field];
+                }
+            }
+            return $this->RestResponse->viewData($results, $this->response->type());
+        }
+        $this->layout = null;
+        $this->set('shortDist', $this->Attribute->shortDist);
+        $this->set('object', $attribute[0]['Attribute']);
+        $this->set('seed', $seed);
+    }
 }

app/Controller/BenchmarksController.php

@@ -0,0 +1,123 @@
+<?php
+App::uses('AppController', 'Controller');
+
+class BenchmarksController extends AppController
+{
+    public $components = array('Session', 'RequestHandler');
+
+    public $paginate = [
+        'limit' => 60,
+        'maxLimit' => 9999,
+
+    ];
+
+    public function beforeFilter()
+    {
+        parent::beforeFilter();
+    }
+
+    public function index()
+    {
+        $this->set('menuData', ['menuList' => 'admin', 'menuItem' => 'index']);
+        $this->loadModel('User');
+        App::uses('BenchmarkTool', 'Tools');
+        $this->Benchmark = new BenchmarkTool($this->User);
+        $passedArgs = $this->passedArgs;
+        $this->paginate['order'] = 'value';
+        $defaults = [
+            'days' => null,
+            'average' => false,
+            'aggregate' => false,
+            'scope' => null,
+            'field' => null,
+            'key' => null,
+            'quickFilter' => null
+        ];
+        $filters = $this->IndexFilter->harvestParameters(array_keys($defaults));
+        foreach ($defaults as $key => $value) {
+            if (!isset($filters[$key])) {
+                $filters[$key] = $defaults[$key];
+            }
+        }
+        $temp = $this->Benchmark->getAllTopLists(
+            $filters['days'] ?? null,
+            $filters['limit'] ?? 100,
+            $filters['average'] ?? null,
+            $filters['aggregate'] ?? null
+        );
+        $settings = $this->Benchmark->getSettings();
+        $units = $this->Benchmark->getUnits();
+        $this->set('settings', $settings);
+        $data = [];
+        $userLookup = [];
+        foreach ($temp as $scope => $t) {
+            if (!empty($filters['scope']) && $filters['scope'] !== 'all' && $scope !== $filters['scope']) {
+                continue;
+            }
+            foreach ($t as $field => $t2) {
+                if (!empty($filters['field']) && $filters['field'] !== 'all' && $field !== $filters['field']) {
+                    continue;
+                }
+                foreach ($t2 as $date => $t3) {
+                    foreach ($t3 as $key => $value) {
+                        if ($scope == 'user') {
+                            if ($key === 'SYSTEM') {
+                                $text = 'SYSTEM';
+                            } else if (isset($userLookup[$key])) {
+                                $text = $userLookup[$key];
+                            } else {
+                                $user = $this->User->find('first', [
+                                    'fields' => ['User.id', 'User.email'],
+                                    'recursive' => -1,
+                                    'conditions' => ['User.id' => $key]
+                                ]);
+                                if (empty($user)) {
+                                    $text = '(' . $key . ') ' . __('Invalid user');
+                                } else {
+                                    $text = '(' . $key . ') ' . $user['User']['email'];
+                                }
+                                $userLookup[$key] = $text;
+                            }
+                        } else {
+                            $text = $key;
+                        }
+                        if (!empty($filters['quickFilter'])) {
+                            $q = strtolower($filters['quickFilter']);
+                            if (
+                                strpos(strtolower($scope), $q) === false &&
+                                strpos(strtolower($field), $q) === false &&
+                                strpos(strtolower($key), $q) === false &&
+                                strpos(strtolower($value), $q) === false &&
+                                strpos(strtolower($date), $q) === false &&
+                                strpos(strtolower($text), $q) === false
+                            ) {
+                                continue;
+                            }
+                        }
+                        if (empty($filters['key']) || $key == $filters['key']) {
+                            $data[] = [
+                                'scope' => $scope,
+                                'field' => $field,
+                                'date' => $date,
+                                'key' => $key,
+                                'text' => $text,
+                                'value' => $value,
+                                'unit' => $units[$field]
+                            ];    
+                        }
+                    }
+                }
+            }
+        }
+        if ($this->_isRest()) {
+            return $this->RestResponse->viewData($data, $this->response->type());
+        }
+        App::uses('CustomPaginationTool', 'Tools');
+        $customPagination = new CustomPaginationTool();
+        $customPagination->truncateAndPaginate($data, $this->params, $this->modelClass, true);
+        $this->set('data', $data);
+        $this->set('passedArgs', json_encode($passedArgs));
+        $this->set('filters', $filters);
+    }
+
+}

app/Controller/CollectionElementsController.php

@@ -0,0 +1,146 @@
+<?php
+
+use PHPUnit\Framework\MockObject\InvalidMethodNameException;
+
+App::uses('AppController', 'Controller');
+
+class CollectionElementsController extends AppController
+{
+
+    public $components = ['Session', 'RequestHandler'];
+
+    public $paginate = [
+        'limit' => 60,
+        'order' => []
+    ];
+
+    public $uses = [
+    ];
+    
+    public function add($collection_id)
+    {   
+        $this->CollectionElement->Collection->current_user = $this->Auth->user();
+        if (!$this->CollectionElement->Collection->mayModify($this->Auth->user('id'), intval($collection_id))) {
+            throw new MethodNotAllowedException(__('Invalid Collection or insuficient privileges'));
+        }
+        $this->CRUD->add([
+            'beforeSave' => function (array $collectionElement) use ($collection_id) {
+                $collectionElement['CollectionElement']['collection_id'] = intval($collection_id);
+                return $collectionElement;
+            }
+        ]);
+        if ($this->restResponsePayload) {
+            return $this->restResponsePayload;
+        }
+        $dropdownData = [
+            'types' => array_combine($this->CollectionElement->valid_types, $this->CollectionElement->valid_types)
+        ];
+        $this->set(compact('dropdownData'));
+        $this->set('menuData', array('menuList' => 'collections', 'menuItem' => 'add_element'));
+    }
+
+    public function delete($element_id)
+    {
+        $collectionElement = $this->CollectionElement->find('first', [
+            'recursive' => -1,
+            'conditions' => [
+                'CollectionElement.id' => $element_id
+            ]
+        ]);
+        $collection_id = $collectionElement['CollectionElement']['collection_id'];
+        if (!$this->CollectionElement->Collection->mayModify($this->Auth->user('id'), $collection_id)) {
+            throw new MethodNotAllowedException(__('Invalid Collection or insuficient privileges'));
+        }
+        $this->CRUD->delete($element_id);
+        if ($this->restResponsePayload) {
+            return $this->restResponsePayload;
+        }
+    }
+
+    public function index($collection_id)
+    {
+        $this->set('menuData', array('menuList' => 'collections', 'menuItem' => 'index'));
+        if (!$this->CollectionElement->Collection->mayView($this->Auth->user('id'), intval($collection_id))) {
+            throw new NotFoundException(__('Invalid collection or no access.'));
+        }
+        $params = [
+            'filters' => ['uuid', 'type', 'name'],
+            'quickFilters' => ['name'],
+            'conditions' => ['collection_id' => $collection_id]
+        ];
+        $this->loadModel('Event');
+        $this->set('distributionLevels', $this->Event->distributionLevels);
+        $this->CRUD->index($params);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }
+    }
+
+    public function addElementToCollection($element_type, $element_uuid)
+    {
+        if ($this->request->is('get')) {
+            $validCollections = $this->CollectionElement->Collection->find('list', [
+                'recursive' => -1,
+                'fields' => ['Collection.id', 'Collection.name'],
+                'conditions' => ['Collection.orgc_id' => $this->Auth->user('org_id')]
+            ]);
+            if (empty($validCollections)) {
+                if ($this->request->is('ajax')) {
+                    return $this->redirect(['controller' => 'collections', 'action' => 'add']);
+                }
+                throw new NotFoundException(__('You don\'t have any collections yet. Make sure you create one first before you can start adding elements.'));
+            }
+            $dropdownData = [
+                'collections' => $validCollections
+            ];
+            $this->set(compact('dropdownData'));
+        } else if ($this->request->is('post')) {
+            if (!isset($this->request->data['CollectionElement'])) {
+                $this->request->data = ['CollectionElement' => $this->request->data];
+            }
+            if (!isset($this->request->data['CollectionElement']['collection_id'])) {
+                throw new NotFoundException(__('No collection_id specified.'));
+            }
+            $collection_id = intval($this->request->data['CollectionElement']['collection_id']);
+            if (!$this->CollectionElement->Collection->mayModify($this->Auth->user('id'), $collection_id)) {
+                throw new NotFoundException(__('Invalid collection or not authorized.'));
+            }
+            $description = empty($this->request->data['CollectionElement']['description']) ? '' : $this->request->data['CollectionElement']['description'];
+            $dataToSave = [
+                'CollectionElement' => [
+                    'element_uuid' => $element_uuid,
+                    'element_type' => $element_type,
+                    'description' => $description,
+                    'collection_id' => $collection_id
+                ]
+            ];
+            $this->CollectionElement->create();
+            $error = '';
+            try {
+                $result = $this->CollectionElement->save($dataToSave);
+            } catch (PDOException $e) {
+                if ($e->errorInfo[0] == 23000) {
+                    $error = __(' Element already in Collection.');
+                }
+            }
+            
+            if ($result) {
+                $message = __('Element added to the Collection.');
+                if ($this->IndexFilter->isRest()) {
+                    return $this->RestResponse->saveSuccessResponse('CollectionElements', 'addElementToCollection', false, $this->response->type(), $message);
+                } else {
+                    $this->Flash->success($message);
+                    $this->redirect(Router::url($this->referer(), true));
+                }
+            } else {
+                $message = __('Element could not be added to the Collection.%s', $error);
+                if ($this->IndexFilter->isRest()) {
+                    return $this->RestResponse->saveFailResponse('CollectionElements', 'addElementToCollection', false, $message, $this->response->type());
+                } else {
+                    $this->Flash->error($message);
+                    $this->redirect(Router::url($this->referer(), true));
+                }
+            }
+        }
+    }
+}

app/Controller/CollectionsController.php

@@ -0,0 +1,171 @@
+<?php
+App::uses('AppController', 'Controller');
+
+class CollectionsController extends AppController
+{
+
+    public $components = ['Session', 'RequestHandler'];
+
+    public $paginate = [
+        'limit' => 60,
+        'order' => []
+    ];
+
+    public $uses = [
+    ];
+
+    private $valid_types = [
+        'campaign',
+        'intrusion_set',
+        'named_threat',
+        'other',
+        'research'
+    ];
+    
+    public function add()
+    {   
+        $this->Collection->current_user = $this->Auth->user();
+        $params = [];
+        if ($this->request->is('post')) {
+            $data = $this->request->data;
+            $params = [
+                'afterSave' => function (array $collection) use ($data) {
+                    $this->Collection->CollectionElement->captureElements($collection);
+                    return $collection;
+                }
+            ];
+        }
+        $this->CRUD->add($params);
+        if ($this->restResponsePayload) {
+            return $this->restResponsePayload;
+        }
+        $this->set('menuData', array('menuList' => 'collections', 'menuItem' => 'add'));
+        $this->loadModel('Event');
+        $dropdownData = [
+            'types' => array_combine($this->valid_types, $this->valid_types),
+            'distributionLevels' => $this->Event->distributionLevels,
+            'sgs' => $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1)  
+        ];
+        $this->set('initialDistribution', Configure::read('MISP.default_event_distribution'));
+        $this->set(compact('dropdownData'));
+        $this->render('add');
+    }
+
+    public function edit($id)
+    {
+        $this->Collection->current_user = $this->Auth->user();
+        if (!$this->Collection->mayModify($this->Auth->user('id'), $id)) {
+            throw new MethodNotAllowedException(__('Invalid Collection or insuficient privileges'));
+        }
+        $params = [];
+        if ($this->request->is('post') || $this->request->is('put')) {
+            $oldCollection = $this->Collection->find('first', [
+                'recursive' => -1,
+                'conditions' => ['Collection.id' => intval($id)]
+            ]);
+            if (empty($oldCollection)) {
+                throw new NotFoundException(__('Invalid collection.'));
+            }
+            if (empty($this->request->data['Collection'])) {
+                $this->request->data = ['Collection' => $this->request->data];
+            }
+            $data = $this->request->data;
+            if (
+                isset($data['Collection']['modified']) &&
+                $data['Collection']['modified'] <= $oldCollection['Collection']['modified']
+            ) {
+                throw new ForbiddenException(__('Collection received older or same as local version.'));
+            }
+            $params = [
+                'afterSave' => function (array &$collection) use ($data) {
+                    $collection = $this->Collection->CollectionElement->captureElements($collection);
+                    return $collection;
+                }
+            ];
+        }
+        $this->set('id', $id);
+        $this->CRUD->edit($id, $params);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }
+        $this->set('menuData', array('menuList' => 'collections', 'menuItem' => 'edit'));
+        $this->loadModel('Event');
+        $dropdownData = [
+            'types' => $this->valid_types,
+            'distributionLevels' => $this->Event->distributionLevels,
+            'sgs' => $this->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1)  
+        ];
+        $this->set(compact('dropdownData'));
+        $this->render('add');
+    }
+
+    public function delete($id)
+    {
+        if (!$this->Collection->mayModify($this->Auth->user('id'), $id)) {
+            throw new MethodNotAllowedException(__('Invalid Collection or insuficient privileges'));
+        }
+        $this->CRUD->delete($id);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }   
+    }
+
+    public function view($id)
+    {
+        $this->set('mayModify', $this->Collection->mayModify($this->Auth->user('id'), $id));
+        if (!$this->Collection->mayView($this->Auth->user('id'), $id)) {
+            throw new MethodNotAllowedException(__('Invalid Collection or insuficient privileges'));
+        }
+        $this->set('menuData', array('menuList' => 'collections', 'menuItem' => 'view'));
+        $params = [
+            'contain' => [
+                'Orgc',
+                'Org',
+                'User',
+                'CollectionElement'
+            ],
+            'afterFind' => function (array $collection){
+                return $this->Collection->rearrangeCollection($collection);
+            }
+        ];
+        $this->CRUD->view($id, $params);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }
+        $this->set('id', $id);
+        $this->loadModel('Event');
+        $this->set('distributionLevels', $this->Event->distributionLevels);
+        $this->render('view');
+    }
+
+    public function index($filter = null)
+    {
+        $this->set('menuData', array('menuList' => 'collections', 'menuItem' => 'index'));
+        $params = [
+            'filters' => ['Collection.uuid', 'Collection.type', 'Collection.name'],
+            'quickFilters' => ['Collection.name'],
+            'contain' => ['Orgc'],
+            'afterFind' => function($collections) {
+                foreach ($collections as $k => $collection) {
+                    $collections[$k]['Collection']['element_count'] = $this->Collection->CollectionElement->find('count', [
+                        'recursive' => -1,
+                        'conditions' => ['CollectionElement.collection_id' => $collection['Collection']['id']]
+                    ]);
+                }
+                return $collections;
+            }
+        ];
+        if ($filter === 'my_collections') {
+            $params['conditions']['Collection.user_id'] = $this->Auth->user('id');
+        }
+        if ($filter === 'org_collections') {
+            $params['conditions']['Collection.orgc_id'] = $this->Auth->user('org_id');
+        }
+        $this->loadModel('Event');
+        $this->set('distributionLevels', $this->Event->distributionLevels);
+        $this->CRUD->index($params);
+        if ($this->IndexFilter->isRest()) {
+            return $this->restResponsePayload;
+        }
+    }
+}

app/Controller/Component/CRUDComponent.php

@@ -25,6 +25,7 @@ class CRUDComponent extends Component
             }
             $options['filters'][] = 'quickFilter';
         }
+        $this->Controller->{$this->Controller->modelClass}->includeAnalystData = true;
         $params = $this->Controller->IndexFilter->harvestParameters(empty($options['filters']) ? [] : $options['filters']);
         $query = [];
         $query = $this->setFilters($params, $query);
@@ -39,6 +40,7 @@ class CRUDComponent extends Component
             if (!empty($this->Controller->paginate['fields'])) {
                 $query['fields'] = $this->Controller->paginate['fields'];
             }
+            $query['includeAnalystData'] = true;
             $data = $this->Controller->{$this->Controller->modelClass}->find('all', $query);
             if (isset($options['afterFind'])) {
                 if (is_callable($options['afterFind'])) {
@@ -49,6 +51,7 @@ class CRUDComponent extends Component
             }
             $this->Controller->restResponsePayload = $this->Controller->RestResponse->viewData($data, 'json');
         } else {
+            $query['includeAnalystData'] = true;
             $this->Controller->paginate = $query;
             $data = $this->Controller->paginate();
             if (isset($options['afterFind'])) {
@@ -93,7 +96,7 @@ class CRUDComponent extends Component
             $savedData = $model->save($data);
             if ($savedData) {
                 if (isset($params['afterSave'])) {
-                    $params['afterSave']($data);
+                    $params['afterSave']($savedData);
                 }
                 $data = $model->find('first', [
                     'recursive' => -1,
@@ -200,7 +203,7 @@ class CRUDComponent extends Component
             if (isset($params['beforeSave'])) {
                 $data = $params['beforeSave']($data);
             }
-            if ($model->save($data)) {
+            if ($data = $model->save($data)) {
                 if (isset($params['afterSave'])) {
                     $params['afterSave']($data);
                 }
@@ -231,6 +234,8 @@ class CRUDComponent extends Component
         if (empty($id)) {
             throw new NotFoundException(__('Invalid %s.', $modelName));
         }
+        $this->Controller->{$modelName}->includeAnalystData = true;
+        $this->Controller->{$modelName}->includeAnalystDataRecursive = true;
         $query = [
             'recursive' => -1,
             'conditions' => [$modelName . '.id' => $id],
@@ -284,6 +289,9 @@ class CRUDComponent extends Component
                 }
             }
         }
+        if (isset($params['afterFind'])) {
+            $data = $params['afterFind']($data);
+        }
         if (isset($params['beforeDelete'])) {
             $data = $params['beforeDelete']($data);
             if (empty($data)) {
@@ -297,6 +305,9 @@ class CRUDComponent extends Component
                 $result = $this->Controller->{$modelName}->delete($id);
             }
             if ($result) {
+                if (isset($params['afterDelete']) && is_callable($params['afterDelete'])) {
+                    $params['afterDelete']($data);
+                }
                 $message = __('%s deleted.', $modelName);
                 if ($this->Controller->IndexFilter->isRest()) {
                     $this->Controller->restResponsePayload = $this->Controller->RestResponse->saveSuccessResponse($modelName, 'delete', $id, 'json', $message);
@@ -336,7 +347,9 @@ class CRUDComponent extends Component
                 if ($filter === 'quickFilter') {
                     continue;
                 }
-                if (strlen(trim($filterValue, '%')) === strlen($filterValue)) {
+                if (is_array($filterValue)) {
+                    $query['conditions']['AND'][] = [$filter => $filterValue];
+                } else if (strlen(trim($filterValue, '%')) === strlen($filterValue)) {
                     $query['conditions']['AND'][] = [$filter => $filterValue];
                 } else {
                     $query['conditions']['AND'][] = [$filter . ' LIKE' => $filterValue];

app/Controller/Component/CompressedRequestHandlerComponent.php

@@ -37,14 +37,17 @@ class CompressedRequestHandlerComponent extends Component
     private function decodeGzipEncodedContent(Controller $controller)
     {
         if (function_exists('gzdecode')) {
-            $decoded = gzdecode($controller->request->input());
+            $input = $controller->request->input();
+            if (empty($input)) {
+                throw new BadRequestException('Request data should be gzip encoded, but request is empty.');
+            }
+            $decoded = gzdecode($input);
             if ($decoded === false) {
                 throw new BadRequestException('Invalid compressed data.');
             }
             return $decoded;
-        } else {
-            throw new BadRequestException("This server doesn't support GZIP compressed requests.");
         }
+        throw new BadRequestException("This server doesn't support GZIP compressed requests.");
     }
 
     /**

app/Controller/Component/RestResponseComponent.php

@@ -213,6 +213,7 @@ class RestResponseComponent extends Component
                     'perm_tag_editor',
                     'default_role',
                     'perm_sighting',
+                    'perm_analyst_data',
                     'permission'
                 )
             ),
@@ -234,6 +235,7 @@ class RestResponseComponent extends Component
                     'perm_tag_editor',
                     'default_role',
                     'perm_sighting',
+                    'perm_analyst_data',
                     'permission'
                 )
             )
@@ -669,9 +671,10 @@ class RestResponseComponent extends Component
         }
 
         if ($response instanceof TmpFileTool) {
-            if (isset($_SERVER['HTTP_IF_NONE_MATCH'])) {
+            $requestEtag = $this->requestEtag();
+            if ($requestEtag !== null) {
                 $etag = '"' . $response->hash('sha1') . '"';
-                if ($_SERVER['HTTP_IF_NONE_MATCH'] === $etag) {
+                if ($requestEtag === $etag) {
                     return new CakeResponse(['status' => 304]);
                 }
                 $headers['ETag'] = $etag;
@@ -687,9 +690,10 @@ class RestResponseComponent extends Component
             }
         } else {
             // Check if resource was changed when `If-None-Match` header is send and return 304 Not Modified
-            if (isset($_SERVER['HTTP_IF_NONE_MATCH'])) {
+            $requestEtag = $this->requestEtag();
+            if ($requestEtag !== null) {
                 $etag = '"' . sha1($response) . '"';
-                if ($_SERVER['HTTP_IF_NONE_MATCH'] === $etag) {
+                if ($requestEtag === $etag) {
                     return new CakeResponse(['status' => 304]);
                 }
                 // Generate etag just when HTTP_IF_NONE_MATCH is set
@@ -722,6 +726,25 @@ class RestResponseComponent extends Component
         return $cakeResponse;
     }
 
+    /**
+     * Return etag from If-None-Match HTTP request header without compression marks added by Apache
+     * @return string|null
+     */
+    private function requestEtag()
+    {
+        if (isset($_SERVER['HTTP_IF_NONE_MATCH'])) {
+            // Remove compression marks that adds Apache for compressed content
+            $requestEtag = $_SERVER['HTTP_IF_NONE_MATCH'];
+            $etagWithoutQuotes = trim($requestEtag, '"');
+            $dashPos = strrpos($etagWithoutQuotes, '-');
+            if ($dashPos && in_array(substr($etagWithoutQuotes, $dashPos + 1), ['br', 'gzip'], true)) {
+                return '"' . substr($etagWithoutQuotes, 0, $dashPos) . '"';
+            }
+            return $requestEtag;
+        }
+        return null;
+    }
+
     /**
      * @param string $response
      * @return string Signature as base64 encoded string
@@ -1052,7 +1075,7 @@ class RestResponseComponent extends Component
                 'input' => 'radio',
                 'type' => 'integer',
                 'values' => array(1 => 'True', 0 => 'False' ),
-                'help' => __('Include deleted elements')
+                'help' => __('Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned')
             ),
             'delta_merge' => array(
                 'input' => 'radio',
@@ -1557,6 +1580,11 @@ class RestResponseComponent extends Component
                 'type' => 'integer',
                 'values' => array(1 => 'True', 0 => 'False')
             ),
+            'perm_analyst_data' => array(
+                'input' => 'radio',
+                'type' => 'integer',
+                'values' => array(1 => 'True', 0 => 'False')
+            ),
             'permission' => array(
                 'input' => 'select',
                 'type' => 'string',

app/Controller/Component/RestSearchComponent.php

@@ -124,6 +124,7 @@ class RestSearchComponent extends Component
             'extended',
             'extensionList',
             'excludeGalaxy',
+            'includeAnalystData',
             'includeRelatedTags',
             'includeDecayScore',
             'includeScoresOnEvent',
@@ -143,7 +144,11 @@ class RestSearchComponent extends Component
             'retry',
             'expiry',
             'minimum_ttl',
-            'ttl'
+            'ttl',
+            'org.sector',
+            'org.local',
+            'org.nationality',
+            'galaxy.*',
         ],
         'Object' => [
             'returnFormat',
@@ -180,7 +185,10 @@ class RestSearchComponent extends Component
             'attackGalaxy',
             'object_relation',
             'metadata',
-            'includeAllTags'
+            'includeAllTags',
+            'object_name',
+            'object_template_uuid',
+            'object_template_version'
         ],
         'Sighting' => [
             'context',

app/Controller/EventReportsController.php

@@ -61,6 +61,8 @@ class EventReportsController extends AppController
 
     public function view($reportId, $ajax=false)
     {
+        $this->EventReport->includeAnalystData = true;
+        $this->EventReport->includeAnalystDataRecursive = true;
         $report = $this->EventReport->simpleFetchById($this->Auth->user(), $reportId);
         if ($this->_isRest()) {
             return $this->RestResponse->viewData($report, $this->response->type());
@@ -175,6 +177,7 @@ class EventReportsController extends AppController
         $filters = $this->IndexFilter->harvestParameters(['event_id', 'value', 'context', 'index_for_event', 'extended_event']);
         $filters['embedded_view']  = $this->request->is('ajax');
         $compiledConditions = $this->__generateIndexConditions($filters);
+        $this->EventReport->includeAnalystData = true;
         if ($this->_isRest()) {
             $reports = $this->EventReport->find('all', [
                 'recursive' => -1,
@@ -210,10 +213,13 @@ class EventReportsController extends AppController
 
     public function extractAllFromReport($reportId)
     {
-        if (!$this->request->is('ajax')) {
+        if (!$this->request->is('ajax') && !$this->_isRest()) {
             throw new MethodNotAllowedException(__('This function can only be reached via AJAX.'));
         }
         if ($this->request->is('post')) {
+            if (!isset($this->data['EventReport'])) {
+                $this->data = ['EventReport' => $this->data];
+            }
             $report = $this->EventReport->fetchIfAuthorized($this->Auth->user(), $reportId, 'edit', $throwErrors=true, $full=false);
             $results = $this->EventReport->getComplexTypeToolResultWithReplacements($this->Auth->user(), $report);
             $report['EventReport']['content'] = $results['replacementResult']['contentWithReplacements'];
@@ -296,13 +302,16 @@ class EventReportsController extends AppController
 
     public function importReportFromUrl($event_id)
     {
-        if (!$this->request->is('ajax')) {
-            throw new MethodNotAllowedException(__('This function can only be reached via AJAX.'));
+        if (!$this->request->is('ajax') && !$this->_isRest()) {
+            throw new MethodNotAllowedException(__('This function can only be reached via AJAX and via the API.'));
         }
         $fetcherModule = $this->EventReport->isFetchURLModuleEnabled();
         if ($this->request->is('post')) {
+            if (empty($this->data['EventReport'])) {
+                $this->data = ['EventReport' => $this->data];
+            }
             if (empty($this->data['EventReport']['url'])) {
-                throw new MethodNotAllowedException(__('An URL must be provided'));
+                throw new MethodNotAllowedException(__('A URL must be provided'));
             }
             $url = $this->data['EventReport']['url'];
             $format = 'html';
@@ -313,7 +322,6 @@ class EventReportsController extends AppController
                     $format = $parsed_format;
                 }
             }
-            
             $content = $this->EventReport->downloadMarkdownFromURL($event_id, $url, $format);
 
             $errors = [];
@@ -515,6 +523,7 @@ class EventReportsController extends AppController
     {
         $distributionLevels = $this->EventReport->Event->Attribute->distributionLevels;
         $this->set('distributionLevels', $distributionLevels);
+        $this->set('shortDist', $this->EventReport->Event->Attribute->shortDist);
         $this->set('initialDistribution', $this->EventReport->Event->Attribute->defaultDistribution());
     }
 
@@ -559,6 +568,12 @@ class EventReportsController extends AppController
                 $savedReport['EventReport'][$field] = $newReport['EventReport'][$field];
             }
         }
+        $this->loadModel('AnalystData');
+        foreach ($this->AnalystData::ANALYST_DATA_TYPES as $type) {
+            if (!empty($newReport['EventReport'][$type])) {
+                $savedReport['EventReport'][$type] = $newReport['EventReport'][$type];
+            }
+        }
         return $savedReport;
     }
 }

app/Controller/EventsController.php

@@ -14,16 +14,16 @@ class EventsController extends AppController
     );
 
     public $paginate = array(
-            'limit' => 60,
-            'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page.
-            'order' => array(
-                    'Event.timestamp' => 'DESC'
-            ),
-            'contain' => array(
-                    'Org' => array('fields' => array('id', 'name', 'uuid')),
-                    'Orgc' => array('fields' => array('id', 'name', 'uuid')),
-                    'SharingGroup' => array('fields' => array('id', 'name', 'uuid'))
-            )
+        'limit' => 60,
+        'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page.
+        'order' => array(
+            'Event.timestamp' => 'DESC'
+        ),
+        'contain' => array(
+            'Org' => array('fields' => array('id', 'name', 'uuid')),
+            'Orgc' => array('fields' => array('id', 'name', 'uuid')),
+            'SharingGroup' => array('fields' => array('id', 'name', 'uuid'))
+        )
     );
 
     // private
@@ -61,46 +61,14 @@ class EventsController extends AppController
         'publish_timestamp'
     ];
 
-    public $paginationFunctions = array('index', 'proposalEventIndex');
-
     public function beforeFilter()
     {
         parent::beforeFilter();
 
-        // what pages are allowed for non-logged-in users
-        $this->Auth->allow('xml');
-        $this->Auth->allow('csv');
-        $this->Auth->allow('nids');
-        $this->Auth->allow('hids_md5');
-        $this->Auth->allow('hids_sha1');
-        $this->Auth->allow('text');
-        $this->Auth->allow('restSearch');
-        $this->Auth->allow('stix');
-        $this->Auth->allow('stix2');
-
         $this->Security->unlockedActions[] = 'viewEventAttributes';
 
-        // TODO Audit, activate logable in a Controller
-        if (count($this->uses) && $this->{$this->modelClass}->Behaviors->attached('SysLogLogable')) {
-            $this->{$this->modelClass}->setUserData($this->activeUser);
-        }
-
-        // convert uuid to id if present in the url, and overwrite id field
-        if (isset($this->request->params->query['uuid'])) {
-            $params = array(
-                    'conditions' => array('Event.uuid' => $this->params->query['uuid']),
-                    'recursive' => 0,
-                    'fields' => 'Event.id'
-            );
-            $result = $this->Event->find('first', $params);
-            if (isset($result['Event']) && isset($result['Event']['id'])) {
-                $id = $result['Event']['id'];
-                $this->params->addParams(array('pass' => array($id))); // FIXME find better way to change id variable if uuid is found. params->url and params->here is not modified accordingly now
-            }
-        }
-
         // if not admin or own org, check private as well..
-        if (!$this->_isSiteAdmin() && in_array($this->request->action, $this->paginationFunctions, true)) {
+        if (!$this->_isSiteAdmin() && in_array($this->request->action, ['index', 'proposalEventIndex'], true)) {
             $conditions = $this->Event->createEventConditions($this->Auth->user());
             if ($this->userRole['perm_sync'] && $this->Auth->user('Server')['push_rules']) {
                 $conditions['AND'][] = $this->Event->filterRulesToConditions($this->Auth->user('Server')['push_rules']);
@@ -497,6 +465,11 @@ class EventsController extends AppController
                         continue 2;
                     }
                     $pieces = is_array($v) ? $v : explode('|', $v);
+                    $isANDed = false;
+                    if (count($pieces) == 1 && strpos($pieces[0], '&') !== false) {
+                        $pieces = explode('&', $pieces[0]);
+                        $isANDed = count($pieces) > 1;
+                    }
                     $filterString = "";
                     $expectOR = false;
                     $tagRules = [];
@@ -563,10 +536,19 @@ class EventsController extends AppController
                     }
 
                     if (!empty($tagRules['include'])) {
-                        $include = $this->Event->EventTag->find('column', array(
-                            'conditions' => array('EventTag.tag_id' => $tagRules['include']),
-                            'fields' => ['EventTag.event_id'],
-                        ));
+                        if ($isANDed) {
+                            $include = $this->Event->EventTag->find('column', array(
+                                'conditions' => ['EventTag.tag_id' => $tagRules['include']],
+                                'fields' => ['EventTag.event_id'],
+                                'group' => ['EventTag.event_id'],
+                                'having' => ['COUNT(*) =' => count($tagRules['include'])],
+                            ));
+                        } else {
+                            $include = $this->Event->EventTag->find('column', array(
+                                'conditions' => array('EventTag.tag_id' => $tagRules['include']),
+                                'fields' => ['EventTag.event_id'],
+                            ));
+                        }
                         if (!empty($include)) {
                             $this->paginate['conditions']['AND'][] = 'Event.id IN (' . implode(",", $include) . ')';
                         } else {
@@ -755,7 +737,8 @@ class EventsController extends AppController
         if ($nothing) {
             $this->paginate['conditions']['AND'][] = ['Event.id' => -1]; // do not fetch any event
         }
-
+        $this->Event->includeAnalystData = isset($passedArgs['includeAnalystData']) ? $passedArgs['includeAnalystData'] : false;
+        $this->paginate['includeAnalystData'] = isset($passedArgs['includeAnalystData']) ? $passedArgs['includeAnalystData'] : false;
         $events = $this->paginate();
 
         if (count($events) === 1 && isset($this->passedArgs['searchall'])) {
@@ -811,6 +794,7 @@ class EventsController extends AppController
             $rules = [
                 'contain' => ['EventTag'],
                 'fields' => array_keys($fieldNames),
+                'includeAnalystData' => isset($passedArgs['includeAnalystData']) ? $passedArgs['includeAnalystData'] : false,
             ];
         }
         if (isset($passedArgs['sort']) && isset($fieldNames[$passedArgs['sort']])) {
@@ -1227,6 +1211,9 @@ class EventsController extends AppController
             }
         }
 
+        $this->Event->Attribute->includeAnalystData = true;
+        $this->Event->Attribute->includeAnalystDataRecursive = true;
+
         if (isset($filters['focus'])) {
             $this->set('focus', $filters['focus']);
         }
@@ -1411,7 +1398,8 @@ class EventsController extends AppController
         $exception = false;
         $warningTagConflicts = array();
         $filters = $this->_harvestParameters($filterData, $exception);
-
+        $analystData = $this->Event->attachAnalystData($event['Event']);
+        $event['Event'] = array_merge($event['Event'], $analystData); 
         $emptyEvent = (empty($event['Object']) && empty($event['Attribute']));
         $this->set('emptyEvent', $emptyEvent);
 
@@ -1485,7 +1473,6 @@ class EventsController extends AppController
                 $containsProposals = true;
             }
         }
-
         foreach ($event['Object'] as $k => $object) {
             $modDate = date("Y-m-d", $object['timestamp']);
             $modificationMap[$modDate] = !isset($modificationMap[$modDate])? 1 : $modificationMap[$modDate] + 1;
@@ -1517,7 +1504,6 @@ class EventsController extends AppController
                 }
             }
         }
-
         if ($containsProposals && $this->__canPublishEvent($event, $user)) {
             $mess = $this->Session->read('Message');
             if (empty($mess)) {
@@ -1691,8 +1677,8 @@ class EventsController extends AppController
         }
 
         $namedParams = $this->request->params['named'];
-
         if ($this->_isRest()) {
+            $conditions['includeAnalystData'] = true;
             $conditions['includeAttachments'] = isset($namedParams['includeAttachments']) ? $namedParams['includeAttachments'] : true;
         } else {
             $conditions['includeAllTags'] = true;
@@ -1786,7 +1772,6 @@ class EventsController extends AppController
         } else {
             $user = $this->Auth->user();
         }
-
         $results = $this->Event->fetchEvent($user, $conditions);
         if (empty($results)) {
             throw new NotFoundException(__('Invalid event'));
@@ -2365,7 +2350,13 @@ class EventsController extends AppController
                     }
 
                     $isXml = $ext === 'xml';
-                    $data = FileAccessTool::readFromFile($file['tmp_name'], $file['size']);
+                    $matches = null;
+                    $tmp_name = $file['tmp_name'];
+                    if (preg_match_all('/[\w\/\-\.]*/', $tmp_name, $matches) && file_exists($file['tmp_name'])) {
+                        $data = FileAccessTool::readFromFile($matches[0][0], $file['size']);
+                    } else {
+                        throw new NotFoundException(__('Invalid file.'));    
+                    }
                 } else {
                     throw new MethodNotAllowedException(__('No file uploaded.'));
                 }
@@ -2374,7 +2365,6 @@ class EventsController extends AppController
                     && (isset($this->request->data['Event']['takeownership']) && $this->request->data['Event']['takeownership'] == 1);
 
                 $publish = $this->request->data['Event']['publish'] ?? false;
-
                 try {
                     $results = $this->Event->addMISPExportFile($this->Auth->user(), $data, $isXml, $takeOwnership, $publish);
                 } catch (Exception $e) {
@@ -2404,24 +2394,41 @@ class EventsController extends AppController
                 }
                 if (isset($this->params['named']['distribution'])) {
                     $distribution = intval($this->params['named']['distribution']);
-                    if (array_key_exists($distribution, $distributionLevels)) {
-                        $initialDistribution = $distribution;
-                    } else {
-                        throw new MethodNotAllowedException(__('Wrong distribution level'));
+                    if (!array_key_exists($distribution, $distributionLevels)) {
+                        throw new BadRequestException(__('Wrong distribution level'));
                     }
+                } else {
+                    $distribution = $initialDistribution;
                 }
                 $sharingGroupId = null;
-                if ($initialDistribution == 4) {
+                if ($distribution == 4) {
                     if (!isset($this->params['named']['sharing_group_id'])) {
-                        throw new MethodNotAllowedException(__('The sharing group id is needed when the distribution is set to 4 ("Sharing group").'));
+                        throw new BadRequestException(__('The sharing group id is needed when the distribution is set to 4 ("Sharing group").'));
                     }
                     $sharingGroupId = intval($this->params['named']['sharing_group_id']);
                     if (!array_key_exists($sharingGroupId, $sgs)) {
-                        throw new MethodNotAllowedException(__('Please select a valid sharing group id.'));
+                        throw new BadRequestException(__('Please select a valid sharing group id.'));
                     }
                 }
+                $clusterDistribution = $initialDistribution;
+                $clusterSharingGroupId = null;
                 if (isset($this->params['named']['galaxies_as_tags'])) {
                     $galaxies_as_tags = $this->params['named']['galaxies_as_tags'];
+                    if (isset($this->params['name']['cluster_distribution'])) {
+                        $clusterDistribution = intval($this->params['named']['cluster_distribution']);
+                        if (!array_key_exists($clusterDistribution, $distributionLevels)) {
+                            throw new BadRequestException(__('Wrong cluster distribution level'));
+                        }
+                        if ($clusterDistribution == 4) {
+                            if (!isset($this->params['named']['cluster_sharing_group_id'])) {
+                                throw new BadRequestException(__('The cluster sharing group id is needed when the cluster distribution is set to 4 ("Sharing group").'));
+                            }
+                            $clusterSharingGroupId = intval($this->params['named']['cluster_sharing_group_id']);
+                            if (!array_key_exists($clusterSharingGroupId, $sgs)) {
+                                throw new BadRequestException(__('Please select a valid cluster sharing group id.'));
+                            }
+                        }
+                    }
                 }
                 if (isset($this->params['named']['debugging'])) {
                     $debug = $this->params['named']['debugging'];
@@ -2433,9 +2440,11 @@ class EventsController extends AppController
                     $stix_version,
                     'uploaded_stix_file.' . ($stix_version == '1' ? 'xml' : 'json'),
                     $publish,
-                    $initialDistribution,
+                    $distribution,
                     $sharingGroupId,
                     $galaxies_as_tags,
+                    $clusterDistribution,
+                    $clusterSharingGroupId,
                     $debug
                 );
                 if (is_numeric($result)) {
@@ -2448,8 +2457,8 @@ class EventsController extends AppController
                 } else {
                     return $this->RestResponse->saveFailResponse('Events', 'upload_stix', false, $result, $this->response->type());
                 }
-            } else {
-                $original_file = !empty($this->data['Event']['original_file']) ? $this->data['Event']['stix']['name'] : '';
+            } else { // not REST request
+                $originalFile = !empty($this->data['Event']['original_file']) ? $this->data['Event']['stix']['name'] : '';
                 if (isset($this->data['Event']['stix']) && $this->data['Event']['stix']['size'] > 0 && is_uploaded_file($this->data['Event']['stix']['tmp_name'])) {
                     $filePath = FileAccessTool::createTempFile();
                     if (!move_uploaded_file($this->data['Event']['stix']['tmp_name'], $filePath)) {
@@ -2462,11 +2471,13 @@ class EventsController extends AppController
                         $this->Auth->user(),
                         $filePath,
                         $stix_version,
-                        $original_file,
+                        $originalFile,
                         $this->data['Event']['publish'],
                         $this->data['Event']['distribution'],
                         $this->data['Event']['sharing_group_id'] ?? null,
-                        $this->data['Event']['galaxies_handling'],
+                        $this->data['Event']['galaxies_handling'] ?? false,
+                        $this->data['Event']['cluster_distribution'] ?? 0,
+                        $this->data['Event']['cluster_sharing_group_id'] ?? null,
                         $debug
                     );
                     if (is_numeric($result)) {
@@ -2692,7 +2703,7 @@ class EventsController extends AppController
                 $this->request->data = $this->request->data['Event'];
             }
             $eventToSave = $event;
-            $capturedObjects = ['Attribute', 'Object', 'Tag', 'Galaxy', 'EventReport'];
+            $capturedObjects = ['Attribute', 'Object', 'Tag', 'Galaxy', 'EventReport', 'Note', 'Opinion', 'Relationship',];
             foreach ($capturedObjects as $objectType) {
                 if (!empty($this->request->data[$objectType])) {
                     if (!empty($regenerateUUIDs)) {
@@ -3082,9 +3093,9 @@ class EventsController extends AppController
                         $errors['Module'] = 'Module failure.';
                     }
                 } else {
+                    $errors['failed_servers'] = $result;
                     $lastResult = array_pop($result);
                     $resultString = (count($result) > 0) ? implode(', ', $result) . ' and ' . $lastResult : $lastResult;
-                    $errors['failed_servers'] = $result;
                     $message = __('Event published but not pushed to %s, re-try later. If the issue persists, make sure that the correct sync user credentials are used for the server link and that the sync user on the remote server has authentication privileges.', $resultString);
                 }
             } else {
@@ -4131,7 +4142,13 @@ class EventsController extends AppController
                 }
             }
             $this->Event->Attribute->fetchRelated($this->Auth->user(), $resultArray);
-            $typeCategoryMapping = array();
+            $typeCategoryMapping = [
+                'ip-src/ip-dst' => [
+                    'Network activity' => 'Network activity',
+                    'Payload delivery' => 'Payload delivery',
+                    'External analysis' => 'External analysis'
+                ],
+            ];
             foreach ($this->Event->Attribute->categoryDefinitions as $k => $cat) {
                 foreach ($cat['types'] as $type) {
                     $typeCategoryMapping[$type][$k] = $k;
@@ -4373,12 +4390,12 @@ class EventsController extends AppController
         $id = $event['Event']['id'];
         $exports = array(
             'json' => array(
-                'url' => $this->baseurl . '/events/restSearch/json/eventid:' . $id . '.json',
+                'url' => $this->baseurl . '/events/restSearch/json/includeAnalystData:1/eventid:' . $id . '.json',
                 'text' => __('MISP JSON (metadata + all attributes)'),
                 'requiresPublished' => false,
                 'checkbox' => true,
                 'checkbox_text' => __('Encode Attachments'),
-                'checkbox_set' => $this->baseurl . '/events/restSearch/json/withAttachments:1/eventid:' . $id . '.json',
+                'checkbox_set' => $this->baseurl . '/events/restSearch/json/withAttachments:1/includeAnalystData:1/eventid:' . $id . '.json',
                 'checkbox_default' => true,
             ),
             'xml' => array(
@@ -4832,16 +4849,18 @@ class EventsController extends AppController
     public function updateGraph($id, $type = 'event')
     {
         $user = $this->_closeSession();
+
         $validTools = array('event', 'galaxy', 'tag');
         if (!in_array($type, $validTools, true)) {
             throw new MethodNotAllowedException(__('Invalid type.'));
         }
+
         $this->loadModel('Taxonomy');
         $this->loadModel('GalaxyCluster');
         App::uses('CorrelationGraphTool', 'Tools');
-        $grapher = new CorrelationGraphTool();
+
         $data = $this->request->is('post') ? $this->request->data : array();
-        $grapher->construct($this->Event, $this->Taxonomy, $this->GalaxyCluster, $user, $data);
+        $grapher = new CorrelationGraphTool($this->Event, $this->Taxonomy, $this->GalaxyCluster, $user, $data);
         $json = $grapher->buildGraphJson($id, $type);
         array_walk_recursive($json, function (&$item, $key) {
             if (!mb_detect_encoding($item, 'utf-8', true)) {

app/Controller/FeedsController.php

@@ -74,6 +74,8 @@ class FeedsController extends AppController
                 );
             }
         }
+        $loggedUser = $this->Auth->user();
+        $this->loadModel('TagCollection');
 
         $this->CRUD->index([
             'filters' => [
@@ -92,7 +94,7 @@ class FeedsController extends AppController
                 'source_format'
             ],
             'conditions' => $conditions,
-            'afterFind' => function (array $feeds) {
+            'afterFind' => function (array $feeds) use ($loggedUser) {
                 if ($this->_isSiteAdmin()) {
                     $feeds = $this->Feed->attachFeedCacheTimestamps($feeds);
                 }
@@ -106,6 +108,19 @@ class FeedsController extends AppController
                     }
                 }
 
+                foreach ($feeds as &$feed) {
+                    if (!empty($feed['Feed']['tag_collection_id'])) {
+                        $tagCollection = $this->TagCollection->fetchTagCollection($loggedUser, [
+                            'conditions' => [
+                                'TagCollection.id' => $feed['Feed']['tag_collection_id'],
+                            ]
+                        ]);
+                        if (!empty($tagCollection)) {
+                            $feed['TagCollection'] = $tagCollection;
+                        }
+                    }
+                }
+
                 return $feeds;
             }
         ]);
@@ -294,6 +309,10 @@ class FeedsController extends AppController
         }
         $tags = $this->Event->EventTag->Tag->find('list', array('fields' => array('Tag.name'), 'order' => array('lower(Tag.name) asc')));
         $tags[0] = 'None';
+        $this->loadModel('TagCollection');
+        $tagCollections = $this->TagCollection->fetchTagCollection($this->Auth->user());
+        $tagCollections = Hash::combine($tagCollections, '{n}.TagCollection.id', '{n}.TagCollection.name');
+        $tagCollections[0] = 'None';
 
         $this->loadModel('Server');
         $allTypes = $this->Server->getAllTypes();
@@ -304,6 +323,7 @@ class FeedsController extends AppController
                 'order' => 'LOWER(name)'
             )),
             'tags' => $tags,
+            'tag_collections' => $tagCollections,
             'feedTypes' => $this->Feed->getFeedTypesOptions(),
             'sharingGroups' => $sharingGroups,
             'distributionLevels' => $distributionLevels,
@@ -340,6 +360,7 @@ class FeedsController extends AppController
                 'distribution',
                 'sharing_group_id',
                 'tag_id',
+                'tag_collection_id',
                 'event_id',
                 'publish',
                 'delta_merge',
@@ -442,8 +463,17 @@ class FeedsController extends AppController
         if (empty(Configure::read('Security.disable_local_feed_access'))) {
             $inputSources['local'] = 'Local';
         }
+        $tags = $this->Event->EventTag->Tag->find('all', [
+            'recursive' => -1,
+            'fields' => ['Tag.name', 'Tag.id'],
+            'order' => ['lower(Tag.name) asc']
+        ]);
         $tags = $this->Event->EventTag->Tag->find('list', array('fields' => array('Tag.name'), 'order' => array('lower(Tag.name) asc')));
         $tags[0] = 'None';
+        $this->loadModel('TagCollection');
+        $tagCollections = $this->TagCollection->fetchTagCollection($this->Auth->user());
+        $tagCollections = Hash::combine($tagCollections, '{n}.TagCollection.id', '{n}.TagCollection.name');
+        $tagCollections[0] = 'None';
 
         $this->loadModel('Server');
         $allTypes = $this->Server->getAllTypes();
@@ -457,6 +487,7 @@ class FeedsController extends AppController
                 'order' => 'LOWER(name)'
             )),
             'tags' => $tags,
+            'tag_collections' => $tagCollections,
             'feedTypes' => $this->Feed->getFeedTypesOptions(),
             'sharingGroups' => $sharingGroups,
             'distributionLevels' => $distributionLevels,

app/Controller/GalaxiesController.php

@@ -202,7 +202,7 @@ class GalaxiesController extends AppController
         }
         $result = $this->Galaxy->save($galaxy);
         if ($result) {
-            $message = __('Galaxy enabled');
+            $message = __('Galaxy %s', $enabled ? __('enabled') : __('disabled'));
             if ($this->_isRest()) {
                 return $this->RestResponse->saveSuccessResponse('Galaxy', 'toggle', false, $this->response->type(), $message);
             } else {

app/Controller/GalaxyClustersController.php

@@ -59,6 +59,8 @@ class GalaxyClustersController extends AppController
             $contextConditions['GalaxyCluster.default'] = true;
         } elseif ($filters['context'] == 'custom') {
             $contextConditions['GalaxyCluster.default'] = false;
+        } elseif ($filters['context'] == 'orgc') {
+            $contextConditions['GalaxyCluster.orgc_id'] = $this->Auth->user('org_id');
         } elseif ($filters['context'] == 'org') {
             $contextConditions['GalaxyCluster.org_id'] = $this->Auth->user('org_id');
         } elseif ($filters['context'] == 'deleted') {
@@ -173,6 +175,8 @@ class GalaxyClustersController extends AppController
      */
     public function view($id)
     {
+        $this->GalaxyCluster->includeAnalystData = true;
+        $this->GalaxyCluster->includeAnalystDataRecursive = true;
         $cluster = $this->GalaxyCluster->fetchIfAuthorized($this->Auth->user(), $id, 'view', $throwErrors=true, $full=true);
         $tag = $this->GalaxyCluster->Tag->find('first', array(
             'conditions' => array(
@@ -208,6 +212,7 @@ class GalaxyClustersController extends AppController
         $this->loadModel('Attribute');
         $distributionLevels = $this->Attribute->distributionLevels;
         $this->set('distributionLevels', $distributionLevels);
+        $this->set('shortDist', $this->Attribute->shortDist);
         if (!$cluster['GalaxyCluster']['default'] && !$cluster['GalaxyCluster']['published'] && $cluster['GalaxyCluster']['orgc_id'] == $this->Auth->user()['org_id']) {
             $this->Flash->warning(__('This cluster is not published. Users will not be able to use it'));
         }

app/Controller/LogsController.php

@@ -117,9 +117,15 @@ class LogsController extends AppController
             $this->paginate['conditions']['Log.action'] = $validFilters[$this->params['named']['filter']]['values'];
         }
         foreach ($filters as $key => $value) {
+            if ($key == 'page' || $key == 'limit') { // These should not be part of the condition parameter
+                continue;
+            }
             if ($key === 'created') {
                 $key = 'created >=';
             }
+            if ($key == 'page' || $key == 'limit') {
+                continue;
+            }
             $this->paginate['conditions']["Log.$key"] = $value;
         }
         $this->set('validFilters', $validFilters);

app/Controller/ObjectsController.php

@@ -1410,4 +1410,34 @@ class ObjectsController extends AppController
         }
         return $conditions;
     }
+
+    public function viewAnalystData($id, $seed = null)
+    {
+        $this->MispObject->includeAnalystDataRecursive = true;
+        $object = $this->MispObject->fetchObjects(
+            $this->Auth->user(),
+            [
+                'conditions' => $this->__objectIdToConditions($id)
+            ]
+        );
+        if(empty($object)) {
+            throw new NotFoundException(__('Invalid Object.'));
+        } else {
+            $object[0]['Object'] = array_merge_recursive($object[0]['Object'], $this->MispObject->attachAnalystData($object[0]['Object']));
+        }
+        if ($this->_isRest()) {
+            $validFields = ['Note', 'Opinion', 'Relationship'];
+            $results = [];
+            foreach ($validFields as $field) {
+                if (!empty($object[0]['Object'][$field])) {
+                    $results[$field] = $object[0]['Object'][$field];
+                }
+            }
+            return $this->RestResponse->viewData($results, $this->response->type());
+        }
+        $this->layout = null;
+        $this->set('shortDist', $this->MispObject->Attribute->shortDist);
+        $this->set('object', $object[0]['Object']);
+        $this->set('seed', $seed);
+    }
 }

app/Controller/OrganisationsController.php

@@ -481,8 +481,8 @@ class OrganisationsController extends AppController
             $extension = pathinfo($logo['name'], PATHINFO_EXTENSION);
             $filename = $orgId . '.' . ($extension === 'svg' ? 'svg' : 'png');
 
-            if ($logo['size'] > 250*1024) {
-                $this->Flash->error(__('This organisation logo is too large, maximum file size allowed is 250kB.'));
+            if ($logo['size'] > 250 * 1024) {
+                $this->Flash->error(__('This organisation logo is too large, maximum file size allowed is 250 kB.'));
                 return false;
             }
 
@@ -490,12 +490,20 @@ class OrganisationsController extends AppController
                 $this->Flash->error(__('Invalid file extension, Only PNG and SVG images are allowed.'));
                 return false;
             }
-
-            $imgMime = mime_content_type($logo['tmp_name']);
-            if ($extension === 'png' && !exif_imagetype($logo['tmp_name'])) {
+            $matches = null;
+            $tmp_name = $logo['tmp_name'];
+            if (preg_match_all('/[\w\/\-\.]*/', $tmp_name, $matches) && file_exists($logo['tmp_name'])) {
+                $tmp_name = $matches[0][0];
+                $imgMime = mime_content_type($tmp_name);
+            } else {
+                throw new NotFoundException(__('Invalid file.'));    
+            }
+            if ($extension === 'png' && (function_exists('exif_imagetype') && !exif_imagetype($logo['tmp_name']))) {
                 $this->Flash->error(__('This is not a valid PNG image.'));
                 return false;
-            } else if ($extension === 'svg' && !($imgMime === 'image/svg+xml' || $imgMime === 'image/svg')) {
+            }
+
+            if ($extension === 'svg' && !($imgMime === 'image/svg+xml' || $imgMime === 'image/svg')) {
                 $this->Flash->error(__('This is not a valid SVG image.'));
                 return false;
             }
@@ -505,8 +513,8 @@ class OrganisationsController extends AppController
                 return false;
             }
 
-            if (!empty($logo['tmp_name']) && is_uploaded_file($logo['tmp_name'])) {
-                return move_uploaded_file($logo['tmp_name'], APP . 'webroot/img/orgs/' . $filename);
+            if (!empty($tmp_name) && is_uploaded_file($tmp_name)) {
+                return move_uploaded_file($tmp_name, APP . 'files/img/orgs/' . $filename);
             }
         }
 

app/Controller/ServersController.php

@@ -529,7 +529,7 @@ class ServersController extends AppController
             
             if (!$fail) {
                 // say what fields are to be updated
-                $fieldList = array('id', 'url', 'push', 'pull', 'push_sightings', 'push_galaxy_clusters', 'pull_galaxy_clusters', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'remove_missing_tags', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
+                $fieldList = array('id', 'url', 'push', 'pull', 'push_sightings', 'push_galaxy_clusters', 'pull_galaxy_clusters', 'push_analyst_data', 'pull_analyst_data', 'caching_enabled', 'unpublish_event', 'publish_without_email', 'remote_org_id', 'name' ,'self_signed', 'remove_missing_tags', 'cert_file', 'client_cert_file', 'push_rules', 'pull_rules', 'internal', 'skip_proxy');
                 $this->request->data['Server']['id'] = $id;
                 if (isset($this->request->data['Server']['authkey']) && "" != $this->request->data['Server']['authkey']) {
                     $fieldList[] = 'authkey';
@@ -776,7 +776,7 @@ class ServersController extends AppController
             if (!Configure::read('MISP.background_jobs')) {
                 $result = $this->Server->pull($this->Auth->user(), $technique, $s);
                 if (is_array($result)) {
-                    $success = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4]);
+                    $success = __('Pull completed. %s events pulled, %s events could not be pulled, %s proposals pulled, %s sightings pulled, %s clusters pulled, %s analyst data pulled.', count($result[0]), count($result[1]), $result[2], $result[3], $result[4], $result[5]);
                 } else {
                     $error = $result;
                 }
@@ -784,6 +784,7 @@ class ServersController extends AppController
                 $this->set('fails', $result[1]);
                 $this->set('pulledProposals', $result[2]);
                 $this->set('pulledSightings', $result[3]);
+                $this->set('pulledAnalystData', $result[5]);
             } else {
                 $this->loadModel('Job');
                 $jobId = $this->Job->createJob(
@@ -1769,6 +1770,7 @@ class ServersController extends AppController
                 $perm_sighting = isset($result['info']['perm_sighting']) ? $result['info']['perm_sighting'] : false;
                 $local_version = $this->Server->checkMISPVersion();
                 $version = explode('.', $result['info']['version']);
+                $uuid = isset($result['info']['uuid']) ? $result['info']['uuid'] : '?';
                 $mismatch = false;
                 $newer = false;
                 $parts = array('major', 'minor', 'hotfix');
@@ -1804,6 +1806,7 @@ class ServersController extends AppController
                     'response_encoding' => isset($result['post']['content-encoding']) ? $result['post']['content-encoding'] : null,
                     'request_encoding' => isset($result['info']['request_encoding']) ? $result['info']['request_encoding'] : null,
                     'client_certificate' => $result['client_certificate'],
+                    'uuid' => $uuid,
                 ], 'json');
             } else {
                 $result['status'] = 3;
@@ -1887,6 +1890,8 @@ class ServersController extends AppController
             'perm_sync' => (bool) $user['Role']['perm_sync'],
             'perm_sighting' => (bool) $user['Role']['perm_sighting'],
             'perm_galaxy_editor' => (bool) $user['Role']['perm_galaxy_editor'],
+            'perm_analyst_data' => (bool) $user['Role']['perm_analyst_data'],
+            'uuid' => $user['Role']['perm_sync'] ? Configure::read('MISP.uuid') : '-',
             'request_encoding' => $this->CompressedRequestHandler->supportedEncodings(),
             'filter_sightings' => true, // check if Sightings::filterSightingUuidsForPush method is supported
         ];
@@ -2079,7 +2084,10 @@ class ServersController extends AppController
 
     public function updateJSON()
     {
-        $results = $this->Server->updateJSON();
+        $results = [];
+        foreach ($this->Server->updateJSON() as $type => $result) {
+            $results[$type] = $results['success'];
+        }
         return $this->RestResponse->viewData($results, $this->response->type());
     }
 

app/Controller/SightingBlocklistsController.php

@@ -0,0 +1,47 @@
+<?php
+App::uses('AppController', 'Controller');
+
+class SightingBlocklistsController extends AppController
+{
+    public $components = array('Session', 'RequestHandler', 'BlockList');
+
+    public function beforeFilter()
+    {
+        parent::beforeFilter();
+        if (!$this->_isSiteAdmin()) {
+            $this->redirect('/');
+        }
+        if (Configure::check('MISP.enableSightingBlocklisting') && !Configure::read('MISP.enableSightingBlocklisting') !== false) {
+            $this->Flash->info(__('Sighting BlockListing is not currently enabled on this instance.'));
+            $this->redirect('/');
+        }
+    }
+
+    public $paginate = array(
+            'limit' => 60,
+            'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page.
+            'order' => array(
+                    'SightingBlocklist.created' => 'DESC'
+            ),
+    );
+
+    public function index()
+    {
+        return $this->BlockList->index($this->_isRest());
+    }
+
+    public function add()
+    {
+        return $this->BlockList->add($this->_isRest());
+    }
+
+    public function edit($id)
+    {
+        return $this->BlockList->edit($this->_isRest(), $id);
+    }
+
+    public function delete($id)
+    {
+        return $this->BlockList->delete($this->_isRest(), $id);
+    }
+}

app/Controller/TagCollectionsController.php

@@ -357,7 +357,7 @@ class TagCollectionsController extends AppController
             if (!$tagCollection) {
                 throw new NotFoundException(__('Invalid tag collection.'));
             }
-            if ($this->ACL->canModifyTagCollection($this->Auth->user(), $tagCollection)) {
+            if (!$this->ACL->canModifyTagCollection($this->Auth->user(), $tagCollection)) {
                 throw new ForbiddenException(__('You dont have a permission to do that'));
             }
             $tagCollectionTag = $this->TagCollection->TagCollectionTag->find('first', [

app/Controller/TagsController.php

@@ -820,6 +820,15 @@ class TagsController extends AppController
                     }
                     $message = 'Global tag ' . $existingTag['Tag']['name'] . '(' . $existingTag['Tag']['id'] . ') successfully attached to ' . $objectType . '(' . $object[$objectType]['id'] . ').';
                 }
+                $this->loadModel('Log');
+                $this->Log->createLogEntry(
+                    $this->Auth->user(),
+                    'attachTagToObject',
+                    $objectType,
+                    $object[$objectType]['id'],
+                    $message,
+                    null
+                );
                 $successes++;
             } else {
                 $fails[] = __('Failed to attach tag to object.');
@@ -896,6 +905,17 @@ class TagsController extends AppController
         $result = $this->$objectType->$connectorObject->delete($existingAssociation[$connectorObject]['id']);
         if ($result) {
             $message = __('%s tag %s (%s) successfully removed from %s(%s).', $local ? __('Local') : __('Global'), $existingTag['Tag']['name'], $existingTag['Tag']['id'], $objectType, $object[$objectType]['id']);
+            $this->loadModel('Log');
+            $this->Log->createLogEntry(
+                $this->Auth->user(),
+                'removeTagFromObject',
+                $objectType,
+                $object[$objectType]['id'],
+                $message,
+                __(
+                    '',
+                )
+            );
             if (!$local) {
                 if ($objectType === 'Attribute') {
                     $this->Attribute->touch($object['Attribute']['id']);

app/Controller/UsersController.php

@@ -1,5 +1,5 @@
 <?php
-App::uses('AppController', 'Controller', 'OTPHP\TOTP');
+App::uses('AppController', 'Controller');
 
 /**
  * @property User $User
@@ -1214,11 +1214,13 @@ class UsersController extends AppController
                     $this->Auth->constructAuthenticate();
                 }
                 // user has TOTP token, check creds and redirect to TOTP validation
-                if (!empty($unauth_user['User']['totp']) && !$unauth_user['User']['disabled'] && class_exists('\OTPHP\TOTP')) {
-                    $user = $this->Auth->identify($this->request, $this->response);
-                    if ($user && !$user['disabled']) {
-                        $this->Session->write('otp_user', $user);
-                        return $this->redirect('otp');
+                if (!Configure::read('Security.otp_disabled')) {
+                    if (!empty($unauth_user['User']['totp']) && !$unauth_user['User']['disabled'] && class_exists('\OTPHP\TOTP')) {
+                        $user = $this->Auth->identify($this->request, $this->response);
+                        if ($user && !$user['disabled']) {
+                            $this->Session->write('otp_user', $user);
+                            return $this->redirect('otp');
+                        }
                     }
                 }
             }
@@ -1811,6 +1813,7 @@ class UsersController extends AppController
             $this->Flash->error(__("The required PHP libraries to support TOTP are not installed. Please contact your administrator to address this."));
             $this->redirect($this->referer());
         }
+
         // only allow the users themselves to generate a TOTP secret.
         // If TOTP is enforced they will be invited to generate it at first login
         $user = $this->User->find('first', array(
@@ -1880,8 +1883,9 @@ class UsersController extends AppController
         $this->set('secret', $secret);
     }
 
-    public function totp_delete($id) {
-        if ($this->request->is('post') || $this->request->is('delete')) {
+    public function totp_delete($id)
+    {
+        if ($this->request->is(['post', 'delete'])) {
             $user = $this->User->find('first', array(
                 'conditions' => $this->__adminFetchConditions($id),
                 'recursive' => -1,
@@ -1988,8 +1992,7 @@ class UsersController extends AppController
     // shows some statistics about the instance
     public function statistics($page = 'data')
     {
-        $user = $this->Auth->user();
-        @session_write_close(); // loading this page can take long time, so we can close session
+        $user = $this->_closeSession(true); // loading this page can take long time, so we can close session
 
         if (!$this->_isRest()) {
             $pages = [
@@ -2071,7 +2074,7 @@ class UsersController extends AppController
 
         $stats['attribute_count'] = $this->User->Event->Attribute->find('count', array('conditions' => array('Attribute.deleted' => 0), 'recursive' => -1));
         $stats['attribute_count_month'] = $this->User->Event->Attribute->find('count', array('conditions' => array('Attribute.timestamp >' => $this_month, 'Attribute.deleted' => 0), 'recursive' => -1));
-        $stats['attributes_per_event'] = round($stats['attribute_count'] / $stats['event_count']);
+        $stats['attributes_per_event'] = $stats['event_count'] != 0 ? round($stats['attribute_count'] / $stats['event_count']) : 0;
 
         $stats['correlation_count'] = $this->User->Event->Attribute->Correlation->find('count', array('recursive' => -1));
 
@@ -2082,7 +2085,7 @@ class UsersController extends AppController
         $stats['org_count'] = count($orgs);
         $stats['local_org_count'] = $local_orgs_count;
         $stats['contributing_org_count'] = $this->User->Event->find('count', array('recursive' => -1, 'group' => array('Event.orgc_id')));
-        $stats['average_user_per_org'] = round($stats['user_count'] / $stats['local_org_count'], 1);
+        $stats['average_user_per_org'] = $stats['local_org_count'] != 0 ?  round($stats['user_count'] / $stats['local_org_count'], 1) : 0;
 
         $this->loadModel('Thread');
         $stats['thread_count'] = $this->Thread->find('count', array('conditions' => array('Thread.post_count >' => 0), 'recursive' => -1));
@@ -2603,6 +2606,7 @@ class UsersController extends AppController
                 'org_name',
                 'org_uuid',
                 'message',
+                'pgp',
                 'custom_perms',
                 'perm_sync',
                 'perm_publish',
@@ -3186,10 +3190,6 @@ class UsersController extends AppController
 
     public function forgot()
     {
-        if (empty(Configure::read('Security.allow_password_forgotten'))) {
-            $this->Flash->error(__('This feature is disabled.'));
-            $this->redirect('/');
-        }
         if (!empty($this->Auth->user()) && !$this->_isRest()) {
             $this->Flash->info(__('You are already logged in, no need to ask for a password reset. Log out first.'));
             $this->redirect('/');
@@ -3215,10 +3215,6 @@ class UsersController extends AppController
 
     public function password_reset($token)
     {
-        if (empty(Configure::read('Security.allow_password_forgotten'))) {
-            $this->Flash->error(__('This feature is disabled.'));
-            $this->redirect('/');
-        }
         $this->loadModel('Server');
         $this->set('complexity', !empty(Configure::read('Security.password_policy_complexity')) ? Configure::read('Security.password_policy_complexity') : $this->Server->serverSettings['Security']['password_policy_complexity']['value']);
         $this->set('length', !empty(Configure::read('Security.password_policy_length')) ? Configure::read('Security.password_policy_length') : $this->Server->serverSettings['Security']['password_policy_length']['value']);
@@ -3235,7 +3231,7 @@ class UsersController extends AppController
                 $this->redirect('/');
             }
         }
-        if ($this->request->is('post') || $this->request->is('put')) {
+        if ($this->request->is(['post', 'put'])) {
             $abortPost = false;
             return $this->__pw_change(['User' => $user], 'password_reset', $abortPost, $token, true);
         }

app/Error/AppExceptionRenderer.php

@@ -0,0 +1,77 @@
+<?php
+
+App::uses('ExceptionRenderer', 'Error');
+
+class AppExceptionRenderer extends ExceptionRenderer {
+
+    public function __construct($exception) {
+        $this->controller = $this->_getController($exception);
+
+        if (method_exists($this->controller, 'appError')) {
+            $this->controller->appError($exception);
+            return;
+        }
+        $method = $template = Inflector::variable(str_replace('Exception', '', get_class($exception)));
+        $code = $exception->getCode();
+
+        $methodExists = method_exists($this, $method);
+        if ($exception instanceof CakeException && !$methodExists) {
+            $this->_customErrorLogging($exception);
+            $method = '_cakeError';
+            if (empty($template) || $template === 'internalError') {
+                $template = 'error500';
+            }
+        } elseif ($exception instanceof PDOException) {
+            $method = 'pdoError';
+            $template = 'pdo_error';
+            $code = 500;
+        } elseif (!$methodExists) {
+            $method = 'error500';
+            if ($code >= 400 && $code < 500) {
+                $method = 'error400';
+            }
+        }
+
+        $isNotDebug = !Configure::read('debug');
+        if ($isNotDebug && $method === '_cakeError') {
+            $method = 'error400';
+        }
+        if ($isNotDebug && $code == 500) {
+            $method = 'error500';
+        }
+        $this->template = $template;
+        $this->method = $method;
+        $this->error = $exception;
+    }
+
+    protected function _customErrorLogging($exception): bool
+    {
+        $message = $exception->getMessage();
+        $errorDetection = [
+            'Maximum execution time of' => 'timeout',
+            'Allowed memory size of' => 'out of memory'
+        ];
+        $user = $this->controller->Auth->user();
+        $ua = env('HTTP_USER_AGENT');
+        $source = $this->controller->IndexFilter->isRest() ? 'API' : 'web UI';
+        if (strpos($ua, 'MISP ') !== false) {
+            $source = 'MISP sync';
+        }
+        foreach ($errorDetection as $search => $errorName) {
+            if (strpos($message, $search) !== false) {
+                $logMessage = sprintf(
+                    '%s %s error triggered by User %s (%s) via the %s on %s.',
+                    date('Y-m-d H:i:s'),
+                    $errorName,
+                    $user['id'],
+                    $user['email'],
+                    $source,
+                    $this->controller->request->here()
+                );
+                file_put_contents(LOGS . 'fatal_error.log', $logMessage . PHP_EOL, FILE_APPEND);
+                return true;
+            }
+        }
+        return true;
+    }
+}

app/Lib/Dashboard/AchievementsWidget.php

@@ -191,4 +191,3 @@ class AchievementsWidget
         return $result;
     }
 }
-?>

app/Lib/Dashboard/AttackWidget.php

@@ -35,4 +35,3 @@ class AttackWidget
         return $data;
     }
 }
-?>

app/Lib/Dashboard/BenchmarkTopListWidget.php

@@ -0,0 +1,115 @@
+<?php
+
+
+class BenchmarkTopListWidget
+{
+    public $title = 'Benchmark top list';
+    public $render = 'MultiLineChart';
+    public $width = 3;
+    public $height = 3;
+    public $description = 'A graph showing the top list for a given scope and field in the captured metrics.';
+    public $cacheLifetime = false;
+    public $autoRefreshDelay = 30;
+    public $params = array(
+        'days' => 'Number of days to consider for the graph. There will be a data entry for each day (assuming the benchmarking has been enabled). Defaults to returning all data.',
+        'weeks' => 'Number of weeks to consider for the graph. There will be a data entry for each day (assuming the benchmarking has been enabled). Defaults to returning all data.',
+        'months' => 'Number of months to consider for the graph. There will be a data entry for each day (assuming the benchmarking has been enabled). Defaults to returning all data.',
+        'scope' => 'The scope of the benchmarking refers to what was being tracked. The following scopes are valid: user, endpoint, user_agent',
+        'field' => 'The individual metric to be queried from the benchmark results. Valid values are: time, sql_time, sql_queries, memory, endpoint',
+        'average' => 'If you wish to view the averages per scope/field, set this variable to true. It will divide the result by the number of executions recorded for the scope/field combination for the given day.'
+    );
+    public $Benchmark;
+    public $User;
+
+    public $placeholder =
+        '{
+    "days": "30",
+    "scope": "endpoints",
+    "field": "sql_time"
+}';
+
+    public function handler($user, $options = array())
+    {
+        $this->User = ClassRegistry::init('User');
+        $currentTime = strtotime("now");
+        $endOfDay = strtotime("tomorrow", $currentTime) - 1;
+        if (!empty($options['days'])) {
+            $limit = (int)($options['days']);
+            $delta = 'day';
+        } else if (!empty($options['weeks'])) {
+            $limit = (int)($options['weeks']);
+            $delta = 'week';
+        } else if (!empty($options['months'])) {
+            $limit = (int)($options['months']);
+            $delta = 'month';
+        } else {
+            $limit = 30;
+            $delta = 'day';
+        }
+        $axis_info = [
+            'time' => 'Total time taken (ms)',
+            'sql_time' => 'SQL time taken (ms)',
+            'sql_queries' => 'Queries (#)',
+            'memory' => 'Memory (MB)',
+            'endpoint' => 'Queries to endpoint (#)'
+        ];
+        $y_axis = $axis_info[isset($options['field']) ? $options['field'] : 'time'];
+        $data = ['y-axis' => $y_axis];
+        $data['data'] = array();
+        // Add total users data for all timestamps
+        
+        for ($i = 0; $i < $limit; $i++) {
+            $itemTime = strtotime('- ' . $i . $delta, $endOfDay);
+            $item = array();
+            $date = strftime('%Y-%m-%d', $itemTime);
+            $item = $this->getData($date, $options);
+            if (!empty($item)) {
+                $item['date'] = $date;
+                $data['data'][] = $item;
+            }
+        }
+        $keys = [];
+        foreach ($data['data'] as $day_data) {
+            foreach ($day_data as $key => $temp) {
+                $keys[$key] = 1;
+            }
+        }
+        $keys = array_keys($keys);
+        foreach ($data['data'] as $k => $day_data) {
+            foreach ($keys as $key) {
+                if (!isset($day_data[$key])) {
+                    $data['data'][$k][$key] = 0;
+                }
+            }
+            foreach ($day_data as $key => $temp) {
+                $keys[$key] = 1;
+            }
+        }
+        return $data;
+    }
+
+    private function getData($time, $options)
+    {
+        $dates = [$time];
+        $this->Benchmark = new BenchmarkTool($this->User);
+        $result = $this->Benchmark->getTopList(
+            isset($options['scope']) ? $options['scope'] : 'endpoint',
+            isset($options['field']) ? $options['field'] : 'memory',
+            $dates,
+            isset($options['limit']) ? $options['limit'] : 5,
+            isset($options['average']) ? $options['average'] : false,
+        );
+        if (!empty($result)) {
+            return $result[$time];
+        }
+        return false;
+    }
+
+    public function checkPermissions($user)
+    {
+        if (empty($user['Role']['perm_site_admin'])) {
+            return false;
+        }
+        return true;
+    }
+}

app/Lib/Dashboard/EventEvolutionLineWidget.php

@@ -58,9 +58,11 @@ class EventEvolutionLineWidget
             'recursive' => -1
         ];
         $eparams = [];
+        $filteringOnOrg = false;
         if (!empty($options['filter']) && is_array($options['filter'])) {
             foreach ($this->validFilterKeys as $filterKey) {
                 if (!empty($options['filter'][$filterKey])) {
+                    $filteringOnOrg = true;
                     if (!is_array($options['filter'][$filterKey])) {
                         $options['filter'][$filterKey] = [$options['filter'][$filterKey]];
                     }
@@ -87,6 +89,9 @@ class EventEvolutionLineWidget
             'conditions' => $oparams['conditions'],
             'fields' => ['id']
         ]);
+        if ($filteringOnOrg) {
+            $eparams['conditions']['AND']['Event.orgc_id IN'] = !empty($org_ids) ? $org_ids : [-1];
+        }
         $this->Event->virtualFields = [
             'published_date' => null
         ];

app/Lib/Dashboard/MispAdminSyncTestWidget.php

@@ -37,6 +37,10 @@ class MispAdminSyncTestWidget
                     $colour = 'orange';
                     $message .= ' ' . __('No sighting access.');
                 }
+                if (empty($result['info']['perm_analyst_data'])) {
+                    $colour = 'orange';
+                    $message .= ' ' . __('No analyst data sync access.');
+                }
             } else {
                 $colour = 'red';
                 $message = $syncTestErrorCodes[$result['status']];

app/Lib/Dashboard/OrgContributionToplistWidget.php

@@ -137,4 +137,3 @@ class OrgContributionToplistWidget
         return ['data' => $results];
     }
 }
-?>

app/Lib/Dashboard/OrganisationListWidget.php

@@ -78,4 +78,3 @@ class OrganisationListWidget
         return ['data' => $results];
     }
 }
-?>

app/Lib/Dashboard/OrganisationMapWidget.php

@@ -85,4 +85,3 @@ class OrganisationMapWidget
         return $results;
     }
 }
-?>

app/Lib/Dashboard/OrgsContributorLastMonthWidget.php

@@ -19,4 +19,3 @@ class OrgsContributorLastMonthWidget extends OrgsContributorsGeneric
         return count($results) > 0;
     }
 }
- ?>

app/Lib/Dashboard/OrgsContributorsGeneric.php

@@ -45,4 +45,3 @@ class OrgsContributorsGeneric
         return $result;
     }
 }
-?>

app/Lib/Dashboard/OrgsUsingMitreWidget.php

@@ -32,4 +32,3 @@ class OrgsUsingMitreWidget extends OrgsContributorsGeneric
         return count($events) > 0;
     }
 }
-?>

app/Lib/Dashboard/OrgsUsingObjectsWidget.php

@@ -25,4 +25,3 @@ class OrgsUsingObjectsWidget extends OrgsContributorsGeneric
         return count($eventsIds) > 0;
     }
 }
-?>

app/Lib/Dashboard/UsageDataWidget.php

@@ -243,7 +243,8 @@ class UsageDataWidget
             $count = $this->Event->Attribute->find('count', [
                 'conditions' => $conditions,
                 'contain' => ['Event'],
-                'recursive' => -1
+                'recursive' => -1,
+                'ignoreIndexHint' => 'deleted'
             ]);
             $this->redis->setEx('misp:dashboard:attribute_count:' . $hash, 3600, $count);
         }

app/Lib/Dashboard/UserContributionToplistWidget.php

@@ -145,4 +145,3 @@ class UserContributionToplistWidget
         return true;
     }
 }
-?>

app/Lib/Tools/AttributeValidationTool.php

@@ -556,7 +556,12 @@ class AttributeValidationTool
                   if (!is_numeric($value) || $value < 0 || $value > 10) {
                       return __('The value has to be a number between 0 and 10.');
                   }
-                  return true;*/
+                return true;*/
+            case 'integer':
+                if (is_int($value)) {
+                    return true;
+                }
+                return __('The value has to be an integer value.');
             case 'iban':
             case 'bic':
             case 'btc':

app/Lib/Tools/BenchmarkTool.php

@@ -0,0 +1,178 @@
+<?php
+
+/**
+ * Get filter parameters from index searches
+ */
+
+class BenchmarkTool
+{
+    /** @var Model */
+    public $Model;
+    
+    /** @var redis */
+    public $redis;
+
+    /** @var retention */
+    private $retention = 0;
+
+    const BENCHMARK_SCOPES = ['user', 'endpoint', 'user_agent'];
+    const BENCHMARK_FIELDS = ['time', 'sql_time', 'sql_queries', 'memory'];
+    const BENCHMARK_UNITS = [
+        'time' => 's',
+        'sql_time' => 'ms',
+        'sql_queries' => '',
+        'memory' => 'MB'
+    ];
+
+    public $namespace = 'misp:benchmark:';
+
+    function __construct(Model $model) {
+        $this->Model = $model;
+    }
+
+    public function getSettings()
+    {
+        return [
+            'scope' => self::BENCHMARK_SCOPES,
+            'field' => self::BENCHMARK_FIELDS,
+            'average' => [0, 1],
+            'aggregate' => [0, 1]
+        ];
+    }
+
+    public function getUnits()
+    {
+        return self::BENCHMARK_UNITS;
+    }
+
+    public function startBenchmark()
+    {
+        $start_time = microtime(true);
+        $this->redis = $this->Model->setupRedis();
+        $this->retention = Configure::check('Plugin.benchmark_retention') ? Configure::read('Plugin.benchmark_retention') : 0;
+        return $start_time;
+    }
+
+    public function stopBenchmark(array $options)
+    {
+        $start_time = $options['start_time'];
+        if (!empty($options['user'])) {
+            $sql = $this->Model->getDataSource()->getLog(false, false);
+            $benchmarkData = [
+                'user' => $options['user'],
+                'endpoint' => $options['controller'] . '/' . $options['action'],
+                'user_agent' => $_SERVER['HTTP_USER_AGENT'],
+                'sql_queries' => $sql['count'],
+                'sql_time' => $sql['time'],
+                'time' => (microtime(true) - $start_time),
+                'memory' => (int)(memory_get_peak_usage(true) / 1024 / 1024),
+                //'date' => date('Y-m-d', strtotime("-3 days"))
+                'date' => date('Y-m-d')
+            ];
+            $this->pushBenchmarkDataToRedis($benchmarkData);
+        } else {
+            $sql = $this->Model->getDataSource()->getLog(false, false);
+            $benchmarkData = [
+                'user' => 'SYSTEM',
+                'endpoint' => $options['controller'] . '/' . $options['action'],
+                'user_agent' => 'CLI',
+                'sql_queries' => $sql['count'],
+                'sql_time' => $sql['time'],
+                'time' => (microtime(true) - $start_time),
+                'memory' => (int)(memory_get_peak_usage(true) / 1024 / 1024),
+                //'date' => date('Y-m-d', strtotime("-3 days"))
+                'date' => date('Y-m-d')
+            ];
+            $this->pushBenchmarkDataToRedis($benchmarkData);
+        }
+    }
+
+    private function pushBenchmarkDataToRedis($benchmarkData)
+    {
+        $this->redis = $this->Model->setupRedis();
+        $this->redis->pipeline();
+        $this->redis->sAdd(
+            $this->namespace . 'days',
+            $benchmarkData['date']
+        );
+        foreach (self::BENCHMARK_SCOPES as $scope) {
+            $this->redis->sAdd(
+                $this->namespace . $scope . ':list',
+                $benchmarkData[$scope]
+            );
+            $this->redis->zIncrBy(
+                $this->namespace . $scope . ':count:' . $benchmarkData['date'],
+                1,
+                $benchmarkData[$scope]
+            );
+            foreach (self::BENCHMARK_FIELDS as $field) {
+                $this->redis->zIncrBy(
+                    $this->namespace . $scope . ':' . $field . ':' . $benchmarkData['date'],
+                    $benchmarkData[$field],
+                    $benchmarkData[$scope]
+                );
+            }
+            $this->redis->zIncrBy(
+                $this->namespace . $scope . ':endpoint:' . $benchmarkData['date'] . ':' . $benchmarkData['user'],
+                1,
+                $benchmarkData['endpoint']
+            );
+        }
+        $this->redis->exec();
+    }
+
+    public function getTopList(string $scope, string $field, array $days = [], $limit = 10, $average = false, $aggregate = false)
+    {
+        if (empty($this->redis)) {
+            $this->redis = $this->Model->setupRedis();
+        }
+        $results = [];
+        if (is_string($days)) {
+            $days = [$days];
+        }
+        foreach ($days as $day) {
+            $temp = $this->redis->zrevrange($this->namespace . $scope . ':' . $field . ':' . $day, 0, $limit, true);
+            foreach ($temp as $k => $v) {
+                if ($average) {
+                    $divisor = $this->redis->zscore($this->namespace . $scope . ':count:' . $day, $k);
+                    if ($aggregate) {
+                        $results['aggregate'][$k] = empty($results['aggregate'][$k]) ? ($v / $divisor) : ($results['aggregate'][$k] + ($v / $divisor));
+                    } else {
+                        $results[$day][$k] = (int)($v / $divisor);
+                    }
+                } else {
+                    if ($aggregate) {
+                        $results['aggregate'][$k] = empty($results['aggregate'][$k]) ? $v : ($results['aggregate'][$k] + $v);
+                    } else {
+                        $results[$day][$k] = $v;
+                    }
+                }
+            }
+        }
+        if ($aggregate && $average) {
+            $count_days = count($days);
+            foreach ($results['aggregate'] as $k => $result) {
+                $results['aggregate'][$k] = (int)($result / $count_days);
+            }
+        }
+        return $results;
+    }
+
+    public function getAllTopLists(array $days = null, $limit = 10, $average = false, $aggregate = false, $scope_filter = [])
+    {
+        if (empty($this->redis)) {
+            $this->redis = $this->Model->setupRedis();
+        }
+        if ($days === null) {
+            $days = $this->redis->smembers($this->namespace . 'days');
+        }
+        foreach (self::BENCHMARK_SCOPES as $scope) {
+            if (empty($scope_filter) || in_array($scope, $scope_filter)) {
+                foreach (self::BENCHMARK_FIELDS as $field) {
+                    $results[$scope][$field] = $this->getTopList($scope, $field, $days, $limit, $average, $aggregate);
+                }
+            }
+        }
+        return $results;
+    }
+}

app/Lib/Tools/CorrelationGraphTool.php

@@ -1,4 +1,6 @@
 <?php
+App::uses('OrgImgHelper', 'View/Helper');
+
   class CorrelationGraphTool
   {
       private $__lookupTables = array();
@@ -9,20 +11,24 @@
       /** @var Taxonomy */
       private $__taxonomyModel;
       private $__galaxyClusterModel = false;
-      private $__user = false;
-      private $__json = array();
+      /** @var User */
+      private $__user;
+      /** @var array */
+      private $data;
+      private $orgImgHelper;
 
-      public function construct(Event $eventModel, $taxonomyModel, $galaxyClusterModel, $user, $json)
+      public function __construct(Event $eventModel, $taxonomyModel, $galaxyClusterModel, array $user, array $data)
       {
           $this->__eventModel = $eventModel;
           $this->__taxonomyModel = $taxonomyModel;
           $this->__galaxyClusterModel = $galaxyClusterModel;
           $this->__user = $user;
-          $this->__json = $json;
+          $this->data = $data;
           $this->__lookupTables = array(
-        'analysisLevels' => $this->__eventModel->analysisLevels,
-        'distributionLevels' => $this->__eventModel->Attribute->distributionLevels
-      );
+            'analysisLevels' => $eventModel->analysisLevels,
+            'distributionLevels' => $eventModel->Attribute->distributionLevels
+          );
+          $this->orgImgHelper = new OrgImgHelper(new View());
           return true;
       }
 
@@ -38,7 +44,7 @@
               'sgReferenceOnly' => true,
           ));
           if (empty($event)) {
-              return $this->__json;
+              return $this->data;
           }
           $this->cleanLinks();
           $event[0]['Event']['Orgc'] = $event[0]['Orgc'];
@@ -75,7 +81,7 @@
       public function buildGraphJson($id, $type = 'event', $action = 'create')
       {
           if ($action == 'delete') {
-              return $this->__json;
+              return $this->data;
           }
           switch ($type) {
         case 'event':
@@ -88,13 +94,13 @@
           $this->__expandTag($id);
           break;
       }
-          return $this->__json;
+          return $this->data;
       }
 
       private function __deleteObject($id)
       {
           $this->cleanLinks();
-          return $this->__json;
+          return $this->data;
       }
 
       private function __handleObjects($objects, $anchor_id, $full = false)
@@ -193,8 +199,8 @@
 
       private function __expandGalaxy($id)
       {
-          if (!empty($this->__json['nodes'])) {
-              foreach ($this->__json['nodes'] as $k => $node) {
+          if (!empty($this->data['nodes'])) {
+              foreach ($this->data['nodes'] as $k => $node) {
                   if ($node['type'] == 'galaxy' && $node['id'] == $id) {
                       $current_galaxy_id = $k;
                       $tag_name = $node['tag_name'];
@@ -205,7 +211,7 @@
               $current_galaxy_id = $this->__addGalaxy($id);
           }
           $this->cleanLinks();
-          $events = $this->__eventModel->EventTag->Tag->fetchSimpleEventsForTag($this->__json['nodes'][$current_galaxy_id]['tag_name'], $this->__user, true);
+          $events = $this->__eventModel->EventTag->Tag->fetchSimpleEventsForTag($this->data['nodes'][$current_galaxy_id]['tag_name'], $this->__user, true);
           foreach ($events as $event) {
               $current_event_id = $this->__createNode('event', $event);
               $this->__addLink($current_event_id, $current_galaxy_id);
@@ -229,7 +235,7 @@
       {
           $link = $this->graphJsonContainsLink($from_id, $to_id);
           if ($link === false) {
-              $this->__json['links'][] = array('source' => $from_id, 'target' => $to_id, 'linkDistance' => $linkDistance);
+              $this->data['links'][] = array('source' => $from_id, 'target' => $to_id, 'linkDistance' => $linkDistance);
           }
       }
 
@@ -240,7 +246,7 @@
           if ($from_uuid == $to_uuid) {
               return false;
           }
-          foreach ($this->__json['nodes'] as $k => $node) {
+          foreach ($this->data['nodes'] as $k => $node) {
               if ($node['uuid'] === $from_uuid) {
                   $from_id = $k;
               }
@@ -277,10 +283,9 @@
             );
             break;
           case 'event':
-            if ($this->orgImgExists($data['Orgc']['name'])) {
-                $image = Configure::read('MISP.baseurl') . '/img/orgs/' . h($data['Orgc']['name']) . '.png';
-            } else {
-                $image = Configure::read('MISP.baseurl') . '/img/orgs/MISP.png';
+            $orgImage = $this->orgImgHelper->getOrgLogoAsBase64($data['Orgc']);
+            if ($orgImage === null) {
+                $orgImage = Configure::read('MISP.baseurl') . '/img/misp-org.png';
             }
             $node = array(
               'unique_id' => 'event-' . $data['id'],
@@ -289,7 +294,7 @@
               'id' => $data['id'],
               'expanded' => $expand,
               'uuid' => $data['uuid'],
-              'image' => $image,
+              'image' => $orgImage,
               'info' => $data['info'],
               'org' => $data['Orgc']['name'],
               'analysis' => $this->__lookupTables['analysisLevels'][$data['analysis']],
@@ -345,11 +350,11 @@
             );
             break;
         }
-              $this->__json['nodes'][] = $node;
-              $current_id = count($this->__json['nodes'])-1;
+              $this->data['nodes'][] = $node;
+              $current_id = count($this->data['nodes'])-1;
           } else {
               if ($expand) {
-                  $this->__json['nodes'][$current_id]['expanded'] = 1;
+                  $this->data['nodes'][$current_id]['expanded'] = 1;
               }
           }
           return $current_id;
@@ -357,11 +362,11 @@
 
       public function cleanLinks()
       {
-          if (isset($this->__json['nodes']) && isset($this->__json['links'])) {
+          if (isset($this->data['nodes']) && isset($this->data['links'])) {
               $links = array();
-              foreach ($this->__json['links'] as $link) {
+              foreach ($this->data['links'] as $link) {
                   $temp = array();
-                  foreach ($this->__json['nodes'] as $k => $node) {
+                  foreach ($this->data['nodes'] as $k => $node) {
                       if ($link['source'] == $node) {
                           $temp['source'] = $k;
                       }
@@ -372,32 +377,24 @@
                   $temp['linkDistance'] = $link['linkDistance'];
                   $links[] = $temp;
               }
-              $this->__json['links'] = $links;
+              $this->data['links'] = $links;
           } else {
-              if (!isset($this->__json['links'])) {
-                  $this->__json['links'] = array();
+              if (!isset($this->data['links'])) {
+                  $this->data['links'] = array();
               }
-              if (!isset($this->__json['nodes'])) {
-                  $this->__json['nodes'] = array();
+              if (!isset($this->data['nodes'])) {
+                  $this->data['nodes'] = array();
               }
           }
           return true;
       }
 
-      public function orgImgExists($org)
-      {
-          if (file_exists(APP . 'webroot' . DS . 'img' . DS . 'orgs' . DS . $org . '.png')) {
-              return true;
-          }
-          return false;
-      }
-
       public function graphJsonContains($type, $element)
       {
-          if (!isset($this->__json['nodes'])) {
+          if (!isset($this->data['nodes'])) {
               return false;
           }
-          foreach ($this->__json['nodes'] as $k => $node) {
+          foreach ($this->data['nodes'] as $k => $node) {
               if ($type == 'event' && $node['type'] == 'event' && $node['id'] == $element['id']) {
                   return $k;
               }
@@ -418,10 +415,10 @@
       }
       public function graphJsonContainsLink($id1, $id2)
       {
-          if (!isset($this->__json['links'])) {
+          if (!isset($this->data['links'])) {
               return false;
           }
-          foreach ($this->__json['links'] as $k => $link) {
+          foreach ($this->data['links'] as $k => $link) {
               if (($link['source'] == $id1 && $link['target'] == $id2) || ($link['source'] == $id2 && $link['target'] == $id1)) {
                   return $k;
               }

app/Lib/Tools/CurlClient.php

@@ -6,8 +6,12 @@ class CurlClient extends HttpSocketExtended
     /** @var resource */
     private $ch;
 
-    /** @var int */
-    private $timeout = 30;
+    /**
+     * Maximum time the transfer is allowed to complete in seconds
+     * 300 seconds is recommended timeout for MISP servers
+     * @var int
+     */
+    private $timeout = 300;
 
     /** @var string|null */
     private $caFile;
@@ -168,6 +172,7 @@ class CurlClient extends HttpSocketExtended
             return;
         }
         $this->proxy = compact('host', 'port', 'method', 'user', 'pass');
+        $this->defaultOptions = $this->generateDefaultOptions(); // regenerate default options in case proxy setting is changed
     }
 
     /**
@@ -198,7 +203,7 @@ class CurlClient extends HttpSocketExtended
             $url .= '?' . http_build_query($query, '', '&', PHP_QUERY_RFC3986);
         }
 
-        $options = $this->defaultOptions;
+        $options = $this->defaultOptions; // this will copy default options
         $options[CURLOPT_URL] = $url;
         $options[CURLOPT_CUSTOMREQUEST] = $method;
 
@@ -206,6 +211,10 @@ class CurlClient extends HttpSocketExtended
             $options[CURLOPT_POSTFIELDS] = $query;
         }
 
+        if ($method === 'HEAD') {
+            $options[CURLOPT_NOBODY] = true;
+        }
+
         if (!empty($request['header'])) {
             $headers = [];
             foreach ($request['header'] as $key => $value) {
@@ -235,7 +244,6 @@ class CurlClient extends HttpSocketExtended
             }
             return $len;
         };
-
         if (!curl_setopt_array($this->ch, $options)) {
             throw new \RuntimeException('curl error: Could not set options');
         }
@@ -272,16 +280,6 @@ class CurlClient extends HttpSocketExtended
      */
     private function constructResponse($body, array $headers, $code)
     {
-        if (isset($responseHeaders['content-encoding']) && $responseHeaders['content-encoding'] === 'zstd') {
-            if (!function_exists('zstd_uncompress')) {
-                throw new SocketException('Response is zstd encoded, but PHP do not support zstd decoding.');
-            }
-            $body = zstd_uncompress($body);
-            if ($body === false) {
-                throw new SocketException('Could not decode zstd encoded response.');
-            }
-        }
-
         $response = new HttpSocketResponseExtended();
         $response->code = $code;
         $response->body = $body;
@@ -320,7 +318,7 @@ class CurlClient extends HttpSocketExtended
             CURLOPT_RETURNTRANSFER => true, // Should cURL return or print out the data? (true = return, false = print)
             CURLOPT_HEADER => false, // Include header in result?
             CURLOPT_TIMEOUT => $this->timeout, // Timeout in seconds
-            CURLOPT_PROTOCOLS => CURLPROTO_HTTPS | CURLPROTO_HTTP, // be sure that only HTTP and HTTPS protocols are enabled,
+            CURLOPT_PROTOCOLS => CURLPROTO_HTTPS | CURLPROTO_HTTP, // be sure that only HTTP and HTTPS protocols are enabled
         ];
 
         if ($this->caFile) {
@@ -336,7 +334,7 @@ class CurlClient extends HttpSocketExtended
         }
 
         if ($this->compress) {
-            $options[CURLOPT_ACCEPT_ENCODING] = $this->supportedEncodings();
+            $options[CURLOPT_ACCEPT_ENCODING] = ''; // empty string means all encodings supported by curl
         }
 
         if ($this->allowSelfSigned) {
@@ -353,25 +351,4 @@ class CurlClient extends HttpSocketExtended
 
         return $options;
     }
-
-    /**
-     * @return string
-     */
-    private function supportedEncodings()
-    {
-        $encodings = [];
-        // zstd is not supported by curl itself, but add support if PHP zstd extension is installed
-        if (function_exists('zstd_uncompress')) {
-            $encodings[] = 'zstd';
-        }
-        // brotli and gzip is supported by curl itself if it is compiled with these features
-        $info = curl_version();
-        if (defined('CURL_VERSION_BROTLI') && $info['features'] & CURL_VERSION_BROTLI) {
-            $encodings[] = 'br';
-        }
-        if ($info['features'] & CURL_VERSION_LIBZ) {
-            $encodings[] = 'gzip, deflate';
-        }
-        return implode(', ', $encodings);
-    }
 }

app/Lib/Tools/HttpSocketExtended.php

@@ -24,7 +24,7 @@ class HttpSocketHttpException extends Exception
             $message .= " for URL $url";
         }
         if ($response->body) {
-            $message .= ': ' . substr($response->body, 0, 100);
+            $message .= ': ' . substr(ltrim($response->body), 0, 100);
         }
 
         parent::__construct($message, (int)$response->code);
@@ -114,10 +114,15 @@ class HttpSocketResponseExtended extends HttpSocketResponse
      */
     public function json()
     {
+        if (strlen($this->body) === 0) {
+            throw new HttpSocketJsonException('Could not parse empty response as JSON.', $this);
+        }
+
         try {
             return JsonTool::decode($this->body);
         } catch (Exception $e) {
-            throw new HttpSocketJsonException('Could not parse response as JSON.', $this, $e);
+            $contentType = $this->getHeader('content-type');
+            throw new HttpSocketJsonException("Could not parse HTTP response as JSON. Received Content-Type $contentType.", $this, $e);
         }
     }
 }

app/Lib/Tools/JSONConverterTool.php

@@ -21,7 +21,7 @@ class JSONConverterTool
 
     public static function convertObject($object, $isSiteAdmin = false, $raw = false)
     {
-        $toRearrange = array('SharingGroup', 'Attribute', 'ShadowAttribute', 'Event', 'CryptographicKey');
+        $toRearrange = array('SharingGroup', 'Attribute', 'ShadowAttribute', 'Event', 'CryptographicKey', 'Note', 'Opinion', 'Relationship');
         foreach ($toRearrange as $element) {
             if (isset($object[$element])) {
                 $object['Object'][$element] = $object[$element];
@@ -40,7 +40,7 @@ class JSONConverterTool
 
     public static function convert($event, $isSiteAdmin=false, $raw = false)
     {
-        $toRearrange = array('Org', 'Orgc', 'SharingGroup', 'Attribute', 'ShadowAttribute', 'RelatedAttribute', 'RelatedEvent', 'Galaxy', 'Object', 'EventReport', 'CryptographicKey');
+        $toRearrange = array('Org', 'Orgc', 'SharingGroup', 'Attribute', 'ShadowAttribute', 'RelatedAttribute', 'RelatedEvent', 'Galaxy', 'Object', 'EventReport', 'CryptographicKey', 'Note', 'Opinion', 'Relationship');
         foreach ($toRearrange as $object) {
             if (isset($event[$object])) {
                 $event['Event'][$object] = $event[$object];
@@ -69,6 +69,16 @@ class JSONConverterTool
             }
         }
 
+        if (isset($event['Event']['Note'])) {
+            $event['Event']['Note'] = self::__cleanAnalystData($event['Event']['Note']);
+        }
+        if (isset($event['Event']['Opinion'])) {
+            $event['Event']['Opinion'] = self::__cleanAnalystData($event['Event']['Opinion']);
+        }
+        if (isset($event['Event']['Relationship'])) {
+            $event['Event']['Relationship'] = self::__cleanAnalystData($event['Event']['Relationship']);
+        }
+
         // cleanup the array from things we do not want to expose
         $tempSightings = array();
         if (!empty($event['Sighting'])) {
@@ -209,6 +219,17 @@ class JSONConverterTool
         return $objects;
     }
 
+    private function __cleanAnalystData($data)
+    {
+        foreach ($data as $k => $entry) {
+            if (empty($entry['SharingGroup'])) {
+                unset($data[$k]['SharingGroup']);
+            }
+        }
+        $data = array_values($data);
+        return $data;
+    }
+
     public static function arrayPrinter($array, $root = true)
     {
         if (is_array($array)) {

app/Lib/Tools/LanguageRFC5646Tool.php

@@ -0,0 +1,245 @@
+<?php
+declare(strict_types=1);
+
+class LanguageRFC5646Tool
+{
+    const RFC5646_LANGUAGE = [
+        'af' => 'Afrikaans',
+        'af-ZA' => 'Afrikaans (South Africa)',
+        'ar' => 'Arabic',
+        'ar-AE' => 'Arabic (U.A.E.)',
+        'ar-BH' => 'Arabic (Bahrain)',
+        'ar-DZ' => 'Arabic (Algeria)',
+        'ar-EG' => 'Arabic (Egypt)',
+        'ar-IQ' => 'Arabic (Iraq)',
+        'ar-JO' => 'Arabic (Jordan)',
+        'ar-KW' => 'Arabic (Kuwait)',
+        'ar-LB' => 'Arabic (Lebanon)',
+        'ar-LY' => 'Arabic (Libya)',
+        'ar-MA' => 'Arabic (Morocco)',
+        'ar-OM' => 'Arabic (Oman)',
+        'ar-QA' => 'Arabic (Qatar)',
+        'ar-SA' => 'Arabic (Saudi Arabia)',
+        'ar-SY' => 'Arabic (Syria)',
+        'ar-TN' => 'Arabic (Tunisia)',
+        'ar-YE' => 'Arabic (Yemen)',
+        'az' => 'Azeri (Latin)',
+        'az-AZ' => 'Azeri (Latin) (Azerbaijan)',
+        'az-Cyrl-AZ' => 'Azeri (Cyrillic) (Azerbaijan)',
+        'be' => 'Belarusian',
+        'be-BY' => 'Belarusian (Belarus)',
+        'bg' => 'Bulgarian',
+        'bg-BG' => 'Bulgarian (Bulgaria)',
+        'bs-BA' => 'Bosnian (Bosnia and Herzegovina)',
+        'ca' => 'Catalan',
+        'ca-ES' => 'Catalan (Spain)',
+        'cs' => 'Czech',
+        'cs-CZ' => 'Czech (Czech Republic)',
+        'cy' => 'Welsh',
+        'cy-GB' => 'Welsh (United Kingdom)',
+        'da' => 'Danish',
+        'da-DK' => 'Danish (Denmark)',
+        'de' => 'German',
+        'de-AT' => 'German (Austria)',
+        'de-CH' => 'German (Switzerland)',
+        'de-DE' => 'German (Germany)',
+        'de-LI' => 'German (Liechtenstein)',
+        'de-LU' => 'German (Luxembourg)',
+        'dv' => 'Divehi',
+        'dv-MV' => 'Divehi (Maldives)',
+        'el' => 'Greek',
+        'el-GR' => 'Greek (Greece)',
+        'en' => 'English',
+        'en-AU' => 'English (Australia)',
+        'en-BZ' => 'English (Belize)',
+        'en-CA' => 'English (Canada)',
+        'en-CB' => 'English (Caribbean)',
+        'en-GB' => 'English (United Kingdom)',
+        'en-IE' => 'English (Ireland)',
+        'en-JM' => 'English (Jamaica)',
+        'en-NZ' => 'English (New Zealand)',
+        'en-PH' => 'English (Republic of the Philippines)',
+        'en-TT' => 'English (Trinidad and Tobago)',
+        'en-US' => 'English (United States)',
+        'en-ZA' => 'English (South Africa)',
+        'en-ZW' => 'English (Zimbabwe)',
+        'eo' => 'Esperanto',
+        'es' => 'Spanish',
+        'es-AR' => 'Spanish (Argentina)',
+        'es-BO' => 'Spanish (Bolivia)',
+        'es-CL' => 'Spanish (Chile)',
+        'es-CO' => 'Spanish (Colombia)',
+        'es-CR' => 'Spanish (Costa Rica)',
+        'es-DO' => 'Spanish (Dominican Republic)',
+        'es-EC' => 'Spanish (Ecuador)',
+        'es-ES' => 'Spanish (Spain)',
+        'es-GT' => 'Spanish (Guatemala)',
+        'es-HN' => 'Spanish (Honduras)',
+        'es-MX' => 'Spanish (Mexico)',
+        'es-NI' => 'Spanish (Nicaragua)',
+        'es-PA' => 'Spanish (Panama)',
+        'es-PE' => 'Spanish (Peru)',
+        'es-PR' => 'Spanish (Puerto Rico)',
+        'es-PY' => 'Spanish (Paraguay)',
+        'es-SV' => 'Spanish (El Salvador)',
+        'es-UY' => 'Spanish (Uruguay)',
+        'es-VE' => 'Spanish (Venezuela)',
+        'et' => 'Estonian',
+        'et-EE' => 'Estonian (Estonia)',
+        'eu' => 'Basque',
+        'eu-ES' => 'Basque (Spain)',
+        'fa' => 'Farsi',
+        'fa-IR' => 'Farsi (Iran)',
+        'fi' => 'Finnish',
+        'fi-FI' => 'Finnish (Finland)',
+        'fo' => 'Faroese',
+        'fo-FO' => 'Faroese (Faroe Islands)',
+        'fr' => 'French',
+        'fr-BE' => 'French (Belgium)',
+        'fr-CA' => 'French (Canada)',
+        'fr-CH' => 'French (Switzerland)',
+        'fr-FR' => 'French (France)',
+        'fr-LU' => 'French (Luxembourg)',
+        'fr-MC' => 'French (Principality of Monaco)',
+        'gl' => 'Galician',
+        'gl-ES' => 'Galician (Spain)',
+        'gu' => 'Gujarati',
+        'gu-IN' => 'Gujarati (India)',
+        'he' => 'Hebrew',
+        'he-IL' => 'Hebrew (Israel)',
+        'hi' => 'Hindi',
+        'hi-IN' => 'Hindi (India)',
+        'hr' => 'Croatian',
+        'hr-BA' => 'Croatian (Bosnia and Herzegovina)',
+        'hr-HR' => 'Croatian (Croatia)',
+        'hu' => 'Hungarian',
+        'hu-HU' => 'Hungarian (Hungary)',
+        'hy' => 'Armenian',
+        'hy-AM' => 'Armenian (Armenia)',
+        'id' => 'Indonesian',
+        'id-ID' => 'Indonesian (Indonesia)',
+        'is' => 'Icelandic',
+        'is-IS' => 'Icelandic (Iceland)',
+        'it' => 'Italian',
+        'it-CH' => 'Italian (Switzerland)',
+        'it-IT' => 'Italian (Italy)',
+        'ja' => 'Japanese',
+        'ja-JP' => 'Japanese (Japan)',
+        'ka' => 'Georgian',
+        'ka-GE' => 'Georgian (Georgia)',
+        'kk' => 'Kazakh',
+        'kk-KZ' => 'Kazakh (Kazakhstan)',
+        'kn' => 'Kannada',
+        'kn-IN' => 'Kannada (India)',
+        'ko' => 'Korean',
+        'ko-KR' => 'Korean (Korea)',
+        'kok' => 'Konkani',
+        'kok-IN' => 'Konkani (India)',
+        'ky' => 'Kyrgyz',
+        'ky-KG' => 'Kyrgyz (Kyrgyzstan)',
+        'lt' => 'Lithuanian',
+        'lt-LT' => 'Lithuanian (Lithuania)',
+        'lv' => 'Latvian',
+        'lv-LV' => 'Latvian (Latvia)',
+        'mi' => 'Maori',
+        'mi-NZ' => 'Maori (New Zealand)',
+        'mk' => 'FYRO Macedonian',
+        'mk-MK' => 'FYRO Macedonian (Former Yugoslav Republic of Macedonia)',
+        'mn' => 'Mongolian',
+        'mn-MN' => 'Mongolian (Mongolia)',
+        'mr' => 'Marathi',
+        'mr-IN' => 'Marathi (India)',
+        'ms' => 'Malay',
+        'ms-BN' => 'Malay (Brunei Darussalam)',
+        'ms-MY' => 'Malay (Malaysia)',
+        'mt' => 'Maltese',
+        'mt-MT' => 'Maltese (Malta)',
+        'nb' => 'Norwegian (Bokm?l)',
+        'nb-NO' => 'Norwegian (Bokm?l) (Norway)',
+        'nl' => 'Dutch',
+        'nl-BE' => 'Dutch (Belgium)',
+        'nl-NL' => 'Dutch (Netherlands)',
+        'nn-NO' => 'Norwegian (Nynorsk) (Norway)',
+        'ns' => 'Northern Sotho',
+        'ns-ZA' => 'Northern Sotho (South Africa)',
+        'pa' => 'Punjabi',
+        'pa-IN' => 'Punjabi (India)',
+        'pl' => 'Polish',
+        'pl-PL' => 'Polish (Poland)',
+        'ps' => 'Pashto',
+        'ps-AR' => 'Pashto (Afghanistan)',
+        'pt' => 'Portuguese',
+        'pt-BR' => 'Portuguese (Brazil)',
+        'pt-PT' => 'Portuguese (Portugal)',
+        'qu' => 'Quechua',
+        'qu-BO' => 'Quechua (Bolivia)',
+        'qu-EC' => 'Quechua (Ecuador)',
+        'qu-PE' => 'Quechua (Peru)',
+        'ro' => 'Romanian',
+        'ro-RO' => 'Romanian (Romania)',
+        'ru' => 'Russian',
+        'ru-RU' => 'Russian (Russia)',
+        'sa' => 'Sanskrit',
+        'sa-IN' => 'Sanskrit (India)',
+        'se' => 'Sami',
+        'se-FI' => 'Sami (Finland)',
+        'se-NO' => 'Sami (Norway)',
+        'se-SE' => 'Sami (Sweden)',
+        'sk' => 'Slovak',
+        'sk-SK' => 'Slovak (Slovakia)',
+        'sl' => 'Slovenian',
+        'sl-SI' => 'Slovenian (Slovenia)',
+        'sq' => 'Albanian',
+        'sq-AL' => 'Albanian (Albania)',
+        'sr-BA' => 'Serbian (Latin) (Bosnia and Herzegovina)',
+        'sr-Cyrl-BA' => 'Serbian (Cyrillic) (Bosnia and Herzegovina)',
+        'sr-SP' => 'Serbian (Latin) (Serbia and Montenegro)',
+        'sr-Cyrl-SP' => 'Serbian (Cyrillic) (Serbia and Montenegro)',
+        'sv' => 'Swedish',
+        'sv-FI' => 'Swedish (Finland)',
+        'sv-SE' => 'Swedish (Sweden)',
+        'sw' => 'Swahili',
+        'sw-KE' => 'Swahili (Kenya)',
+        'syr' => 'Syriac',
+        'syr-SY' => 'Syriac (Syria)',
+        'ta' => 'Tamil',
+        'ta-IN' => 'Tamil (India)',
+        'te' => 'Telugu',
+        'te-IN' => 'Telugu (India)',
+        'th' => 'Thai',
+        'th-TH' => 'Thai (Thailand)',
+        'tl' => 'Tagalog',
+        'tl-PH' => 'Tagalog (Philippines)',
+        'tn' => 'Tswana',
+        'tn-ZA' => 'Tswana (South Africa)',
+        'tr' => 'Turkish',
+        'tr-TR' => 'Turkish (Turkey)',
+        'tt' => 'Tatar',
+        'tt-RU' => 'Tatar (Russia)',
+        'ts' => 'Tsonga',
+        'uk' => 'Ukrainian',
+        'uk-UA' => 'Ukrainian (Ukraine)',
+        'ur' => 'Urdu',
+        'ur-PK' => 'Urdu (Islamic Republic of Pakistan)',
+        'uz' => 'Uzbek (Latin)',
+        'uz-UZ' => 'Uzbek (Latin) (Uzbekistan)',
+        'uz-Cyrl-UZ' => 'Uzbek (Cyrillic) (Uzbekistan)',
+        'vi' => 'Vietnamese',
+        'vi-VN' => 'Vietnamese (Viet Nam)',
+        'xh' => 'Xhosa',
+        'xh-ZA' => 'Xhosa (South Africa)',
+        'zh' => 'Chinese',
+        'zh-CN' => 'Chinese (S)',
+        'zh-HK' => 'Chinese (Hong Kong)',
+        'zh-MO' => 'Chinese (Macau)',
+        'zh-SG' => 'Chinese (Singapore)',
+        'zh-TW' => 'Chinese (T)',
+        'zu' => 'Zulu',
+        'zu-ZA' => 'Zulu (South Africa)'
+    ];
+
+    public static function getLanguages()
+    {
+        return self::RFC5646_LANGUAGE;
+    }
+}

app/Lib/Tools/ProcessTool.php

@@ -55,8 +55,19 @@ class ProcessTool
         if ($logToFile) {
             self::logMessage('Running command ' . implode(' ', $command));
         }
-
-        $process = proc_open($command, $descriptorSpec, $pipes, $cwd);
+        if (version_compare(phpversion(), '7.4.0', '<')) {
+            $temp = [];
+            foreach ($command as $k => $part) {
+                if ($k >= 1) {
+                    $part = escapeshellarg($part);
+                }
+                $temp[] = $part;
+            }
+            $command_stringified = implode(' ', $temp);
+            $process = proc_open($command_stringified, $descriptorSpec, $pipes, $cwd);
+        } else {
+            $process = proc_open($command, $descriptorSpec, $pipes, $cwd);
+        }
         if (!$process) {
             $commandForException = self::commandFormat($command);
             throw new Exception("Command '$commandForException' could be started.");

app/Lib/Tools/ServerSyncTool.php

@@ -14,6 +14,7 @@ class ServerSyncTool
         FEATURE_EDIT_OF_GALAXY_CLUSTER = 'edit_of_galaxy_cluster',
         PERM_SYNC = 'perm_sync',
         PERM_GALAXY_EDITOR = 'perm_galaxy_editor',
+        PERM_ANALYST_DATA = 'perm_analyst_data',
         FEATURE_SIGHTING_REST_SEARCH = 'sighting_rest';
 
     /** @var array */
@@ -215,6 +216,72 @@ class ServerSyncTool
         return $this->post('/galaxies/pushCluster', [$cluster], $logMessage);
     }
 
+    /**
+     * @param array $candidates
+     * @return HttpSocketResponseExtended
+     * @throws HttpSocketHttpException
+     * @throws HttpSocketJsonException
+     */
+    public function filterAnalystDataForPush(array $candidates)
+    {
+        if (!$this->isSupported(self::PERM_ANALYST_DATA)) {
+            throw new RuntimeException("Remote server do not support analyst data");
+        }
+
+        return $this->post('/analyst_data/filterAnalystDataForPush', $candidates);
+    }
+
+    /**
+     * @param array $rules
+     * @return HttpSocketResponseExtended
+     * @throws HttpSocketHttpException
+     * @throws HttpSocketJsonException
+     */
+    public function fetchIndexMinimal(array $rules)
+    {
+        if (!$this->isSupported(self::PERM_ANALYST_DATA)) {
+            throw new RuntimeException("Remote server do not support analyst data");
+        }
+
+        return $this->post('/analyst_data/indexMinimal', $rules);
+    }
+
+    /**
+     * @param string $type
+     * @param array $uuids
+     * @return HttpSocketResponseExtended
+     * @throws HttpSocketJsonException
+     * @throws HttpSocketHttpException
+     */
+    public function fetchAnalystData($type, array $uuids)
+    {
+        if (!$this->isSupported(self::PERM_ANALYST_DATA)) {
+            throw new RuntimeException("Remote server do not support analyst data");
+        }
+
+        $params = [
+            'uuid' => $uuids,
+        ];
+
+        $url = '/analyst_data/index/' . $type;
+        $url .= $this->createParams($params);
+        $url .= '.json';
+        return $this->get($url);
+    }
+
+    /**
+     * @param string $type
+     * @param array $analystData
+     * @return HttpSocketResponseExtended
+     * @throws HttpSocketHttpException
+     * @throws HttpSocketJsonException
+     */
+    public function pushAnalystData($type, array $analystData)
+    {
+        $logMessage = "Pushing Analyst Data #{$analystData[$type]['uuid']} to Server #{$this->serverId()}";
+        return $this->post('/analyst_data/pushAnalystData', $analystData, $logMessage);
+    }
+
     /**
      * @param array $params
      * @return HttpSocketResponseExtended
@@ -230,19 +297,26 @@ class ServerSyncTool
 
     /**
      * @param array $eventUuids
+     * @param array $blockedOrgs Blocked organisation UUIDs
      * @return array
      * @throws HttpSocketHttpException
      * @throws HttpSocketJsonException
      * @throws JsonException
      */
-    public function fetchSightingsForEvents(array $eventUuids)
+    public function fetchSightingsForEvents(array $eventUuids, array $blockedOrgs = [])
     {
-        return $this->post('/sightings/restSearch/event', [
+        $postParams = [
             'returnFormat' => 'json',
             'last' => 0, // fetch all
             'includeUuid' => true,
             'uuid' => $eventUuids,
-        ])->json()['response'];
+        ];
+        if (!empty($blockedOrgs)) {
+            $postParams['org_id'] = array_map(function ($uuid) {
+                return "!$uuid";
+            }, $blockedOrgs);
+        }
+        return $this->post('/sightings/restSearch/event', $postParams)->json()['response'];
     }
 
     /**
@@ -414,6 +488,8 @@ class ServerSyncTool
                 return isset($info['perm_sync']) && $info['perm_sync'];
             case self::PERM_GALAXY_EDITOR:
                 return isset($info['perm_galaxy_editor']) && $info['perm_galaxy_editor'];
+            case self::PERM_ANALYST_DATA:
+                return isset($info['perm_analyst_data']) && $info['perm_analyst_data'];
             case self::FEATURE_SIGHTING_REST_SEARCH:
                 $version = explode('.', $info['version']);
                 return $version[0] == 2 && $version[1] == 4 && $version[2] > 164;
@@ -430,6 +506,16 @@ class ServerSyncTool
         return $this->socket->getMetaData();
     }
 
+    /**
+     * @param string $message
+     * @return void
+     */
+    public function debug($message)
+    {
+        $memoryUsage = round(memory_get_usage() / 1024 / 1024, 2);
+        CakeLog::debug("[Server sync #{$this->serverId()}]: $message. Memory: $memoryUsage MB");
+    }
+
     /**
      * @params string $url Relative URL
      * @return HttpSocketResponseExtended
@@ -480,6 +566,7 @@ class ServerSyncTool
 
         if ($etag) {
             // Remove compression marks that adds Apache for compressed content
+            // This can be removed in future as this is already checked by MISP itself since 2024-03
             $etagWithoutQuotes = trim($etag, '"');
             $dashPos = strrpos($etagWithoutQuotes, '-');
             if ($dashPos && in_array(substr($etagWithoutQuotes, $dashPos + 1), ['br', 'gzip'], true)) {

app/Lib/Tools/SyncTool.php

@@ -1,7 +1,6 @@
 <?php
 class SyncTool
 {
-
     const ALLOWED_CERT_FILE_EXTENSIONS = ['pem', 'crt'];
 
     /**
@@ -50,7 +49,7 @@ class SyncTool
      * @return HttpSocketExtended
      * @throws Exception
      */
-    public function createHttpSocket($params = array())
+    public function createHttpSocket(array $params = [])
     {
         // Use own CA PEM file
         $caPath = Configure::read('MISP.ca_path');
@@ -84,6 +83,9 @@ class SyncTool
         }
 
         if (function_exists('curl_init')) {
+            if (!isset($params['timeout']) && Configure::check('MISP.curl_request_timeout')) {
+                $params['timeout'] = (int)Configure::read('MISP.curl_request_timeout');
+            }
             App::uses('CurlClient', 'Tools');
             $HttpSocket = new CurlClient($params);
         } else {

app/Locale/ara/LC_MESSAGES/default.po

@@ -14679,7 +14679,7 @@ msgstr ""
 
 #: View/Events/automation.ctp:67
 #: View/Events/legacy_automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr ""
 
 #: View/Events/automation.ctp:68

app/Locale/cze/LC_MESSAGES/default.po

@@ -14651,7 +14651,7 @@ msgstr ""
 
 #: View/Events/automation.ctp:67
 #: View/Events/legacy_automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr ""
 
 #: View/Events/automation.ctp:68

app/Locale/dan/LC_MESSAGES/default.po

@@ -14624,7 +14624,7 @@ msgstr ""
 
 #: View/Events/automation.ctp:67
 #: View/Events/legacy_automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr ""
 
 #: View/Events/automation.ctp:68

app/Locale/default.pot

@@ -15934,7 +15934,7 @@ msgstr ""
 
 #: View/Events/automation.ctp:67
 #: View/Events/legacy_automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr ""
 
 #: View/Events/automation.ctp:68

app/Locale/deu/LC_MESSAGES/default.po

@@ -14622,7 +14622,7 @@ msgstr ""
 
 #: View/Events/automation.ctp:67
 #: View/Events/legacy_automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr ""
 
 #: View/Events/automation.ctp:68

app/Locale/fra/LC_MESSAGES/default.po

@@ -14626,7 +14626,7 @@ msgstr "Par défaut (0), tout les attributs qui correspondent aux autres paramè
 
 #: View/Events/automation.ctp:67
 #: View/Events/legacy_automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr "Si le paramètre est défini à 1, cela va retourner les attributs mis à la corbeille ainsi que les attributs actifs. En utilisant \"only\" en tant que paramètre, cela va seulement retourner les données mises à la corbeille."
 
 #: View/Events/automation.ctp:68

app/Locale/hun/LC_MESSAGES/default.po

@@ -14622,7 +14622,7 @@ msgstr ""
 
 #: View/Events/automation.ctp:67
 #: View/Events/legacy_automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr ""
 
 #: View/Events/automation.ctp:68

app/Locale/ita/LC_MESSAGES/default.po

@@ -14625,7 +14625,7 @@ msgstr "Per impostazione predefinita (0) tutti gli attributi restituiti rispondo
 
 #: View/Events/automation.ctp:67
 #: View/Events/legacy_automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr "Se questo parametro è impostato a 1, verranno restituiti attributi \"soft-deleted\" insieme a quelli attivi. Utilizzando \"only\" come parametro verranno restituiti solo gli attributi \"soft-deleted\"."
 
 #: View/Events/automation.ctp:68

app/Locale/jpn/LC_MESSAGES/default.po

@@ -14608,7 +14608,7 @@ msgstr "デフォルトの (0) では、to_ids の設定に関係なく、他の
 
 #: View/Events/automation.ctp:67
 #: View/Events/legacy_automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr "このパラメーターを 1 に設定すると、ソフト削除されたアトリビュートがアクティなアトリビュートと共に返されます。\"only\"をパラメーターとして使用すると、返されるデータはソフト削除されたデータのみに制限されます。"
 
 #: View/Events/automation.ctp:68

app/Locale/kor/LC_MESSAGES/default.po

@@ -14609,7 +14609,7 @@ msgstr ""
 
 #: View/Events/automation.ctp:67
 #: View/Events/legacy_automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr ""
 
 #: View/Events/automation.ctp:68

app/Locale/no/LC_MESSAGES/default.po

@@ -14625,7 +14625,7 @@ msgstr "Som standard (0) returneres alle attributter som samsvarer med de andre
 
 #: View/Events/automation.ctp:67
 #: View/Events/legacy_automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr "Hvis denne parameteren er satt til 1, vil den returnere myke slettede attributter sammen med aktive. Ved å bruke \"only\" som en parameter, vil det begrense det returnerte datasettet til bare slettede data."
 
 #: View/Events/automation.ctp:68

app/Locale/pol/LC_MESSAGES/default.po

@@ -14651,7 +14651,7 @@ msgstr ""
 
 #: View/Events/automation.ctp:67
 #: View/Events/legacy_automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr ""
 
 #: View/Events/automation.ctp:68

app/Locale/pt/LC_MESSAGES/default.po

@@ -7964,7 +7964,7 @@ msgstr ""
 
 #: View/Events/automation.ctp:42
 #: View/Events/legacy_automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr ""
 
 #: View/Events/automation.ctp:43

app/Locale/pt_BR/LC_MESSAGES/default.po

@@ -14623,7 +14623,7 @@ msgstr ""
 
 #: View/Events/automation.ctp:67
 #: View/Events/legacy_automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr ""
 
 #: View/Events/automation.ctp:68

app/Locale/ro/LC_MESSAGES/default.po

@@ -14636,7 +14636,7 @@ msgstr ""
 
 #: View/Events/automation.ctp:67
 #: View/Events/legacy_automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr ""
 
 #: View/Events/automation.ctp:68

app/Locale/rus/LC_MESSAGES/default.po

@@ -14650,7 +14650,7 @@ msgstr "По-умолчанию (а также при значении 0) в п
 
 #: View/Events/automation.ctp:67
 #: View/Events/legacy_automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr "По-умолчанию (а также при значении 0) в поиск попадают только активные атрибуты. Если параметр равен 1, то в поиск попадут дополнительно удаленные атрибуты. Если используется ключевое слово \"only\", то в результаты поиска попадут только удаленные атрибуты. "
 
 #: View/Events/automation.ctp:68

app/Locale/si-LK/LC_MESSAGES/default.po

@@ -14638,7 +14638,7 @@ msgstr "පෙරනිමියෙන් (0) to_ids සිටුවම් න
 
 #: View/Events/automation.ctp:67
 #: View/Events/legacy_automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr "මෙම පරාමිතිය 1 ලෙස සකසා ඇත්නම්, එය සක්‍රිය ඒවා සමඟ මෘදු-මකා දැමූ ගුණාංග ලබා දෙනු ඇත. පරාමිතියක් ලෙස \"පමණක්\" භාවිතා කිරීමෙන් එය ආපසු ලබා දෙන දත්ත කට්ටලය මෘදු-මකා දැමූ දත්ත වලට පමණක් සීමා කරයි."
 
 #: View/Events/automation.ctp:68

app/Locale/spa/LC_MESSAGES/default.po

@@ -14624,7 +14624,7 @@ msgstr ""
 
 #: View/Events/automation.ctp:67
 #: View/Events/legacy_automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr ""
 
 #: View/Events/automation.ctp:68

app/Locale/swe/LC_MESSAGES/default.po

@@ -10898,7 +10898,7 @@ msgstr ""
 
 #: View/Events/automation.ctp:52
 #: View/Events/legacy_automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr ""
 
 #: View/Events/automation.ctp:53

app/Locale/th_TH/LC_MESSAGES/default.po

@@ -14620,7 +14620,7 @@ msgstr ""
 
 #: View/Events/automation.ctp:67
 #: View/Events/legacy_automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr ""
 
 #: View/Events/automation.ctp:68

app/Locale/ukr/LC_MESSAGES/default.po

@@ -6030,7 +6030,7 @@ msgid "By default (0) all attributes are returned that match the other filter pa
 msgstr ""
 
 #: View/Events/automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr ""
 
 #: View/Events/automation.ctp:316

app/Locale/zh-s/LC_MESSAGES/default.po

@@ -14619,7 +14619,7 @@ msgstr "默认情况下(0), 返回所有与其他过滤器参数匹配的属性,
 
 #: View/Events/automation.ctp:67
 #: View/Events/legacy_automation.ctp:315
-msgid "If this parameter is set to 1, it will return soft-deleted attributes along with active ones. By using \"only\" as a parameter it will limit the returned data set to soft-deleted data only."
+msgid "Default value 0. If set to 1, only soft-deleted attributes will be returned. If set to [0,1] , both deleted and non-deleted attributes wil be returned."
 msgstr "如果这个参数被设置为1, 它将返回软删除的属性和活动属性. 如果使用\"only\"作为参数, 则返回的数据集将只限于软删除的数据."
 
 #: View/Events/automation.ctp:68

app/Model/AnalystDataBlocklist.php

@@ -0,0 +1,58 @@
+<?php
+App::uses('AppModel', 'Model');
+
+class AnalystDataBlocklist extends AppModel
+{
+    public $useTable = 'analyst_data_blocklists';
+
+    public $recursive = -1;
+
+    public $actsAs = array(
+        'AuditLog',
+        'SysLogLogable.SysLogLogable' => array( // TODO Audit, logable
+            'userModel' => 'User',
+            'userKey' => 'user_id',
+            'change' => 'full'
+        ),
+        'Containable',
+    );
+
+    public $blocklistFields = array('analyst_data_uuid', 'comment', 'analyst_data_info', 'analyst_data_orgc');
+    public $blocklistTarget = 'analyst_data';
+
+    public $validate = array(
+        'analyst_data_uuid' => array(
+            'unique' => array(
+                'rule' => 'isUnique',
+                'message' => 'Analyst Data already blocklisted.'
+            ),
+            'uuid' => array(
+                'rule' => array('uuid'),
+                'message' => 'Please provide a valid UUID'
+            ),
+        )
+    );
+
+    public function beforeValidate($options = array())
+    {
+        parent::beforeValidate();
+        if (empty($this->data['AnalystDataBlocklist']['id'])) {
+            $this->data['AnalystDataBlocklist']['date_created'] = date('Y-m-d H:i:s');
+        }
+        if (empty($this->data['AnalystDataBlocklist']['comment'])) {
+            $this->data['AnalystDataBlocklist']['comment'] = '';
+        }
+        return true;
+    }
+
+    /**
+     * @param string $analystDataUUID
+     * @return bool
+     */
+    public function checkIfBlocked($analystDataUUID)
+    {
+        return $this->hasAny([
+            'analyst_data_uuid' => $analystDataUUID,
+        ]);
+    }
+}

app/Model/AppModel.php

@@ -27,6 +27,7 @@ App::uses('FileAccessTool', 'Tools');
 App::uses('JsonTool', 'Tools');
 App::uses('RedisTool', 'Tools');
 App::uses('BetterCakeEventManager', 'Tools');
+App::uses('Folder', 'Utility');
 
 class AppModel extends Model
 {
@@ -47,6 +48,9 @@ class AppModel extends Model
     /** @var Workflow|null */
     private $Workflow;
 
+    public $includeAnalystData;
+    public $includeAnalystDataRecursive;
+
     // deprecated, use $db_changes
     // major -> minor -> hotfix -> requires_logout
     const OLD_DB_CHANGES = array(
@@ -86,7 +90,8 @@ class AppModel extends Model
         99 => false, 100 => false, 101 => false, 102 => false, 103 => false, 104 => false,
         105 => false, 106 => false, 107 => false, 108 => false, 109 => false, 110 => false,
         111 => false, 112 => false, 113 => true, 114 => false, 115 => false, 116 => false,
-        117 => false, 118 => false, 119 => false,
+        117 => false, 118 => false, 119 => false, 120 => false, 121 => false, 122 => false,
+        123 => false, 124 => false, 125 => false,
     );
 
     const ADVANCED_UPDATES_DESCRIPTION = array(
@@ -272,6 +277,9 @@ class AppModel extends Model
                 $this->removeDuplicatedUUIDs();
                 $dbUpdateSuccess = $this->updateDatabase('createUUIDsConstraints');
                 break;
+            case 120:
+                $dbUpdateSuccess = $this->moveImages();
+                break;
             default:
                 $dbUpdateSuccess = $this->updateDatabase($command);
                 break;
@@ -1982,7 +1990,7 @@ class AppModel extends Model
                 $sqlArray[] = "ALTER TABLE `event_reports` modify `content` mediumtext";
                 break;
             case 117:
-                $sqlArray[] = "CREATE TABLE `user_login_profiles` (
+                $sqlArray[] = "CREATE TABLE IF NOT EXISTS `user_login_profiles` (
                     `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
                     `created_at` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
                     `user_id` int(11) NOT NULL,
@@ -2009,6 +2017,168 @@ class AppModel extends Model
             case 119:
                 $sqlArray[] = "ALTER TABLE `access_logs` MODIFY `action` varchar(191) NOT NULL";
                 break;
+            case 121:
+                $sqlArray[] = "CREATE TABLE IF NOT EXISTS `notes` (
+                    `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
+                    `uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    `object_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    `object_type` varchar(80) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    `authors` text,
+                    `org_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    `orgc_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    `created` datetime NOT NULL,
+                    `modified` datetime NOT NULL,
+                    `distribution` tinyint(4) NOT NULL,
+                    `sharing_group_id` int(10) unsigned,
+                    `locked` tinyint(1) NOT NULL DEFAULT 0,
+                    `note` mediumtext,
+                    `language` varchar(16) DEFAULT 'en',
+                    PRIMARY KEY (`id`),
+                    UNIQUE KEY `uuid` (`uuid`),
+                    KEY `object_uuid` (`object_uuid`),
+                    KEY `object_type` (`object_type`),
+                    KEY `org_uuid` (`org_uuid`),
+                    KEY `orgc_uuid` (`orgc_uuid`),
+                    KEY `distribution` (`distribution`),
+                    KEY `sharing_group_id` (`sharing_group_id`)
+                  ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;";
+
+                $sqlArray[] = "CREATE TABLE IF NOT EXISTS `opinions` (
+                    `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
+                    `uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    `object_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    `object_type` varchar(80) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    `authors` text,
+                    `org_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    `orgc_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    `created` datetime NOT NULL,
+                    `modified` datetime NOT NULL,
+                    `distribution` tinyint(4) NOT NULL,
+                    `sharing_group_id` int(10) unsigned,
+                    `locked` tinyint(1) NOT NULL DEFAULT 0,
+                    `opinion` int(10) unsigned,
+                    `comment` text,
+                    PRIMARY KEY (`id`),
+                    UNIQUE KEY `uuid` (`uuid`),
+                    KEY `object_uuid` (`object_uuid`),
+                    KEY `object_type` (`object_type`),
+                    KEY `org_uuid` (`org_uuid`),
+                    KEY `orgc_uuid` (`orgc_uuid`),
+                    KEY `distribution` (`distribution`),
+                    KEY `sharing_group_id` (`sharing_group_id`),
+                    KEY `opinion` (`opinion`)
+                ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;";
+
+                $sqlArray[] = "CREATE TABLE IF NOT EXISTS `relationships` (
+                    `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
+                    `uuid` varchar(40) CHARACTER SET ascii NOT NULL,
+                    `object_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    `object_type` varchar(80) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    `authors` text,
+                    `org_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    `orgc_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    `created` datetime NOT NULL,
+                    `modified` datetime NOT NULL,
+                    `distribution` tinyint(4) NOT NULL,
+                    `sharing_group_id` int(10) unsigned,
+                    `locked` tinyint(1) NOT NULL DEFAULT 0,
+                    `relationship_type` varchar(255) CHARACTER SET utf8 COLLATE utf8_unicode_ci,
+                    `related_object_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    `related_object_type` varchar(80) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    PRIMARY KEY (`id`),
+                    UNIQUE KEY `uuid` (`uuid`),
+                    KEY `object_uuid` (`object_uuid`),
+                    KEY `object_type` (`object_type`),
+                    KEY `org_uuid` (`org_uuid`),
+                    KEY `orgc_uuid` (`orgc_uuid`),
+                    KEY `distribution` (`distribution`),
+                    KEY `sharing_group_id` (`sharing_group_id`),
+                    KEY `relationship_type` (`relationship_type`),
+                    KEY `related_object_uuid` (`related_object_uuid`),
+                    KEY `related_object_type` (`related_object_type`)
+                ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;";
+
+                $sqlArray[] = "CREATE TABLE IF NOT EXISTS `analyst_data_blocklists` (
+                    `id` int(11) NOT NULL AUTO_INCREMENT,
+                    `analyst_data_uuid` varchar(40) COLLATE utf8_bin NOT NULL,
+                    `created` datetime NOT NULL,
+                    `analyst_data_info` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci NOT NULL,
+                    `comment` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci,
+                    `analyst_data_orgc` VARCHAR(255) CHARACTER SET utf8 COLLATE utf8_bin NOT NULL,
+                    PRIMARY KEY (`id`),
+                    KEY `analyst_data_uuid` (`analyst_data_uuid`),
+                    KEY `analyst_data_orgc` (`analyst_data_orgc`)
+                ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;";
+
+                $sqlArray[] = "ALTER TABLE `roles` ADD `perm_analyst_data` tinyint(1) NOT NULL DEFAULT 0;";
+                $sqlArray[] = "UPDATE `roles` SET `perm_analyst_data`=1 WHERE `perm_add` = 1;";
+
+                $sqlArray[] = "ALTER TABLE `servers` ADD `push_analyst_data` tinyint(1) NOT NULL DEFAULT 0 AFTER `push_galaxy_clusters`;";
+                $sqlArray[] = "ALTER TABLE `servers` ADD `pull_analyst_data` tinyint(1) NOT NULL DEFAULT 0 AFTER `push_analyst_data`;";
+                break;
+            case 122:
+                $sqlArray[] = "CREATE TABLE IF NOT EXISTS `collections` (
+                    `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
+                    `uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    `org_id` int(10) unsigned NOT NULL,
+                    `orgc_id` int(10) unsigned NOT NULL,
+                    `user_id` int(10) unsigned NOT NULL,
+                    `created` datetime NOT NULL,
+                    `modified` datetime NOT NULL,
+                    `distribution` tinyint(4) NOT NULL,
+                    `sharing_group_id` int(10) unsigned,
+                    `name` varchar(191) NOT NULL,
+                    `type` varchar(80) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    `description` mediumtext,
+                    PRIMARY KEY (`id`),
+                    UNIQUE KEY `uuid` (`uuid`),
+                    KEY `name` (`name`),
+                    KEY `type` (`type`),
+                    KEY `org_id` (`org_id`),
+                    KEY `orgc_id` (`orgc_id`),
+                    KEY `user_id` (`user_id`),
+                    KEY `distribution` (`distribution`),
+                    KEY `sharing_group_id` (`sharing_group_id`)
+                    ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;";
+
+                $sqlArray[] = "CREATE TABLE IF NOT EXISTS `collection_elements` (
+                    `id` int(10) unsigned NOT NULL AUTO_INCREMENT,
+                    `uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    `element_uuid` varchar(40) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    `element_type` varchar(80) CHARACTER SET ascii COLLATE ascii_general_ci NOT NULL,
+                    `collection_id` int(10) unsigned NOT NULL,
+                    `description` text,
+                    PRIMARY KEY (`id`),
+                    UNIQUE KEY `uuid` (`uuid`),
+                    KEY `element_uuid` (`element_uuid`),
+                    KEY `element_type` (`element_type`),
+                    KEY `collection_id` (`collection_id`),
+                    UNIQUE KEY `unique_element` (`element_uuid`, `collection_id`)
+                ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;";
+                break;
+            case 123:
+                $sqlArray[] = 'ALTER TABLE `notes` MODIFY `created` datetime NOT NULL';
+                $sqlArray[] = 'ALTER TABLE `opinions` MODIFY `created` datetime NOT NULL;';
+                $sqlArray[] = 'ALTER TABLE `relationships` MODIFY `created` datetime NOT NULL;';
+                $sqlArray[] = 'ALTER TABLE `notes` MODIFY `modified` datetime NOT NULL;';
+                $sqlArray[] = 'ALTER TABLE `opinions` MODIFY `modified` datetime NOT NULL;';
+                $sqlArray[] = 'ALTER TABLE `relationships` MODIFY `modified` datetime NOT NULL;';
+                break;
+            case 124:
+                $sqlArray[] = 'CREATE TABLE IF NOT EXISTS `sighting_blocklists` (
+                    `id` int(11) NOT NULL AUTO_INCREMENT,
+                    `org_uuid` varchar(40) COLLATE utf8_bin NOT NULL,
+                    `created` datetime NOT NULL,
+                    `org_name` varchar(255) COLLATE utf8_bin NOT NULL,
+                    `comment` TEXT CHARACTER SET utf8 COLLATE utf8_unicode_ci,
+                    PRIMARY KEY (`id`),
+                    INDEX `org_uuid` (`org_uuid`),
+                    INDEX `org_name` (`org_name`)
+                  ) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;';
+                break;
+            case 125:
+                $sqlArray[] = "ALTER TABLE `feeds` ADD COLUMN `tag_collection_id` INT(11) NOT NULL DEFAULT 0;";
+                break;
             case 'fixNonEmptySharingGroupID':
                 $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
                 $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
@@ -3658,7 +3828,7 @@ class AppModel extends Model
     protected function logException($message, Exception $exception, $type = LOG_ERR)
     {
         // If Sentry is installed, send exception to Sentry
-        if (function_exists('\Sentry\captureException') && $type === LOG_ERR) {
+        if (function_exists('\Sentry\captureException') && $type <= LOG_ERR) {
             \Sentry\captureException(new Exception($message, $type, $exception));
         }
 
@@ -4047,8 +4217,17 @@ class AppModel extends Model
         if (!empty($query['order']) && $this->validOrderClause($query['order']) === false) {
             throw new InvalidArgumentException('Invalid order clause');
         }
-
-        return parent::find($type, $query);
+        $results = parent::find($type, $query);
+        if (!empty($query['includeAnalystData']) && $this->Behaviors->enabled('AnalystDataParent')) {
+            if ($type === 'first') {
+                $results[$this->alias] = array_merge($results[$this->alias], $this->attachAnalystData($results[$this->alias]));
+            } else if ($type === 'all') {
+                foreach ($results as $k => $result) {
+                    $results[$k][$this->alias] = array_merge($results[$k][$this->alias], $this->attachAnalystData($results[$k][$this->alias]));
+                }
+            }
+        }
+        return $results;
     }
 
     private function validOrderClause($order)
@@ -4076,4 +4255,31 @@ class AppModel extends Model
     {
         return preg_match('/^[\w\_\-\. ]+$/', $param);
     }
+
+    public function moveImages()
+    {
+        $oldImageDir = APP . 'webroot/img';
+        $newImageDir = APP . 'files/img';
+        $oldOrgDir = new Folder($oldImageDir . '/orgs');
+        $oldCustomDir = new Folder($oldImageDir . '/custom');
+        $result = $oldOrgDir->copy([
+            'from' => $oldImageDir . '/orgs',
+            'to' => $newImageDir . '/orgs',
+            'scheme' => Folder::OVERWRITE,
+            'recursive' => true
+        ]);
+        if ($result) {
+            $oldOrgDir->delete();
+        }
+        $result = $oldCustomDir->copy([
+            'from' => $oldImageDir . '/custom',
+            'to' => $newImageDir . '/custom',
+            'scheme' => Folder::OVERWRITE,
+            'recursive' => true
+        ]);
+        if ($result) {
+            $oldCustomDir->delete();
+        }
+        return true;
+    }
 }

app/Model/AttachmentScan.php

@@ -258,7 +258,7 @@ class AttachmentScan extends AppModel
                     $scanned++;
                 }
             } catch (Exception $e) {
-                $this->logException("Could not scan attachment for $type {$attribute['Attribute']['id']}", $e, LOG_WARNING);
+                $this->logException("Could not scan attachment for $type {$attribute[$type]['id']}", $e, LOG_WARNING);
                 $fails++;
             }
 

app/Model/Attribute.php

@@ -36,7 +36,8 @@ class Attribute extends AppModel
         'Trim',
         'Containable',
         'Regexp' => array('fields' => array('value')),
-        'LightPaginator'
+        'LightPaginator',
+        'AnalystDataParent',
     );
 
     public $displayField = 'value';
@@ -117,6 +118,7 @@ class Attribute extends AppModel
         'aba-rtn',
         'gender',
         'counter',
+        'integer',
         'float',
         'port',
         'nationality',
@@ -968,7 +970,7 @@ class Attribute extends AppModel
         $maxWidth = $maxWidth ?: $defaultMaxSize;
         $maxHeight = $maxHeight ?: $defaultMaxSize;
         $suffix = null;
-        
+
         if ($maxWidth == $defaultMaxSize && $maxHeight == $defaultMaxSize) {
             $thumbnailInRedis = Configure::read('MISP.thumbnail_in_redis');
             if ($thumbnailInRedis) {
@@ -1140,29 +1142,36 @@ class Attribute extends AppModel
         }
         $temp = array();
         if (!empty($tagArray[1])) {
-            if ($options['scope'] == 'all' || $options['scope'] == 'Event') {
-                $subquery_options = array(
-                    'conditions' => array(
-                        'tag_id' => $tagArray[1]
-                    ),
-                    'fields' => array(
-                        'event_id'
-                    )
-                );
-                $lookup_field = ($options['scope'] === 'Event') ? 'Event.id' : 'Attribute.event_id';
-                $conditions['AND'][] = array_merge($temp, $this->subQueryGenerator($tag->EventTag, $subquery_options, $lookup_field, 1));
-            }
-            if ($options['scope'] == 'all' || $options['scope'] == 'Attribute') {
-                $subquery_options = array(
-                    'conditions' => array(
-                        'tag_id' => $tagArray[1]
-                    ),
-                    'fields' => array(
-                        $options['scope'] === 'Event' ? 'event.id' : 'attribute_id'
-                    )
-                );
-                $lookup_field = $options['scope'] === 'Event' ? 'Event.id' : 'Attribute.id';
-                $conditions['AND'][] = array_merge($temp, $this->subQueryGenerator($tag->AttributeTag, $subquery_options, $lookup_field, 1));
+            /* 
+             * If we didn't find the given negation tag, no need to use the -1 trick,
+             * it is basically a hack to block the search from finding anything if no positive lookup was valid.
+             * However, if none of the negated tags exist, there's nothing to filter here
+             */
+            if (count($tagArray[1]) !== 1 || $tagArray[1][0] != -1) {
+                if ($options['scope'] == 'all' || $options['scope'] == 'Event') {
+                    $subquery_options = array(
+                        'conditions' => array(
+                            'tag_id' => $tagArray[1]
+                        ),
+                        'fields' => array(
+                            'event_id'
+                        )
+                    );
+                    $lookup_field = ($options['scope'] === 'Event') ? 'Event.id' : 'Attribute.event_id';
+                    $conditions['AND'][] = array_merge($temp, $this->subQueryGenerator($tag->EventTag, $subquery_options, $lookup_field, 1));
+                }
+                if ($options['scope'] == 'all' || $options['scope'] == 'Attribute') {
+                    $subquery_options = array(
+                        'conditions' => array(
+                            'tag_id' => $tagArray[1]
+                        ),
+                        'fields' => array(
+                            $options['scope'] === 'Event' ? 'event.id' : 'attribute_id'
+                        )
+                    );
+                    $lookup_field = $options['scope'] === 'Event' ? 'Event.id' : 'Attribute.id';
+                    $conditions['AND'][] = array_merge($temp, $this->subQueryGenerator($tag->AttributeTag, $subquery_options, $lookup_field, 1));
+                }
             }
         }
         $temp = array();
@@ -1547,33 +1556,57 @@ class Attribute extends AppModel
         $conditions = array();
         if (!$user['Role']['perm_site_admin']) {
             $sgids = $this->SharingGroup->authorizedIds($user);
-            $eventConditions = $this->Event->createEventConditions($user);
-            $conditions = array(
-                'AND' => array(
-                    $eventConditions['AND'],
-                    array(
-                        'OR' => array(
-                            'Event.org_id' => $user['org_id'],
-                            'Attribute.distribution' => array(1, 2, 3, 5),
-                            'AND '=> array(
-                                'Attribute.distribution' => 4,
-                                'Attribute.sharing_group_id' => $sgids,
-                            )
-                        )
-                    ),
-                    array(
-                        'OR' => array(
-                            'Attribute.object_id' => 0,
-                            'Event.org_id' => $user['org_id'],
-                            'Object.distribution' => array(1, 2, 3, 5),
-                            'AND' => array(
-                                'Object.distribution' => 4,
-                                'Object.sharing_group_id' => $sgids,
-                            )
-                        )
-                    )
-                )
-            );
+            $subQuery1 = [
+                'conditions' => ['org_id' => $user['org_id']],
+                'fields' => ['id']
+            ];
+            $subQuery2 = [
+                'conditions' => [
+                    'distribution IN' => [1, 2, 3]
+                ],
+                'fields' => ['id']
+            ];
+            $subQuery3 = [
+                'conditions' => [
+                    'Event.distribution' => 4,
+                    'Event.sharing_group_id IN' => $sgids
+                ],
+                'fields' => ['id']
+            ];
+            if (Configure::read('MISP.unpublishedprivate')) {
+                $subQuery2['conditions']['Event.published'] = 1;
+                $subQuery3['conditions']['Event.published'] = 1;
+            }
+            $conditions = [
+                'OR' => [
+                    $this->subQueryGenerator($this->Event, $subQuery1, 'Attribute.event_id'),
+                    'AND' => [
+                        'OR' => [
+                            $this->subQueryGenerator($this->Event, $subQuery2, 'Attribute.event_id'),
+                            $this->subQueryGenerator($this->Event, $subQuery3, 'Attribute.event_id')
+                        ],
+                        [
+                            'OR' => [
+                                'Attribute.distribution' => [1, 2, 3, 5],
+                                'AND '=> [
+                                    'Attribute.distribution' => 4,
+                                    'Attribute.sharing_group_id' => $sgids,
+                                ]
+                            ]
+                        ],
+                        [
+                            'OR' => [
+                                'Attribute.object_id' => 0,
+                                'Object.distribution' => [1, 2, 3, 5],
+                                'AND' => [
+                                    'Object.distribution' => 4,
+                                    'Object.sharing_group_id' => $sgids,
+                                ]
+                            ]
+                        ]
+                    ]
+                ]
+            ];
         }
         return $conditions;
     }
@@ -1786,7 +1819,7 @@ class Attribute extends AppModel
         if (isset($options['fields'])) {
             $params['fields'] = $options['fields'];
         }
-        if (isset($options['conditions'])) {
+        if (!empty($options['conditions'])) {
             $params['conditions']['AND'][] = $options['conditions'];
         }
         if (empty($options['flatten'])) {
@@ -1882,11 +1915,10 @@ class Attribute extends AppModel
                 return [];
             }
         }
-
         $eventTags = []; // tag cache
         $attributes = [];
+        $params['ignoreIndexHint'] = 'deleted';
         do {
-            $continue = true;
             $results = $this->find('all', $params);
             if (empty($results)) {
                 break;
@@ -2402,11 +2434,15 @@ class Attribute extends AppModel
                 $timestamp[0] = $timestamp[1];
                 $timestamp[1] = $temp;
             }
-            $conditions['AND'][] = array($scope . ' >=' => $timestamp[0]);
+            if ($timestamp[0] != 0) {
+                $conditions['AND'][] = array($scope . ' >=' => $timestamp[0]);
+            }
             $conditions['AND'][] = array($scope . ' <=' => $timestamp[1]);
         } else {
             $timestamp = $this->resolveTimeDelta($timestamp);
-            $conditions['AND'][] = array($scope . ' >=' => $timestamp);
+            if ($timestamp !== 0) {
+                $conditions['AND'][] = array($scope . ' >=' => $timestamp);
+            }
         }
         if ($returnRaw) {
             return $timestamp;
@@ -2428,7 +2464,7 @@ class Attribute extends AppModel
             $conditions['AND'][] = array($scope . ' <=' => $timestamp[1]);
         } else {
             $timestamp = intval($this->resolveTimeDelta($timestamp)) * 1000000; // seen in stored in micro-seconds in the DB
-            if ($scope == 'Attribute.first_seen') {
+            if ($scope == 'Attribute.first_seen' || $scope == 'Object.first_seen') {
                 $conditions['AND'][] = array($scope . ' >=' => $timestamp);
             } else {
                 $conditions['AND'][] = array($scope . ' <=' => $timestamp);
@@ -2616,7 +2652,8 @@ class Attribute extends AppModel
                 $this->id = $attribute['id'];
             }
         }
-        if (!$this->save(['Attribute' => $attribute], $params)) {
+        $savedAttribute = $this->save(['Attribute' => $attribute], $params);
+        if (!$savedAttribute) {
             $this->logDropped($user, $attribute);
         } else {
             if (!empty($attribute['AttributeTag'])) {
@@ -2654,6 +2691,7 @@ class Attribute extends AppModel
             if (!empty($attribute['Sighting'])) {
                 $this->Sighting->captureSightings($attribute['Sighting'], $this->id, $eventId, $user);
             }
+            $this->Event->captureAnalystData($user, $attribute, 'Attribute', $savedAttribute['Attribute']['uuid']);
         }
         if (!empty($this->validationErrors)) {
             $validationErrors = $this->validationErrors;
@@ -2794,6 +2832,7 @@ class Attribute extends AppModel
             if (!empty($attribute['Sighting'])) {
                 $this->Sighting->captureSightings($attribute['Sighting'], $attributeId, $eventId, $user);
             }
+            $this->Event->captureAnalystData($user, $attribute, 'Attribute', $attribute['uuid']);
             if ($user['Role']['perm_tagger']) {
                 /*
                     We should unwrap the line below and remove the server option in the future once we have tag soft-delete
@@ -2858,7 +2897,7 @@ class Attribute extends AppModel
             if (!empty($tagActions['attach'])) {
                 $this->AttributeTag->saveMany($tagActions['attach']);
             }
-            
+
         }
         if (!empty($tagActions['detach'])) {
             foreach ($tagActions['detach'] as $detach) {
@@ -2876,7 +2915,7 @@ class Attribute extends AppModel
                     $this->Correlation->generateCorrelation(false, $event['Event']['id'], $attributeId);
                 }
             }
-            // Instead of incrementing / decrementing the event 
+            // Instead of incrementing / decrementing the event
             $attribute_count = $this->find('count', [
                 'conditions' => [
                     'Attribute.event_id' => $event['Event']['id'],
@@ -2893,7 +2932,7 @@ class Attribute extends AppModel
         }
         return true;
     }
-    
+
 
     /**
      * @param int $id Attribute ID
@@ -2973,9 +3012,15 @@ class Attribute extends AppModel
         return $adata;
     }
 
-    public function buildFilterConditions(array $user, array &$params)
+    public function buildFilterConditions(array $user, array &$params, $skipBuildConditions = false)
     {
-        $conditions = $this->buildConditions($user);
+        // in some cases we'll build the user ACL conditions elsewhere,
+        // for example when calling this function via restsearch
+        if ($skipBuildConditions) {
+            $conditions = [];
+        } else {
+            $conditions = $this->buildConditions($user);
+        }
         if (isset($params['wildcard'])) {
             $temp = array();
             $options = array(
@@ -3099,7 +3144,7 @@ class Attribute extends AppModel
         $subqueryElements = $this->Event->harvestSubqueryElements($filters);
         $filters = $this->Event->addFiltersFromSubqueryElements($filters, $subqueryElements, $user);
         $filters = $this->Event->addFiltersFromUserSettings($user, $filters);
-        $conditions = $this->buildFilterConditions($user, $filters);
+        $conditions = $this->buildFilterConditions($user, $filters, true);
         $params = array(
             'conditions' => $conditions,
             'fields' => array('Attribute.*', 'Event.org_id', 'Event.distribution'),
@@ -3119,6 +3164,7 @@ class Attribute extends AppModel
             'includeFullModel' => !empty($filters['includeFullModel']) ? $filters['includeFullModel'] : 0,
             'allow_proposal_blocking' => !empty($filters['allow_proposal_blocking']) ? $filters['allow_proposal_blocking'] : 0
         );
+
         if (!empty($filters['attackGalaxy'])) {
             $params['attackGalaxy'] = $filters['attackGalaxy'];
         }
@@ -3344,20 +3390,60 @@ class Attribute extends AppModel
         if (!empty($params['uuid'])) {
             $params['uuid'] = $this->convert_filters($params['uuid']);
             if (!empty($params['uuid']['OR'])) {
-                $conditions['AND'][] = array(
-                    'OR' => array(
-                        'Event.uuid' => $params['uuid']['OR'],
-                        'Attribute.uuid' => $params['uuid']['OR']
-                    )
-                );
+                if ($options['scope'] == 'Attribute') {
+                    $subQuery = [
+                        'conditions' => ['uuid' => $params['uuid']['OR']],
+                        'fields' => ['id']
+                    ];
+                    $pre_lookup = $this->Event->find('first', [
+                        'conditions' => ['Event.uuid' => $params['uuid']['OR']],
+                        'recursive' => -1,
+                        'fields' => ['Event.id']
+                    ]);
+                    if (empty($pre_lookup)) {
+                        $conditions['AND'][] = array(
+                            'OR' => array(
+                                'Attribute.uuid' => $params['uuid']['OR']
+                            )
+                        );
+                    } else {
+                        $conditions['AND'][] = array(
+                            'OR' => array(
+                                $this->subQueryGenerator($this->Event, $subQuery, 'Attribute.event_id'),
+                                'Attribute.uuid' => $params['uuid']['OR']
+                            )
+                        );
+                    }
+                    
+                } else {
+                    $conditions['AND'][] = array(
+                        'OR' => array(
+                            'Event.uuid' => $params['uuid']['OR'],
+                            'Attribute.uuid' => $params['uuid']['OR']
+                        )
+                    );
+                }
             }
             if (!empty($params['uuid']['NOT'])) {
-                $conditions['AND'][] = array(
-                    'NOT' => array(
-                        'Event.uuid' => $params['uuid']['NOT'],
-                        'Attribute.uuid' =>  $params['uuid']['NOT']
-                    )
-                );
+                if ($options['scope'] == 'Attribute') {
+                    $subQuery = [
+                        'conditions' => ['uuid' => $params['uuid']['OR']],
+                        'fields' => ['id']
+                    ];
+                    $conditions['AND'][] = [
+                        'NOT' => [
+                            $this->subQueryGenerator($this->Event, $subQuery, 'Attribute.event_id'),
+                            'Attribute.uuid' =>  $params['uuid']['NOT']
+                        ]
+                    ];
+                } else {
+                    $conditions['AND'][] = array(
+                        'NOT' => array(
+                            'Event.uuid' => $params['uuid']['NOT'],
+                            'Attribute.uuid' =>  $params['uuid']['NOT']
+                        )
+                    );
+                }
             }
         }
         return $conditions;
@@ -3539,7 +3625,7 @@ class Attribute extends AppModel
             ),
             'Other' => array(
                 'desc' => __('Attributes that are not part of any other category or are meant to be used as a component in MISP objects in the future'),
-                'types' => array('comment', 'text', 'other', 'size-in-bytes', 'counter', 'datetime', 'cpe', 'port', 'float', 'hex', 'phone-number', 'boolean', 'anonymised', 'pgp-public-key', 'pgp-private-key')
+                'types' => array('comment', 'text', 'other', 'size-in-bytes', 'counter', 'integer', 'datetime', 'cpe', 'port', 'float', 'hex', 'phone-number', 'boolean', 'anonymised', 'pgp-public-key', 'pgp-private-key')
             )
         );
     }
@@ -3692,6 +3778,7 @@ class Attribute extends AppModel
             'dns-soa-email' => array('desc' => __('RFC 1035 mandates that DNS zones should have a SOA (Statement Of Authority) record that contains an email address where a PoC for the domain could be contacted. This can sometimes be used for attribution/linkage between different domains even if protected by whois privacy'), 'default_category' => 'Attribution', 'to_ids' => 0),
             'size-in-bytes' => array('desc' => __('Size expressed in bytes'), 'default_category' => 'Other', 'to_ids' => 0),
             'counter' => array('desc' => __('An integer counter, generally to be used in objects'), 'default_category' => 'Other', 'to_ids' => 0),
+            'integer' => array('desc' => __('A generic integer generally to be used in objects'), 'default_category' => 'Other', 'to_ids' => 0),
             'datetime' => array('desc' => __('Datetime in the ISO 8601 format'), 'default_category' => 'Other', 'to_ids' => 0),
             'port' => array('desc' => __('Port number'), 'default_category' => 'Network activity', 'to_ids' => 0),
             'ip-dst|port' => array('desc' => __('IP destination and port number separated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1),
@@ -3777,7 +3864,7 @@ class Attribute extends AppModel
         if (isset($attribute['id'])) {
             $conditions['Attribute.id !='] = $attribute['id'];
         }
-        
+
         return $this->find('first', [
             'recursive' => -1,
             'conditions' => $conditions,

app/Model/Behavior/AnalystDataBehavior.php

@@ -0,0 +1,88 @@
+<?php
+
+/**
+ * Common functions for the 3 analyst objects
+ */
+class AnalystDataBehavior extends ModelBehavior
+{
+    public $SharingGroup;
+
+    private $__current_type = null;
+
+    protected $__valid_sharing_groups = null;
+
+    public function setup(Model $Model, $settings = array()) {
+        // We want to know whether we're a Note, Opinion or Relationship
+        $this->__current_type = $Model->alias;
+    }
+
+    // Return the analystData of the current type for a given UUID (this only checks the ACL of the analystData, NOT of the parent.)
+    public function fetchForUuid(Model $Model, $uuid, $user = null)
+    {
+        $conditions = [
+            'object_uuid' => $uuid
+        ];
+        $type = $Model->current_type;
+        if (empty($user['Role']['perm_site_admin'])) {
+            if ($this->__valid_sharing_groups === null) {
+                $this->__valid_sharing_groups = $Model->SharingGroup->authorizedIds($user, true);
+            }
+            $conditions['AND'][] = [
+                'OR' => [
+                    $type . '.orgc_uuid' => $user['Organisation']['uuid'],
+                    $type . '.org_uuid' => $user['Organisation']['uuid'],
+                    $type . '.distribution IN' => [1, 2, 3],
+                    'AND' => [
+                        $type . '.distribution' => 4,
+                        $type . '.sharing_group_id IN' => $this->__valid_sharing_groups
+                    ]
+                ]
+            ];
+        }
+        return $Model->find('all', [
+            'recursive' => -1,
+            'conditions' => $conditions,
+            'contain' => ['Org', 'Orgc', 'SharingGroup'],
+        ]);
+    }
+
+        // Return the analystData of the current type for a given UUID (this only checks the ACL of the analystData, NOT of the parent.)
+        public function fetchForUuids(Model $Model, $uuids, $user = null)
+        {
+            $conditions = [
+                'object_uuid' => $uuids
+            ];
+            $type = $Model->current_type;
+            if (empty($user['Role']['perm_site_admin'])) {
+                if ($this->__valid_sharing_groups === null) {
+                    $this->__valid_sharing_groups = $Model->SharingGroup->authorizedIds($user, true);
+                }
+                $conditions['AND'][] = [
+                    'OR' => [
+                        $type . '.orgc_uuid' => $user['Organisation']['uuid'],
+                        $type . '.org_uuid' => $user['Organisation']['uuid'],
+                        $type . '.distribution IN' => [1, 2, 3],
+                        'AND' => [
+                            $type . '.distribution' => 4,
+                            $type . '.sharing_group_id IN' => $this->__valid_sharing_groups
+                        ]
+                    ]
+                ];
+            }
+            $temp = $Model->find('all', [
+                'recursive' => -1,
+                'conditions' => $conditions,
+                'contain' => ['Org', 'Orgc', 'SharingGroup'],
+            ]);
+            $results = [];
+            foreach ($temp as $result) {
+                $results[$result[$type]['object_uuid']][$type][] = $result[$type];
+            }
+            return $results;
+        }
+
+    public function checkACL()
+    {
+
+    }
+}

app/Model/Behavior/AnalystDataParentBehavior.php

@@ -0,0 +1,119 @@
+<?php
+
+/**
+ * Common functions for the 3 analyst objects
+ */
+class AnalystDataParentBehavior extends ModelBehavior
+{
+    private $__currentUser = null;
+
+    public $User;
+
+
+
+    public function attachAnalystData(Model $model, array $object, array $types = ['Note', 'Opinion', 'Relationship'])
+    {
+        // No uuid, nothing to attach
+        if (empty($object['uuid'])) {
+            return $object;
+        }
+        if (empty($this->__currentUser)) {
+            $user_id = Configure::read('CurrentUserId');
+            $this->User = ClassRegistry::init('User');
+            if ($user_id) {
+                $this->__currentUser = $this->User->getAuthUser($user_id);
+            }
+        }
+        $data = [];
+        foreach ($types as $type) {
+            $this->{$type} = ClassRegistry::init($type);
+            $this->{$type}->fetchRecursive = !empty($model->includeAnalystDataRecursive);
+            $temp = $this->{$type}->fetchForUuid($object['uuid'], $this->__currentUser);
+            if (!empty($temp)) {
+                foreach ($temp as $k => $temp_element) {
+                    if (in_array($type, ['Note', 'Opinion', 'Relationship'])) {
+                        $temp_element[$type] = $this->{$type}->fetchChildNotesAndOpinions($this->__currentUser, $temp_element[$type], 1);
+                    }
+                    $data[$type][] = $temp_element[$type];
+                }
+            }
+        }
+
+        // include inbound relationship
+        $data['RelationshipInbound'] = Hash::extract($this->Relationship->getInboundRelationships($this->__currentUser, $model->alias, $object['uuid']), '{n}.Relationship');
+        return $data;
+    }
+
+    public function fetchAnalystDataBulk(Model $model, array $uuids, array $types = ['Note', 'Opinion', 'Relationship']) {
+        $uuids = array_chunk($uuids, 100000);
+        if (empty($this->__currentUser)) {
+            $user_id = Configure::read('CurrentUserId');
+            $this->User = ClassRegistry::init('User');
+            if ($user_id) {
+                $this->__currentUser = $this->User->getAuthUser($user_id);
+            }
+        }
+        $results = [];
+        foreach ($uuids as $uuid_chunk) {
+            foreach ($types as $type) {
+                $this->{$type} = ClassRegistry::init($type);
+                $this->{$type}->fetchRecursive = !empty($model->includeAnalystDataRecursive);
+                $temp = $this->{$type}->fetchForUuids($uuid_chunk, $this->__currentUser);
+                $results = array_merge_recursive($results, $temp);
+            }
+        }
+        return $results;
+    }
+
+    public function attachAnalystDataBulk(Model $model, array $objects, array $types = ['Note', 'Opinion', 'Relationship'])
+    {
+        $uuids = [];
+        $objects = array_chunk($objects, 100000, true);
+        if (empty($this->__currentUser)) {
+            $user_id = Configure::read('CurrentUserId');
+            $this->User = ClassRegistry::init('User');
+            if ($user_id) {
+                $this->__currentUser = $this->User->getAuthUser($user_id);
+            }
+        }
+        foreach ($objects as $chunk => $chunked_objects) {
+            foreach ($chunked_objects as $k => $object) {
+                if (!empty($object['uuid'])) {
+                    $uuids[] = $object['uuid'];
+                }
+            }
+            // No uuids, nothing to attach
+            if (empty($uuids)) {
+                continue;
+            }
+            foreach ($types as $type) {
+                $this->{$type} = ClassRegistry::init($type);
+                $this->{$type}->fetchRecursive = !empty($model->includeAnalystDataRecursive);
+                $temp = $this->{$type}->fetchForUuids($uuids, $this->__currentUser);
+                if (!empty($temp)) {
+                    foreach ($chunked_objects as $k => $object) {
+                        if (!empty($temp[$object['uuid']])) {
+                            $objects[$chunk][$k][$type] = !empty($objects[$chunk][$k][$type]) ? $objects[$chunk][$k][$type] : [];
+                            $objects[$chunk][$k][$type] = array_merge($objects[$chunk][$k][$type], $temp[$object['uuid']][$type]);
+                        }
+                    }
+                }
+            }
+        }
+        $objects = call_user_func_array('array_merge', $objects);
+        return $objects;
+    }
+
+    public function afterFind(Model $model, $results, $primary = false)
+    {
+        if (!empty($model->includeAnalystData)) {
+            foreach ($results as $k => $item) {
+                if (isset($item[$model->alias])) {
+                    $results[$k] = array_merge($results[$k], $this->attachAnalystData($model, $item[$model->alias]));
+                }
+            }
+        }
+        return $results;
+    }
+
+}

app/Model/Benchmark.php

@@ -0,0 +1,6 @@
+<?php
+App::uses('AppModel', 'Model');
+
+class Benchmark extends AppModel
+{
+}

app/Model/Collection.php

@@ -0,0 +1,145 @@
+<?php
+App::uses('AppModel', 'Model');
+
+class Collection extends AppModel
+{
+
+    public $recursive = -1;
+
+    public $actsAs = array(
+            'Containable'
+    );
+
+    public $belongsTo = [
+        'Orgc' => array(
+            'className' => 'Organisation',
+            'foreignKey' => 'orgc_id',
+            'fields' => [
+                'Orgc.id',
+                'Orgc.uuid',
+                'Orgc.name'
+            ]
+        ),
+        'Org' => array(
+            'className' => 'Organisation',
+            'foreignKey' => 'org_id',
+            'fields' => [
+                'Org.id',
+                'Org.uuid',
+                'Org.name'
+            ]
+        ),
+        'User' => array(
+            'className' => 'User',
+            'foreignKey' => 'user_id',
+            'fields' => [
+                'User.id',
+                'User.email'
+            ]
+        )
+    ];
+
+    public $hasMany = [
+        'CollectionElement' => [
+            'dependent' => true
+        ]
+    ];
+ 
+    public $valid_targets = [
+        'Attribute',
+        'Event',
+        'GalaxyCluster',
+        'Galaxy',
+        'Object',
+        'Note',
+        'Opinion',
+        'Relationship',
+        'Organisation',
+        'SharingGroup'
+    ];
+
+    public $current_user = null;
+
+
+    public function beforeValidate($options = array())
+    {
+        if (empty($this->data['Collection'])) {
+            $this->data = ['Collection' => $this->data];
+        }
+        if (empty($this->id) && empty($this->data['Collection']['uuid'])) {
+            $this->data['Collection']['uuid'] = CakeText::uuid();
+        }
+        if (empty($this->id)) {
+            $this->data['Collection']['user_id'] = $this->current_user['id'];
+            if (empty($this->data['Collection']['orgc_id']) || empty($this->current_user['Role']['perm_sync'])) {
+                $this->data['Collection']['orgc_id'] = $this->current_user['Organisation']['id'];
+            }
+            $this->data['Collection']['org_id'] = $this->current_user['Organisation']['id'];
+            $this->data['Collection']['user_id'] = $this->current_user['id'];
+        }
+        return true;
+    }
+
+    public function mayModify($user_id, $collection_id)
+    {
+        $user = $this->User->getAuthUser($user_id);
+        $collection = $this->find('first', [
+            'recursive' => -1,
+            'conditions' => ['Collection.id' => $collection_id]
+        ]);
+        if ($user['Role']['perm_site_admin']) {
+            return true;
+        }
+        if (empty($user['Role']['perm_modify'])) {
+            return false;
+        }
+        if (!empty($user['Role']['perm_modify_org'])) {
+            if ($user['org_id'] == $collection['Collection']['Orgc_id']) {
+                return true;
+            }
+            if ($user['Role']['perm_sync'] && $user['org_id'] == $collection['Collection']['Org_id']) {
+                return true;
+            }            
+        }
+        if (!empty($user['Role']['perm_modify']) && $user['id'] === $collection['Collection']['user_id']) {
+        }
+    }
+
+    public function mayView($user_id, $collection_id)
+    {
+        $user = $this->User->getAuthUser($user_id);
+        $collection = $this->find('first', [
+            'recursive' => -1,
+            'conditions' => ['Collection.id' => $collection_id]
+        ]);
+        if ($user['Role']['perm_site_admin']) {
+            return true;
+        }
+        if ($collection['Collection']['org_id'] == $user('org_id')) {
+            return true;
+        }
+        if (in_array($collection['Collection']['distribution'], [1,2,3])) {
+            return true;
+        }
+        if ($collection['Collection']['distribution'] === 4) {
+            $SharingGroup = ClassRegistry::init('SharingGroup');
+            $sgs = $this->SharingGroup->fetchAllAuthorised($user, 'uuid');
+            if (isset($sgs[$collection['Collection']['sharing_group_id']])) {
+                return true;
+            } else {
+                return false;
+            }
+        }
+        return false;
+    }
+
+    public function rearrangeCollection(array $collection) {
+        foreach ($collection as $key => $elements) {
+            if ($key !== 'Collection') {
+                $collection['Collection'][$key] = $elements;
+                unset($collection[$key]);
+            }
+        }
+        return $collection;
+    }
+}

app/Model/CollectionElement.php

@@ -0,0 +1,205 @@
+<?php
+App::uses('AppModel', 'Model');
+
+class CollectionElement extends AppModel
+{
+
+    public $recursive = -1;
+
+    public $actsAs = array(
+            'Containable'
+    );
+
+    public $belongsTo = array(
+        'Collection' => array(
+            'className' => 'Collection',
+            'foreignKey' => 'collection_id'
+        )
+    );
+
+    // Make sure you also update the validation for element_type to include anything you add here.
+    public $valid_types = [
+        'Event',
+        'GalaxyCluster'
+    ];
+
+    public $validate = [
+        'collection_id' => [
+            'numeric' => [
+                'rule' => ['numeric']
+            ]
+        ],
+        'uuid' => [
+            'uuid' => [
+                'rule' => 'uuid',
+                'message' => 'Please provide a valid RFC 4122 UUID'
+            ]
+        ],
+        'element_uuid' => [
+            'element_uuid' => [
+                'rule' => 'uuid',
+                'message' => 'Please provide a valid RFC 4122 UUID'
+            ]
+        ],
+        'element_type' => [
+            'element_type' => [
+                'rule' => ['inList', ['Event', 'GalaxyCluster']],
+                'message' => 'Invalid object type.'
+            ]
+        ]
+    ];
+
+
+    public function beforeValidate($options = array())
+    {
+        // Massage to a common format
+        if (empty($this->data['CollectionElement'])) {
+            $this->data = ['CollectionElement' => $this->data];
+        }
+
+        // if we're creating a new element, assign a uuid (unless provided)
+        if (empty($this->id) && empty($this->data['CollectionElement']['uuid'])) {
+            $this->data['CollectionElement']['uuid'] = CakeText::uuid();
+        }
+        if (
+            empty($this->id) &&
+            empty($this->data['CollectionElement']['element_type']) &&
+            !empty($this->data['CollectionElement']['element_uuid'])
+        ) {
+            $this->data['CollectionElement']['element_type'] = $this->deduceType($this->data['CollectionElement']['element_uuid']);
+        }
+        return true;
+    }
+
+    public function mayModify(int $user_id, int $collection_id)
+    {
+        $user = $this->User->getAuthUser($user_id);
+        $collection = $this->find('first', [
+            'recursive' => -1,
+            'conditions' => ['Collection.id' => $collection_id]
+        ]);
+        if ($user['Role']['perm_site_admin']) {
+            return true;
+        }
+        if (empty($user['Role']['perm_modify'])) {
+            return false;
+        }
+        if (!empty($user['Role']['perm_modify_org'])) {
+            if ($user['org_id'] == $collection['Collection']['Orgc_id']) {
+                return true;
+            }
+            if ($user['Role']['perm_sync'] && $user['org_id'] == $collection['Collection']['Org_id']) {
+                return true;
+            }            
+        }
+        if (!empty($user['Role']['perm_modify']) && $user['id'] === $collection['Collection']['user_id']) {
+        }
+    }
+
+    public function mayView(int $user_id, int $collection_id)
+    {
+        $user = $this->User->getAuthUser($user_id);
+        $collection = $this->find('first', [
+            'recursive' => -1,
+            'conditions' => ['Collection.id' => $collection_id]
+        ]);
+        if ($user['Role']['perm_site_admin']) {
+            return true;
+        }
+        if ($collection['Collection']['org_id'] == $user('org_id')) {
+            return true;
+        }
+        if (in_array($collection['Collection']['distribution'], [1,2,3])) {
+            return true;
+        }
+        if ($collection['Collection']['distribution'] === 4) {
+            $SharingGroup = ClassRegistry::init('SharingGroup');
+            $sgs = $this->SharingGroup->fetchAllAuthorised($user, 'uuid');
+            if (isset($sgs[$collection['Collection']['sharing_group_id']])) {
+                return true;
+            } else {
+                return false;
+            }
+        }
+        return false;
+    }
+
+    public function deduceType(string $uuid)
+    {
+        foreach ($this->valid_types as $valid_type) {
+            $this->{$valid_type} = ClassRegistry::init($valid_type);
+            $result = $this->$valid_type->find('first', [
+                'conditions' => [$valid_type.'.uuid' => $uuid],
+                'recursive' => -1
+            ]);
+            if (!empty($result)) {
+                return $valid_type;
+            }
+        }
+        throw new NotFoundException(__('Invalid UUID'));
+    }
+
+    /*
+     *  Pass a Collection as received from another instance to this function to capture the elements
+     *  The received object is authoritative, so all elements that no longer exist in the upstream will be culled.
+     */
+    public function captureElements($data) {
+        $temp = $this->find('all', [
+            'recursive' => -1,
+            'conditions' => ['CollectionElement.collection_id' => $data['Collection']['id']]
+        ]);
+        $oldElements = [];
+        foreach ($temp as $oldElement) {
+            $oldElements[$oldElement['CollectionElement']['uuid']] = $oldElement['CollectionElement'];
+        }
+        if (isset($data['Collection']['CollectionElement'])) {
+            $elementsToSave = [];
+            foreach ($data['Collection']['CollectionElement'] as $k => $element) {
+                if (empty($element['uuid'])) {
+                    $element['uuid'] = CakeText::uuid();
+                }
+                if (isset($oldElements[$element['uuid']])) {
+                    if (isset($element['description'])) {
+                        $oldElements[$element['uuid']]['description'] = $element['description'];
+                    }
+                    $elementsToSave[$k] = $oldElements[$element['uuid']];
+                    unset($oldElements[$element['uuid']]);
+                } else {
+                    $elementsToSave[$k] = [
+                        'CollectionElement' => [
+                            'uuid' => $element['uuid'],
+                            'element_uuid' => $element['element_uuid'],
+                            'element_type' => $element['element_type'],
+                            'description' => $element['description'],
+                            'collection_id' => $data['Collection']['id']
+                        ]
+                    ];
+                    
+                }
+            }
+            foreach ($elementsToSave as $k => $element) {
+                if (empty($element['CollectionElement']['id'])) {
+                    $this->create();
+                }
+                try{
+                    $this->save($element);
+                } catch (PDOException $e) {
+                    // duplicate value?
+                }
+            }
+            foreach ($oldElements as $toDelete) {
+                $this->delete($toDelete['id']);
+            }
+            $temp = $this->find('all', [
+                'conditions' => ['CollectionElement.collection_id' => $data['Collection']['id']],
+                'recursive' => -1
+            ]);
+            $data['Collection']['CollectionElement'] = [];
+            foreach ($temp as $element) {
+                $data['Collection']['CollectionElement'][] = $element['CollectionElement'];
+            }
+        }
+
+        return $data;
+    }
+}

app/Model/Datasource/Database/MysqlExtended.php

@@ -58,6 +58,7 @@ class MysqlExtended extends Mysql
             'having' => $this->having($query['having'], true, $Model),
             'lock' => $this->getLockingHint($query['lock']),
             'indexHint' => $this->__buildIndexHint($query['forceIndexHint'] ?? null),
+            'ignoreIndexHint' => $this->__buildIgnoreIndexHint($query['ignoreIndexHint'] ?? null)
         ));
     }
 
@@ -90,6 +91,7 @@ class MysqlExtended extends Mysql
                 'having' => $queryData['having'],
                 'lock' => $queryData['lock'],
                 'forceIndexHint' => $queryData['forceIndexHint'] ?? null,
+                'ignoreIndexHint' => $queryData['ignoreIndexHint'] ?? null,
             ),
             $Model
         );
@@ -117,14 +119,25 @@ class MysqlExtended extends Mysql
     }
 
     /**
-     * Builds the index hint for the query
+     * Builds the force index hint for the query
      * 
-     * @param string|null $forceIndexHint FORCE INDEX hint
+     * @param string|null $forceIndexHint INDEX hint
      * @return string
      */
     private function __buildIndexHint($forceIndexHint = null): ?string
     {
-        return isset($forceIndexHint) ? ('FORCE INDEX ' . $forceIndexHint) : null;
+        return isset($forceIndexHint) ? ('FORCE INDEX (' . $forceIndexHint . ')') : null;
+    }
+
+        /**
+     * Builds the ignore index hint for the query
+     * 
+     * @param string|null $ignoreIndexHint INDEX hint
+     * @return string
+     */
+    private function __buildIgnoreIndexHint($ignoreIndexHint = null): ?string
+    {
+        return isset($ignoreIndexHint) ? ('IGNORE INDEX (' . $ignoreIndexHint . ')') : null;
     }
 
     /**
@@ -139,7 +152,9 @@ class MysqlExtended extends Mysql
     public function execute($sql, $options = [], $params = [])
     {
         $log = $options['log'] ?? $this->fullDebug;
-
+        if (Configure::read('Plugin.Benchmarking_enable')) {
+            $log = true;
+        }
         if ($log) {
             $t = microtime(true);
             $this->_result = $this->_execute($sql, $params);
@@ -164,6 +179,10 @@ class MysqlExtended extends Mysql
      */
     public function insertMulti($table, $fields, $values)
     {
+        if (empty($values)) {
+            return true;
+        }
+
         $table = $this->fullTableName($table);
         $holder = substr(str_repeat('?,', count($fields)), 0, -1);
         $fields = implode(',', array_map([$this, 'name'], $fields));

app/Model/Datasource/Database/MysqlObserverExtended.php

@@ -50,6 +50,7 @@ class MysqlObserverExtended extends Mysql
             'having' => $this->having($query['having'], true, $Model),
             'lock' => $this->getLockingHint($query['lock']),
             'indexHint' => $this->__buildIndexHint($query['forceIndexHint'] ?? null),
+            'ignoreIndexHint' => $this->__buildIgnoreIndexHint($query['ignoreIndexHint'] ?? null),
         ));
     }
 
@@ -82,6 +83,7 @@ class MysqlObserverExtended extends Mysql
                 'having' => $queryData['having'],
                 'lock' => $queryData['lock'],
                 'forceIndexHint' => $queryData['forceIndexHint'] ?? null,
+                'ignoreIndexHint' => $queryData['ignoreIndexHint'] ?? null,
             ),
             $Model
         );
@@ -103,20 +105,31 @@ class MysqlObserverExtended extends Mysql
             extract($data);
             $having = !empty($having) ? " $having" : '';
             $lock = !empty($lock) ? " $lock" : '';
-            return rtrim("SELECT {$fields} FROM {$table} {$alias} {$indexHint} {$joins} {$conditions} {$group}{$having} {$order} {$limit}{$lock}");
+            return rtrim("SELECT {$fields} FROM {$table} {$alias} {$indexHint} {$ignoreIndexHint} {$joins} {$conditions} {$group}{$having} {$order} {$limit}{$lock}");
         }
         return parent::renderStatement($type, $data);
     }
 
     /**
-     * Builds the index hint for the query
+     * Builds the force index hint for the query
      * 
-     * @param string|null $forceIndexHint FORCE INDEX hint
+     * @param string|null $forceIndexHint INDEX hint
      * @return string
      */
     private function __buildIndexHint($forceIndexHint = null): ?string
     {
-        return isset($forceIndexHint) ? ('FORCE INDEX ' . $forceIndexHint) : null;
+        return isset($forceIndexHint) ? ('FORCE INDEX (' . $forceIndexHint . ')') : null;
+    }
+
+        /**
+     * Builds the ignore index hint for the query
+     * 
+     * @param string|null $ignoreIndexHint INDEX hint
+     * @return string
+     */
+    private function __buildIgnoreIndexHint($ignoreIndexHint = null): ?string
+    {
+        return isset($ignoreIndexHint) ? ('IGNORE INDEX (' . $ignoreIndexHint . ')') : null;
     }
 
     /**
@@ -131,6 +144,9 @@ class MysqlObserverExtended extends Mysql
     public function execute($sql, $options = [], $params = [])
     {
         $log = $options['log'] ?? $this->fullDebug;
+        if (Configure::read('Plugin.Benchmarking_enable')) {
+            $log = true;
+        }
         $comment = sprintf(
             '%s%s%s',
             empty(Configure::read('CurrentUserId')) ? '' : sprintf(