Merge branch '2.4' into CRUD

.travis.yml

@@ -29,7 +29,7 @@ install:
     - sudo apt-get -y install haveged python3 python3-venv python3-pip python3-dev python3-nose python3-redis python3-lxml python3-dateutil python3-msgpack libxml2-dev libzmq3-dev zlib1g-dev apache2 curl php-mysql php-dev php-cli libapache2-mod-php libfuzzy-dev php-mbstring libonig4 php-json php-xml php-opcache php-readline php-redis php-gnupg php-gd
     - sudo pip3 install --upgrade pip setuptools requests pyzmq
     - sudo pip3 install --upgrade -r requirements.txt
-    - sudo pip3 install poetry
+    - pip3 install --user poetry
     - phpenv rehash
     - sudo mkdir $HOME/.composer ; sudo chown $USER:www-data $HOME/.composer
     - pushd app
@@ -146,6 +146,7 @@ before_script:
     - popd
 
 script:
+    - ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/
     - ./app/Vendor/bin/phpunit app/Test/ComplexTypeToolTest.php
     - pushd tests
     - ./curl_tests.sh $AUTH
@@ -153,7 +154,7 @@ script:
     - pushd PyMISP
     - git submodule init
     - git submodule update
-    - poetry install -E fileobjects -E openioc -E virustotal -E docs -E pdfexport
+    - travis_retry poetry install -E fileobjects -E openioc -E virustotal -E docs -E pdfexport
     - poetry run python tests/testlive_comprehensive.py
     - poetry run python tests/test_mispevent.py
     - popd
@@ -161,12 +162,9 @@ script:
     - pushd PyMISP/examples/events/
     - poetry run python ./create_massive_dummy_events.py -l 5 -a 30
     - popd
-    - pushd app/files/feed-metadata
-    - jsonschema -i defaults.json schema.json
-    - popd
+    - python3 tools/misp-feed/validate.py
 
 after_failure:
-    - ls -aRl `pwd`
     - curl http://misp.local
     - cat /etc/apache2/sites-available/misp.local.conf
     - sudo tail -n +1 `pwd`/app/tmp/logs/*

AUTHORS

@@ -3,11 +3,13 @@ Developers
 
 * Christophe Vandeplas <christophe@vandeplas.com> (original author)
 * Andras Iklody <andras.iklody@gmail.com> (lead developer)
+* Many more
 
 Contributors
 ------------
 
 Aaron Kaplan
+Andreas Ziegler
 Airbus Group CERT (AiG CERT)
 Alexander Jaeger
 Alexandre Dulaunoy
@@ -17,25 +19,35 @@ Andreas Ziegler
 Andrzej Dereszowski
 Bâkır Emre
 Chris Clark
+Christian Studer
 Christophe Vandeplas
 David André
 Guilherme Capilé
 Gábor Molnár
 Iglocska
+Jakub Onderka
 Koen Van Impe
 L. Aaron Kaplan
 Noud de Brouwer
 Raphaël Vinot
 Richard van den Berg
+Sami Mokaddem
+Steve Clement
 nullprobe
 remg427
 
-Copyright (C) 2012 Christophe Vandeplas
-Copyright (C) 2012 Belgian Defence
-Copyright (C) 2012 NATO / NCIRC
-Copyright (C) 2013-2018 Andras Iklody
-Copyright (C) 2015-2018 CIRCL - Computer Incident Response Center Luxembourg
-Copyright (C) 2016 Andreas Ziegler
+
+* Copyright (C) 2012-2020 Christophe Vandeplas
+* Copyright (C) 2012 Belgian Defence
+* Copyright (C) 2012 NATO / NCIRC
+* Copyright (C) 2013-2020 Andras Iklody
+* Copyright (C) 2015-2020 CIRCL - Computer Incident Response Center Luxembourg
+* Copyright (C) 2016 Andreas Ziegler
+* Copyright (C) 2018-2020 Sami Mokaddem
+* Copyright (C) 2018-2020 Christian Studer
+* Copyright (C) 2015-2020 Alexandre Dulaunoy
+* Copyright (C) 2018-2020 Steve Clement
+* Copyright (C) 2020 Jakub Onderka
 
 MISP is licensed under the GNU AFFERO GENERAL PUBLIC LICENSE version 3.
 

INSTALL/MYSQL.sql

@@ -604,7 +604,6 @@ CREATE TABLE IF NOT EXISTS `object_references` (
   `relationship_type` varchar(255) CHARACTER SET utf8 COLLATE utf8_unicode_ci,
   `comment` text COLLATE utf8_bin NOT NULL,
   `deleted` tinyint(1) NOT NULL DEFAULT 0,
-
   PRIMARY KEY (id),
   INDEX `source_uuid` (`source_uuid`),
   INDEX `referenced_uuid` (`referenced_uuid`),
@@ -1012,8 +1011,8 @@ CREATE TABLE IF NOT EXISTS tag_collection_tags (
     `tag_collection_id` int(11) NOT NULL,
     `tag_id` int(11) NOT NULL,
     PRIMARY KEY (id),
-    INDEX `uuid` (`tag_collection_id`),
-    INDEX `user_id` (`tag_id`)
+    INDEX `tag_collection_id` (`tag_collection_id`),
+    INDEX `tag_id` (`tag_id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8;
 
 -- --------------------------------------------------------

PyMISP

@@ -1 +1 @@
-Subproject commit 92c5d11f4799591b2a62e25cd7128d58f9bba88b
+Subproject commit f248a8bfffe400a4be54c34d222775f74831d1e9

README.md

@@ -104,11 +104,16 @@ License
 
 This software is licensed under [GNU Affero General Public License version 3](http://www.gnu.org/licenses/agpl-3.0.html)
 
-* Copyright (C) 2012 Christophe Vandeplas
+* Copyright (C) 2012-2020 Christophe Vandeplas
 * Copyright (C) 2012 Belgian Defence
 * Copyright (C) 2012 NATO / NCIRC
-* Copyright (C) 2013-2019 Andras Iklody
-* Copyright (C) 2015-2019 CIRCL - Computer Incident Response Center Luxembourg
+* Copyright (C) 2013-2020 Andras Iklody
+* Copyright (C) 2015-2020 CIRCL - Computer Incident Response Center Luxembourg
 * Copyright (C) 2016 Andreas Ziegler
+* Copyright (C) 2018-2020 Sami Mokaddem
+* Copyright (C) 2018-2020 Christian Studer
+* Copyright (C) 2015-2020 Alexandre Dulaunoy
+* Copyright (C) 2018-2020 Steve Clement
+* Copyright (C) 2020 Jakub Onderka
 
 For more information, [the list of authors and contributors](AUTHORS) is available.

VERSION.json

@@ -1 +1 @@
-{"major":2, "minor":4, "hotfix":130}
+{"major":2, "minor":4, "hotfix":133}

app/Config/config.default.php

@@ -22,6 +22,7 @@ $config = array(
 			'email_subject_include_tag_name' => true,
 			'background_jobs'                => true,
 			'cached_attachments'             => true,
+			'osuser'                         => 'www-data',
 			'email'                          => 'email@address.com',
 			'contact'                        => 'email@address.com',
 			'cveurl'                         => 'https://cve.circl.lu/cve/',

app/Config/core.default.php

@@ -181,9 +181,16 @@ Configure::write('Session', array(
 	'cookie_timeout' => 10080 ,  // Cookie timeout, default is 1 week
 	'defaults'       => 'php',
 	'autoRegenerate' => false,
-	'checkAgent' => false
+	'checkAgent' => false,
 ));
 
+// Set session cookie SameSite parameter to Lax if this param is supported and no default value is set
+if (PHP_VERSION_ID >= 70300 && empty(ini_get('session.cookie_samesite'))) {
+    Configure::write('Session.ini', [
+        'session.cookie_samesite' => 'Lax'
+    ]);
+}
+
 /**
  * The level of CakePHP security.
  */

app/Console/Command/AdminShell.php

@@ -2,7 +2,7 @@
 App::uses('AppShell', 'Console/Command');
 class AdminShell extends AppShell
 {
-    public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Whitelist', 'Server', 'Organisation', 'AdminSetting', 'Galaxy', 'Taxonomy', 'Warninglist', 'Noticelist', 'ObjectTemplate', 'Bruteforce', 'Role', 'Feed');
+    public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Allowedlist', 'Server', 'Organisation', 'AdminSetting', 'Galaxy', 'Taxonomy', 'Warninglist', 'Noticelist', 'ObjectTemplate', 'Bruteforce', 'Role', 'Feed');
 
     public $tasks = array('ConfigLoad');
 
@@ -351,13 +351,14 @@ class AdminShell extends AppShell
     {
         $this->ConfigLoad->execute();
         $whoami = exec('whoami');
-        if ($whoami === 'httpd' || $whoami === 'www-data' || $whoami === 'apache' || $whoami === 'wwwrun' || $whoami === 'travis' || $whoami === 'www') {
+        $osuser = Configure::read('MISP.osuser');
+        if ($whoami === 'httpd' || $whoami === 'www-data' || $whoami === 'apache' || $whoami === 'wwwrun' || $whoami === 'travis' || $whoami === 'www' || $whoami === $osuser) {
             echo 'Executing all updates to bring the database up to date with the current version.' . PHP_EOL;
             $processId = empty($this->args[0]) ? false : $this->args[0];
             $this->Server->runUpdates(true, false, $processId);
             echo 'All updates completed.' . PHP_EOL;
         } else {
-            die('This OS user is not allowed to run this command.'. PHP_EOL. 'Run it under `www-data` or `httpd` or `apache` or `wwwrun`.' . PHP_EOL . 'You tried to run this command as: ' . $whoami . PHP_EOL);
+            die('This OS user is not allowed to run this command.'. PHP_EOL. 'Run it under `www-data` or `httpd` or `apache` or `wwwrun` or set MISP.osuser in the configuration.' . PHP_EOL . 'You tried to run this command as: ' . $whoami . PHP_EOL);
         }
     }
 

app/Console/Command/EventShell.php

@@ -2,9 +2,15 @@
 App::uses('Folder', 'Utility');
 App::uses('File', 'Utility');
 require_once 'AppShell.php';
+
+/**
+ * @property User $User
+ * @property Event $Event
+ * @property Job $Job
+ */
 class EventShell extends AppShell
 {
-    public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Whitelist', 'Server', 'Organisation');
+    public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Allowedlist', 'Server', 'Organisation');
     public $tasks = array('ConfigLoad');
 
     public function doPublish()
@@ -131,19 +137,14 @@ class EventShell extends AppShell
     {
         $this->ConfigLoad->execute();
         $userId = $this->args[0];
-        $processId = $this->args[1];
-        $job = $this->Job->read(null, $processId);
+        $jobId = $this->args[1];
         $eventId = $this->args[2];
         $oldpublish = $this->args[3];
-        $user = $this->User->getAuthUser($userId);
+        $user = $this->User->getUserById($userId);
         if (empty($user)) {
             die("Invalid user ID '$userId' provided.");
         }
-        $result = $this->Event->sendAlertEmail($eventId, $user, $oldpublish, $processId);
-        $job['Job']['progress'] = 100;
-        $job['Job']['message'] = 'Emails sent.';
-        //$job['Job']['date_modified'] = date("Y-m-d H:i:s");
-        $this->Job->save($job);
+        $this->Event->sendAlertEmail($eventId, $user, $oldpublish, $jobId);
     }
 
     public function contactemail()
@@ -153,17 +154,14 @@ class EventShell extends AppShell
         $message = $this->args[1];
         $all = $this->args[2];
         $userId = $this->args[3];
-        $isSiteAdmin = $this->args[4];
-        $processId = $this->args[5];
-        $this->Job->id = $processId;
-        $user = $this->User->getAuthUser($userId);
+        $processId = $this->args[4];
+
+        $user = $this->User->getUserById($userId);
         if (empty($user)) {
             die("Invalid user ID '$userId' provided.");
         }
-        $result = $this->Event->sendContactEmail($id, $message, $all, array('User' => $user), $isSiteAdmin);
-        $this->Job->saveField('progress', '100');
-        $this->Job->saveField('date_modified', date("Y-m-d H:i:s"));
-        if ($result != true) $this->Job->saveField('message', 'Job done.');
+        $result = $this->Event->sendContactEmail($id, $message, $all, $user);
+        $this->Job->saveStatus($processId, $result);
     }
 
     public function postsemail()
@@ -381,4 +379,21 @@ class EventShell extends AppShell
         );
         return true;
     }
+
+    public function recoverEvent()
+    {
+        $this->ConfigLoad->execute();
+        $jobId = $this->args[0];
+        $id = $this->args[1];
+        $job = $this->Job->read(null, $jobId);
+        $job['Job']['progress'] = 1;
+        $job['Job']['date_modified'] = date("Y-m-d H:i:s");
+        $job['Job']['message'] = __('Recovering event %s', $id);
+        $this->Job->save($job);
+        $result = $this->Event->recoverEvent($id);
+        $job['Job']['progress'] = 100;
+        $job['Job']['date_modified'] = date("Y-m-d H:i:s");
+        $job['Job']['message'] = __('Recovery complete. Event #%s recovered, using %s log entries.', $id, $result);
+        $this->Job->save($job);
+    }
 }

app/Console/Command/ServerShell.php

@@ -338,7 +338,7 @@ class ServerShell extends AppShell
             $message = __n(
                 '%s feed from %s cached. Failed: %s',
                 '%s feeds from %s cached. Failed: %s',
-                $result['successes'], $total, $result['fails']
+                $result['successes'], $result['successes'], $total, $result['fails']
             );
             if ($result['fails'] > 0) {
                 $message .= ' ' . __('See error logs for more details.');

app/Controller/AllowedlistsController.php

@@ -2,7 +2,7 @@
 
 App::uses('AppController', 'Controller');
 
-class WhitelistsController extends AppController
+class AllowedlistsController extends AppController
 {
     public $components = array(
         'Security',
@@ -12,7 +12,7 @@ class WhitelistsController extends AppController
     public $paginate = array(
             'limit' => 60,
             'order' => array(
-                    'Whitelist.name' => 'ASC'
+                    'Allowedlist.name' => 'ASC'
             )
     );
 
@@ -27,7 +27,7 @@ class WhitelistsController extends AppController
     public function admin_index()
     {
         if (!$this->userRole['perm_regexp_access']) {
-            $this->redirect(array('controller' => 'whitelists', 'action' => 'index', 'admin' => false));
+            $this->redirect(array('controller' => 'allowedlists', 'action' => 'index', 'admin' => false));
         }
         $this->AdminCrud->adminIndex();
     }
@@ -35,7 +35,7 @@ class WhitelistsController extends AppController
     public function admin_edit($id = null)
     {
         if (!$this->userRole['perm_regexp_access']) {
-            $this->redirect(array('controller' => 'whitelists', 'action' => 'index', 'admin' => false));
+            $this->redirect(array('controller' => 'allowedlists', 'action' => 'index', 'admin' => false));
         }
         $this->AdminCrud->adminEdit($id);
     }
@@ -43,7 +43,7 @@ class WhitelistsController extends AppController
     public function admin_delete($id = null)
     {
         if (!$this->userRole['perm_regexp_access']) {
-            $this->redirect(array('controller' => 'whitelists', 'action' => 'index', 'admin' => false));
+            $this->redirect(array('controller' => 'allowedlists', 'action' => 'index', 'admin' => false));
         }
         $this->AdminCrud->adminDelete($id);
     }

app/Controller/AppController.php

@@ -37,6 +37,8 @@ App::uses('RequestRearrangeTool', 'Tools');
  * @link http://book.cakephp.org/2.0/en/controllers.html#the-app-controller
  *
  * @throws ForbiddenException // TODO Exception
+ * @property ACLComponent $ACL
+ * @property RestResponseComponent $RestResponse
  */
 class AppController extends Controller
 {
@@ -46,8 +48,8 @@ class AppController extends Controller
 
     public $helpers = array('Utility', 'OrgImg', 'FontAwesome', 'UserName', 'DataPathCollector');
 
-    private $__queryVersion = '109';
-    public $pyMispVersion = '2.4.130';
+    private $__queryVersion = '113';
+    public $pyMispVersion = '2.4.133';
     public $phpmin = '7.2';
     public $phprec = '7.4';
     public $pythonmin = '3.6';
@@ -455,6 +457,7 @@ class AppController extends Controller
             $this->set('isAclZmq', isset($role['perm_publish_zmq']) ? $role['perm_publish_zmq'] : false);
             $this->set('isAclKafka', isset($role['perm_publish_kafka']) ? $role['perm_publish_kafka'] : false);
             $this->set('isAclDecaying', isset($role['perm_decaying']) ? $role['perm_decaying'] : false);
+            $this->set('aclComponent', $this->ACL);
             $this->userRole = $role;
 
             $this->set('loggedInUserName', $this->__convertEmailToName($this->Auth->user('email')));
@@ -949,9 +952,9 @@ class AppController extends Controller
         ));
         $counter = 0;
 
-        // load this so we can remove the blacklist item that will be created, this is the one case when we do not want it.
-        if (Configure::read('MISP.enableEventBlacklisting') !== false) {
-            $this->EventBlacklist = ClassRegistry::init('EventBlacklist');
+        // load this so we can remove the blocklist item that will be created, this is the one case when we do not want it.
+        if (Configure::read('MISP.enableEventBlocklisting') !== false) {
+            $this->EventBlocklist = ClassRegistry::init('EventBlocklist');
         }
 
         foreach ($duplicates as $duplicate) {
@@ -964,10 +967,10 @@ class AppController extends Controller
                     $uuid = $event['Event']['uuid'];
                     $this->Event->delete($event['Event']['id']);
                     $counter++;
-                    // remove the blacklist entry that we just created with the event deletion, if the feature is enabled
+                    // remove the blocklist entry that we just created with the event deletion, if the feature is enabled
                     // We do not want to block the UUID, since we just deleted a copy
-                    if (Configure::read('MISP.enableEventBlacklisting') !== false) {
-                        $this->EventBlacklist->deleteAll(array('EventBlacklist.event_uuid' => $uuid));
+                    if (Configure::read('MISP.enableEventBlocklisting') !== false) {
+                        $this->EventBlocklist->deleteAll(array('EventBlocklist.event_uuid' => $uuid));
                     }
                 }
             }

app/Controller/AttributesController.php

@@ -158,7 +158,6 @@ class AttributesController extends AppController
             if (!isset($attributes[0])) {
                 $attributes = array(0 => $attributes);
             }
-            $this->Warninglist = ClassRegistry::init('Warninglist');
             $fails = array();
             $successes = 0;
             $attributeCount = count($attributes);
@@ -1448,7 +1447,6 @@ class AttributesController extends AppController
                 $event['Event']['timestamp'] = $timestamp;
                 $event['Event']['published'] = 0;
                 $this->Attribute->Event->save($event, array('fieldList' => array('published', 'timestamp', 'id')));
-                return new CakeResponse(array('body'=> json_encode(array('saved' => true)), 'status' => 200, 'type' => 'json'));
             } else {
                 return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'validationErrors' => $this->Attribute->validationErrors)), 'status' => 200, 'type' => 'json'));
             }
@@ -1951,14 +1949,10 @@ class AttributesController extends AppController
             throw new MethodNotAllowedException(__('This function can only be accessed via AJAX.'));
         }
         $params = array(
-                'conditions' => array('Attribute.id' => $id),
-                'fields' => array('id', 'distribution', 'event_id', $field),
-                'contain' => array(
-                        'Event' => array(
-                                'fields' => array('distribution', 'id', 'org_id'),
-                        )
-                ),
-                'flatten' => 1
+            'conditions' => array('Attribute.id' => $id),
+            'fields' => array('id', 'category', 'type', $field),
+            'contain' => ['Event'],
+            'flatten' => 1,
         );
         $attribute = $this->Attribute->fetchAttributes($this->Auth->user(), $params);
         if (empty($attribute)) {
@@ -1978,6 +1972,8 @@ class AttributesController extends AppController
             }
         }
         $this->set('value', $result);
+        $this->set('object', $attribute);
+        $this->set('field', $field);
         $this->layout = 'ajax';
         $this->render('ajax/attributeViewFieldForm');
     }
@@ -2406,23 +2402,20 @@ class AttributesController extends AppController
         if (empty($attribute)) {
             throw new NotFoundException(__('Invalid Attribute'));
         }
-        $this->loadModel('Server');
         $this->loadModel('Module');
         $modules = $this->Module->getEnabledModules($this->Auth->user());
         $validTypes = array();
         if (isset($modules['hover_type'][$attribute[0]['Attribute']['type']])) {
             $validTypes = $modules['hover_type'][$attribute[0]['Attribute']['type']];
         }
-        $url = Configure::read('Plugin.Enrichment_services_url') ? Configure::read('Plugin.Enrichment_services_url') : $this->Server->serverSettings['Plugin']['Enrichment_services_url']['value'];
-        $port = Configure::read('Plugin.Enrichment_services_port') ? Configure::read('Plugin.Enrichment_services_port') : $this->Server->serverSettings['Plugin']['Enrichment_services_port']['value'];
         $resultArray = array();
         foreach ($validTypes as $type) {
             $options = array();
             $found = false;
             foreach ($modules['modules'] as $temp) {
-                if ($temp['name'] == $type) {
+                if ($temp['name'] === $type) {
                     $found = true;
-                    $format = (isset($temp['mispattributes']['format']) ? $temp['mispattributes']['format'] : 'simplified');
+                    $format = isset($temp['mispattributes']['format']) ? $temp['mispattributes']['format'] : 'simplified';
                     if (isset($temp['meta']['config'])) {
                         foreach ($temp['meta']['config'] as $conf) {
                             $options[$conf] = Configure::read('Plugin.Enrichment_' . $type . '_' . $conf);
@@ -2450,29 +2443,32 @@ class AttributesController extends AppController
             $result = $this->Module->queryModuleServer('/query', $data, true);
             if ($result) {
                 if (!is_array($result)) {
-                    $resultArray[$type][] = array($type => $result);
+                    $resultArray[$type] = ['error' => $result];
+                    continue;
                 }
             } else {
                 // TODO: i18n?
-                $resultArray[$type][] = array($type => 'Enrichment service not reachable.');
+                $resultArray[$type] = ['error' => 'Enrichment service not reachable.'];
                 continue;
             }
             $current_result = array();
             if (isset($result['results']['Object'])) {
                 if (!empty($result['results']['Object'])) {
                     $objects = array();
-                    foreach($result['results']['Object'] as $object) {
+                    foreach ($result['results']['Object'] as $object) {
                         if (isset($object['Attribute']) && !empty($object['Attribute'])) {
                             $object_attributes = array();
                             foreach($object['Attribute'] as $object_attribute) {
-                                array_push($object_attributes, array('object_relation' => $object_attribute['object_relation'], 'value' => $object_attribute['value']));
+                                $object_attributes[] = [
+                                    'object_relation' => $object_attribute['object_relation'],
+                                    'value' => $object_attribute['value'],
+                                    'type' => $object_attribute['type'],
+                                ];
                             }
-                            array_push($objects, array('name' => $object['name'], 'Attribute' => $object_attributes));
+                            $objects[] = array('name' => $object['name'], 'Attribute' => $object_attributes);
                         }
                     }
-                    if (!empty($objects)) {
-                        $current_result['Object'] = $objects;
-                    }
+                    $current_result['Object'] = $objects;
                 }
                 unset($result['results']['Object']);
             }
@@ -2506,6 +2502,7 @@ class AttributesController extends AppController
                 }
             }
         }
+        $this->set('persistent', $persistent);
         $this->set('results', $resultArray);
         $this->layout = 'ajax';
         $this->render('ajax/hover_enrichment');
@@ -2938,17 +2935,37 @@ class AttributesController extends AppController
                 'all',
                 array(
                     'conditions' => array('Attribute.type' => array('attachment', 'malware-sample')),
-                    'recursive' => -1)
+                    'contain' => ['Event.orgc_id', 'Event.org_id'],
+                    'recursive' => -1
+                )
             );
         $counter = 0;
         $attachmentTool = new AttachmentTool();
+        $results = [];
         foreach ($attributes as $attribute) {
             $exists = $attachmentTool->exists($attribute['Attribute']['event_id'], $attribute['Attribute']['id']);
             if (!$exists) {
+                $results['affectedEvents'][$attribute['Attribute']['event_id']] = $attribute['Attribute']['event_id'];
+                $results['affectedAttributes'][] = $attribute['Attribute']['id'];
+                foreach (['orgc', 'org'] as $type) {
+                    if (empty($results['affectedOrgs'][$type][$attribute['Event'][$type . '_id']])) {
+                        $results['affectedOrgs'][$type][$attribute['Event'][$type . '_id']] = 0;
+                    } else {
+                        $results['affectedOrgs'][$type][$attribute['Event'][$type . '_id']] += 1;
+                    }
+                }
                 $counter++;
             }
         }
-
+        if (!empty($results)) {
+            $results['affectedEvents'] = array_values($results['affectedEvents']);
+            rsort($results['affectedEvents']);
+            rsort($results['affectedAttributes']);
+            foreach (['orgc', 'org'] as $type) {
+                arsort($results['affectedOrgs'][$type]);
+            }
+        }
+        file_put_contents(APP . '/tmp/logs/missing_attachments.log', json_encode($results, JSON_PRETTY_PRINT));
         return new CakeResponse(array('body' => $counter, 'status' => 200));
     }
 

app/Controller/Component/ACLComponent.php

@@ -111,7 +111,7 @@ class ACLComponent extends Component
                     'requestAccess' => array(),
                     'view' => array()
             ),
-            'eventBlacklists' => array(
+            'eventBlocklists' => array(
                     'add' => [
                         'AND' => [
                             'host_org_user',
@@ -150,6 +150,16 @@ class ACLComponent extends Component
                     'index' => array('*'),
                     'view' => array('*'),
             ),
+            'eventReports' => array(
+                'add' => array('perm_add'),
+                'view' => array('*'),
+                'viewSummary' => array('*'),
+                'edit' => array('perm_add'),
+                'delete' => array('perm_add'),
+                'restore' => array('perm_add'),
+                'index' => array('*'),
+                'getProxyMISPElements' => array('*'),
+            ),
             'events' => array(
                     'add' => array('perm_add'),
                     'addIOC' => array('perm_add'),
@@ -203,9 +213,11 @@ class ACLComponent extends Component
                     'pushEventToKafka' => array('perm_publish_kafka'),
                     'pushProposals' => array('perm_sync'),
                     'queryEnrichment' => array('perm_add'),
+                    'recoverEvent' => array('perm_site_admin'),
                     'removePivot' => array('*'),
                     'removeTag' => array('perm_tagger'),
                     'reportValidationIssuesEvents' => array(),
+                    'restoreDeletedEvents' => array('perm_site_admin'),
                     'restSearch' => array('*'),
                     'saveFreeText' => array('perm_add'),
                     'stix' => array('*'),
@@ -219,7 +231,6 @@ class ACLComponent extends Component
                     'upload_stix' => array('perm_add'),
                     'view' => array('*'),
                     'viewEventAttributes' => array('*'),
-                    'viewEventGraph' => array('*'),
                     'viewGraph' => array('*'),
                     'viewGalaxyMatrix' => array('*'),
                     'xml' => array('*')
@@ -242,11 +253,17 @@ class ACLComponent extends Component
                     'fetchSelectedFromFreetextIndex' => array(),
                     'getEvent' => array(),
                     'importFeeds' => array(),
-                    'index' => array('*'),
+                    'index' => ['OR' => [
+                        'host_org_user',
+                        'perm_site_admin',
+                    ]],
                     'loadDefaultFeeds' => array('perm_site_admin'),
                     'previewEvent' => array('*'),
                     'previewIndex' => array('*'),
-                    'searchCaches' => array('*'),
+                    'searchCaches' => ['OR' => [
+                        'host_org_user',
+                        'perm_site_admin',
+                    ]],
                     'toggleSelected' => array('perm_site_admin'),
                     'view' => array('*'),
             ),
@@ -349,7 +366,7 @@ class ACLComponent extends Component
             'objectTemplateElements' => array(
                 'viewElements' => array('*')
             ),
-            'orgBlacklists' => array(
+            'orgBlocklists' => array(
                     'add' => array(),
                     'delete' => array(),
                     'edit' => array(),
@@ -365,7 +382,6 @@ class ACLComponent extends Component
                     'fetchSGOrgRow' => array('*'),
                     'getUUIDs' => array('perm_sync'),
                     'index' => array('*'),
-                    'landingpage' => array('*'),
                     'view' => array('*'),
             ),
             'pages' => array(
@@ -423,7 +439,10 @@ class ACLComponent extends Component
                     'getSubmoduleQuickUpdateForm' => array(),
                     'getWorkers' => array(),
                     'getVersion' => array('*'),
-                    'idTranslator' => array('*'),
+                    'idTranslator' => ['OR' => [
+                        'host_org_user',
+                        'perm_site_admin',
+                    ]],
                     'import' => array(),
                     'index' => array(),
                     'ondemandAction' => array(),
@@ -452,7 +471,8 @@ class ACLComponent extends Component
                     'updateProgress' => array(),
                     'updateSubmodule' => array(),
                     'uploadFile' => array(),
-                    'viewDeprecatedFunctionUse' => array()
+                    'viewDeprecatedFunctionUse' => array(),
+                    'killAllWorkers' => ['perm_site_admin'],
             ),
             'shadowAttributes' => array(
                     'accept' => array('perm_add'),
@@ -633,7 +653,7 @@ class ACLComponent extends Component
                     'update' => array(),
                     'view' => array('*')
             ),
-            'whitelists' => array(
+            'allowedlists' => array(
                     'admin_add' => array('perm_regexp_access'),
                     'admin_delete' => array('perm_regexp_access'),
                     'admin_edit' => array('perm_regexp_access'),
@@ -745,7 +765,9 @@ class ACLComponent extends Component
         foreach ($aclList as $k => $v) {
             $aclList[$k] = array_change_key_case($v);
         }
-        $this->__checkLoggedActions($user, $controller, $action);
+        if (!$soft) {
+            $this->__checkLoggedActions($user, $controller, $action);
+        }
         if ($user && $user['Role']['perm_site_admin']) {
             return true;
         }
@@ -801,7 +823,6 @@ class ACLComponent extends Component
         switch ($code) {
             case 404:
                 throw new NotFoundException($message);
-                break;
             case 403:
                 throw new MethodNotAllowedException($message);
             default:

app/Controller/Component/BlockListComponent.php

@@ -4,7 +4,7 @@
  * create, read, update and delete (CRUD)
  */
 
-class BlackListComponent extends Component
+class BlocklistComponent extends Component
 {
     public $settings = array();
     public $defaultModel = '';
@@ -17,12 +17,15 @@ class BlackListComponent extends Component
             $this->controller->paginate['conditions'] = $filters;
         }
         if ($rest) {
-            $blackList = $this->controller->paginate();
-            $blacklist= array();
-            foreach ($blackList as $item) {
-                $blacklist[] = $item[$this->controller->defaultModel];
+            $data = $this->controller->{$this->controller->defaultModel}->find('all', array(
+                'recursive' => -1,
+                'conditions' => isset($this->controller->paginate['conditions']) ? $this->controller->paginate['conditions'] : []
+            ));
+            $blocklist = [];
+            foreach ($data as $item) {
+                $blocklist[] = $item[$this->controller->defaultModel];
             }
-            return $this->RestResponse->viewData($blacklist);
+            return $this->RestResponse->viewData($blocklist);
         } else {
             $this->controller->set('response', $this->controller->paginate());
         }
@@ -62,8 +65,8 @@ class BlackListComponent extends Component
                 if (strlen($uuid) == 36) {
                     $this->controller->{$this->controller->defaultModel}->create();
                     $object = array();
-                    foreach ($this->controller->{$this->controller->defaultModel}->blacklistFields as $f) {
-                        if ($f === $this->controller->{$this->controller->defaultModel}->blacklistTarget . '_uuid') {
+                    foreach ($this->controller->{$this->controller->defaultModel}->blocklistFields as $f) {
+                        if ($f === $this->controller->{$this->controller->defaultModel}->blocklistTarget . '_uuid') {
                             $object[$f] = $uuid;
                         } else {
                             $object[$f] = !empty($data[$this->controller->defaultModel][$f]) ? $data[$this->controller->defaultModel][$f] : '';
@@ -78,7 +81,7 @@ class BlackListComponent extends Component
                     $fails[] = $uuid;
                 }
             }
-            $message = sprintf(__('Done. Added %d new entries to the blacklist. %d entries could not be saved.'), count($successes), count($fails));
+            $message = sprintf(__('Done. Added %d new entries to the blocklist. %d entries could not be saved.'), count($successes), count($fails));
             if ($rest) {
                 $result = [
                     'result' => [
@@ -100,14 +103,14 @@ class BlackListComponent extends Component
         if (Validation::uuid($id)) {
             $blockEntry = $this->controller->{$this->controller->defaultModel}->find('first', [
                 'conditions' => array(
-                    $this->controller->{$this->controller->defaultModel}->blacklistTarget . '_uuid' => $id
+                    $this->controller->{$this->controller->defaultModel}->blocklistTarget . '_uuid' => $id
                 )
             ]);
         } else {
             $blockEntry = $this->controller->{$this->controller->defaultModel}->find('first', array('conditions' => array('id' => $id)));
         }
         if (empty($blockEntry)) {
-            throw new NotFoundException('Blacklist item not found.');
+            throw new NotFoundException('Blocklist item not found.');
         }
         $this->controller->set('blockEntry', $blockEntry);
         if ($this->controller->request->is('post')) {
@@ -127,7 +130,7 @@ class BlackListComponent extends Component
             } else {
                 $data = $this->controller->request->data;
             }
-            $fields = $this->controller->{$this->controller->defaultModel}->blacklistFields;
+            $fields = $this->controller->{$this->controller->defaultModel}->blocklistFields;
             foreach ($fields as $f) {
                 if ($f == 'uuid') {
                     continue;
@@ -147,14 +150,14 @@ class BlackListComponent extends Component
                         ])
                     );
                 } else {
-                    $this->controller->Session->setFlash(__('Blacklist item added.'));
+                    $this->controller->Session->setFlash(__('Blocklist item added.'));
                     $this->controller->redirect(array('action' => 'index'));
                 }
             } else {
                 if ($rest) {
-                    throw new MethodNotAllowedException('Could not save the blacklist item.');
+                    throw new MethodNotAllowedException('Could not save the blocklist item.');
                 } else {
-                    $this->controller->Session->setFlash(__('Could not save the blacklist item'));
+                    $this->controller->Session->setFlash(__('Could not save the blocklist item'));
                     $this->controller->redirect(array('action' => 'index'));
                 }
             }
@@ -166,24 +169,24 @@ class BlackListComponent extends Component
         if (Validation::uuid($id)) {
             $blockEntry = $this->controller->{$this->controller->defaultModel}->find('first', [
                 'conditions' => array(
-                    $this->controller->{$this->controller->defaultModel}->blacklistTarget . '_uuid' => $id
+                    $this->controller->{$this->controller->defaultModel}->blocklistTarget . '_uuid' => $id
                 )
             ]);
         } else {
             $blockEntry = $this->controller->{$this->controller->defaultModel}->find('first', array('conditions' => array('id' => $id)));
         }
         if (empty($blockEntry)) {
-            throw new NotFoundException(__('Invalid blacklist entry'));
+            throw new NotFoundException(__('Invalid blocklist entry'));
         }
 
         if ($this->controller->{$this->controller->defaultModel}->delete($blockEntry[$this->controller->defaultModel]['id'])) {
-            $message = __('Blacklist entry removed');
+            $message = __('Blocklist entry removed');
             if ($rest) {
                 return $this->RestResponse->saveSuccessResponse($this->controller->defaultModel, 'delete', $id, false, $message);
             }
             $this->controller->Flash->success($message);
         } else {
-            $message = __('Could not remove the blacklist entry');
+            $message = __('Could not remove the blocklist entry');
             if ($rest) {
                 return $this->RestResponse->saveFailResponse($this->controller->defaultModel, 'delete', $id, $message);
             }

app/Controller/Component/DeprecationComponent.php

@@ -36,7 +36,7 @@ class DeprecationComponent extends Component
                 'add' => false,
                 'populateEventFromTemplate' => false
             ),
-            'whitelists' => array(
+            'allowedlists' => array(
                 'admin_add' => false
             )
         );

app/Controller/Component/RestResponseComponent.php

@@ -15,7 +15,10 @@ class RestResponseComponent extends Component
         )
     );
 
-    private $___setup = false;
+    private $__setup = false;
+
+    /** @var array */
+    private $__fieldsConstraint;
 
     private $__descriptions = array(
         'Attribute' => array(
@@ -83,6 +86,18 @@ class RestResponseComponent extends Component
                 'optional' => array('network_name')
             )
         ),
+        'EventReport' => array(
+            'add' => array(
+                'description' => "POST a report in JSON format to create a report for the provided event",
+                'mandatory' => array('name'),
+                'optional' => array('distribution', 'content')
+            ),
+            'edit' => array(
+                'description' => "POST a report in JSON format to update the report",
+                'mandatory' => array(),
+                'optional' => array('name', 'distribution', 'content')
+            )
+        ),
         'Feed' => array(
             'add' => array(
                 'description' => "POST a MISP Feed descriptor JSON to this API to add a Feed.",
@@ -276,8 +291,11 @@ class RestResponseComponent extends Component
 
     private $__scopedFieldsConstraint = array();
 
-    public function initialize(Controller $controller) {
-        $this->__configureFieldConstraints();
+    /** @var Controller */
+    private $Controller;
+
+    public function initialize(Controller $controller)
+    {
         $this->Controller = $controller;
     }
 
@@ -383,7 +401,7 @@ class RestResponseComponent extends Component
         return '[]';
     }
 
-    public function saveFailResponse($controller, $action, $id = false, $validationErrors, $format = false)
+    public function saveFailResponse($controller, $action, $id = false, $validationErrors, $format = false, $data = null)
     {
         $this->autoRender = false;
         $response = array();
@@ -395,12 +413,15 @@ class RestResponseComponent extends Component
         $response['saved'] = false;
         $response['name'] = 'Could not ' . $stringifiedAction . ' ' . Inflector::singularize($controller);
         $response['message'] = $response['name'];
+        if (!is_null($data)) {
+            $response['data'] = $data;
+        }
         $response['url'] = $this->__generateURL($action, $controller, $id);
         $response['errors'] = $validationErrors;
         return $this->__sendResponse($response, 403, $format);
     }
 
-    public function saveSuccessResponse($controller, $action, $id = false, $format = false, $message = false)
+    public function saveSuccessResponse($controller, $action, $id = false, $format = false, $message = false, $data = null)
     {
         $action = $this->__dissectAdminRouting($action);
         if (!$message) {
@@ -410,13 +431,26 @@ class RestResponseComponent extends Component
         $response['success'] = true;
         $response['name'] = $message;
         $response['message'] = $response['name'];
+        if (!is_null($data)) {
+            $response['data'] = $data;
+        }
         $response['url'] = $this->__generateURL($action, $controller, $id);
         return $this->__sendResponse($response, 200, $format);
     }
 
+    /**
+     * @param mixed $response
+     * @param int $code
+     * @param string|false $format
+     * @param bool $raw
+     * @param bool $download
+     * @param array $headers
+     * @return CakeResponse
+     */
     private function __sendResponse($response, $code, $format = false, $raw = false, $download = false, $headers = array())
     {
-        if (strtolower($format) === 'application/xml' || strtolower($format) === 'xml') {
+        $format = strtolower($format);
+        if ($format === 'application/xml' || $format === 'xml') {
             if (!$raw) {
                 if (isset($response[0])) {
                     if (count(array_keys($response[0])) == 1) {
@@ -433,9 +467,9 @@ class RestResponseComponent extends Component
                 $response = $response->asXML();
             }
             $type = 'xml';
-        } elseif (strtolower($format) == 'openioc') {
+        } elseif ($format === 'openioc') {
             $type = 'xml';
-        } elseif (strtolower($format) == 'csv') {
+        } elseif ($format === 'csv') {
             $type = 'csv';
         } else {
             if (empty($format)) {
@@ -471,7 +505,7 @@ class RestResponseComponent extends Component
                 }
             }
         }
-        $cakeResponse = new CakeResponse(array('body'=> $response, 'status' => $code, 'type' => $type));
+        $cakeResponse = new CakeResponse(array('body' => $response, 'status' => $code, 'type' => $type));
 
         if (Configure::read('Security.allow_cors')) {
             $headers["Access-Control-Allow-Headers"] =  "Origin, Content-Type, Authorization, Accept";
@@ -480,14 +514,10 @@ class RestResponseComponent extends Component
             $headers["Access-Control-Expose-Headers"] = ["X-Result-Count"];
         }
         if (!empty($this->headers)) {
-            foreach ($this->headers as $key => $value) {
-                $cakeResponse->header($key, $value);
-            }
+            $cakeResponse->header($this->headers);
         }
         if (!empty($headers)) {
-            foreach ($headers as $key => $value) {
-                $cakeResponse->header($key, $value);
-            }
+            $cakeResponse->header($headers);
         }
         if (!empty($deprecationWarnings)) {
             $cakeResponse->header('X-Deprecation-Warning', $deprecationWarnings);
@@ -495,7 +525,6 @@ class RestResponseComponent extends Component
         if ($download) {
             $cakeResponse->download($download);
         }
-
         return $cakeResponse;
     }
 
@@ -584,13 +613,13 @@ class RestResponseComponent extends Component
                     'params' => $this->__descriptions[$scope]['restSearch']['params']
                 );
             }
+            $this->__configureFieldConstraints();
             $this->__setupFieldsConstraint();
+            $this->__setup = true;
         }
         return true;
     }
 
-    private $__fieldConstraint = array();
-
     // default value and input for API field
     private function __configureFieldConstraints()
     {
@@ -606,7 +635,7 @@ class RestResponseComponent extends Component
                 'input' => 'radio',
                 'type' => 'integer',
                 'values' => array(1 => 'True', 0 => 'False' ),
-                'help' => __('Is the sharing group selectable (active) when chosing distribution')
+                'help' => __('Is the sharing group selectable (active) when choosing distribution')
             ),
             'all' => array(
                 'input' => 'text',
@@ -836,7 +865,7 @@ class RestResponseComponent extends Component
                 'input' => 'radio',
                 'type' => 'integer',
                 'values' => array(1 => 'True', 0 => 'False' ),
-                'help' => __('When uploading malicious samples, set this flag to tell MISP to encrpyt the sample and extract the file hashes. This will create a MISP object with the appropriate attributes.')
+                'help' => __('When uploading malicious samples, set this flag to tell MISP to encrypt the sample and extract the file hashes. This will create a MISP object with the appropriate attributes.')
             ),
             //'enforceWarningList' => array(
             //    'input' => 'radio',
@@ -1099,7 +1128,7 @@ class RestResponseComponent extends Component
                 'input' => 'select',
                 'type' => 'string',
                 'operators' => array('equal'),
-                'values' => array('Attribute', 'Event', 'EventBlacklist', 'EventTag', 'MispObject', 'Organisation', 'Post', 'Regexp', 'Role', 'Server', 'ShadowAttribute', 'SharingGroup', 'Tag', 'Task', 'Taxonomy', 'Template', 'Thread', 'User', 'Whitelist'),
+                'values' => array('Attribute', 'Event', 'EventBlocklist', 'EventTag', 'MispObject', 'Organisation', 'Post', 'Regexp', 'Role', 'Server', 'ShadowAttribute', 'SharingGroup', 'Tag', 'Task', 'Taxonomy', 'Template', 'Thread', 'User', 'Whitelist'),
             ),
             'model_id' => array(
                 'input' => 'number',
@@ -1775,5 +1804,4 @@ class RestResponseComponent extends Component
             $field['help'] = __('Seen within the last x amount of time, where x can be defined in days, hours, minutes (for example 5d or 12h or 30m)');
         }
     }
-
 }

app/Controller/EventBlocklistsController.php

@@ -1,15 +1,15 @@
 <?php
 App::uses('AppController', 'Controller');
 
-class EventBlacklistsController extends AppController
+class EventBlocklistsController extends AppController
 {
-    public $components = array('Session', 'RequestHandler', 'BlackList');
+    public $components = array('Session', 'RequestHandler', 'BlockList');
 
     public function beforeFilter()
     {
         parent::beforeFilter();
-        if (false === Configure::read('MISP.enableEventBlacklisting')) {
-            $this->Flash->info(__('Event Blacklisting is not currently enabled on this instance.'));
+        if (false === Configure::read('MISP.enableEventBlocklisting')) {
+            $this->Flash->info(__('Event Blocklisting is not currently enabled on this instance.'));
             $this->redirect('/');
         }
     }
@@ -18,7 +18,7 @@ class EventBlacklistsController extends AppController
             'limit' => 60,
             'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page.
             'order' => array(
-                    'EventBlacklist.created' => 'DESC'
+                    'EventBlocklist.created' => 'DESC'
             ),
     );
 
@@ -43,56 +43,56 @@ class EventBlacklistsController extends AppController
         }
         $this->set('passedArgs', json_encode($passedArgs));
         $this->set('passedArgsArray', $passedArgsArray);
-        return $this->BlackList->index($this->_isRest(), $params);
+        return $this->BlockList->index($this->_isRest(), $params);
     }
 
     public function add()
     {
-        return $this->BlackList->add($this->_isRest());
+        return $this->BlockList->add($this->_isRest());
     }
 
     public function edit($id)
     {
-        return $this->BlackList->edit($this->_isRest(), $id);
+        return $this->BlockList->edit($this->_isRest(), $id);
     }
 
     public function delete($id)
     {
-        return $this->BlackList->delete($this->_isRest(), $id);
+        return $this->BlockList->delete($this->_isRest(), $id);
     }
 
     public function massDelete()
     {
         if ($this->request->is('post') || $this->request->is('put')) {
-            if (!isset($this->request->data['EventBlacklist'])) {
-                $this->request->data = array('EventBlacklist' => $this->request->data);
+            if (!isset($this->request->data['EventBlocklist'])) {
+                $this->request->data = array('EventBlocklist' => $this->request->data);
             }
-            $ids = $this->request->data['EventBlacklist']['ids'];
+            $ids = $this->request->data['EventBlocklist']['ids'];
             $event_ids = json_decode($ids, true);
             if (empty($event_ids)) {
                 throw new NotFoundException(__('Invalid event IDs.'));
             }
-            $result = $this->EventBlacklist->deleteAll(array('EventBlacklist.id' => $event_ids));
+            $result = $this->EventBlocklist->deleteAll(array('EventBlocklist.id' => $event_ids));
             if ($result) {
                 if ($this->_isRest()) {
-                    return $this->RestResponse->saveSuccessResponse('EventBlacklist', 'Deleted', $ids, $this->response->type());
+                    return $this->RestResponse->saveSuccessResponse('EventBlocklist', 'Deleted', $ids, $this->response->type());
                 } else {
-                    $this->Flash->success('Blacklist entry removed');
-                    $this->redirect(array('controller' => 'eventBlacklists', 'action' => 'index'));
+                    $this->Flash->success('Blocklist entry removed');
+                    $this->redirect(array('controller' => 'eventBlocklists', 'action' => 'index'));
                 }
             } else {
-                $error = __('Failed to delete Event from EventBlacklist. Error: ') . PHP_EOL . h($result);
+                $error = __('Failed to delete Event from EventBlocklist. Error: ') . PHP_EOL . h($result);
                 if ($this->_isRest()) {
-                    return $this->RestResponse->saveFailResponse('EventBlacklist', 'Deleted', false, $error, $this->response->type());
+                    return $this->RestResponse->saveFailResponse('EventBlocklist', 'Deleted', false, $error, $this->response->type());
                 } else {
                     $this->Flash->error($error);
-                    $this->redirect(array('controller' => 'eventBlacklists', 'action' => 'index'));
+                    $this->redirect(array('controller' => 'eventBlocklists', 'action' => 'index'));
                 }
             }
         } else {
             $ids = json_decode($this->request->query('ids'), true);
             if (empty($ids)) {
-                throw new NotFoundException(__('Invalid event IDs.'));
+                throw new NotFoundException(__('Invalid event blocklist IDs.'));
 
             }
             $this->set('event_ids', $ids);

app/Controller/EventGraphController.php

@@ -13,7 +13,7 @@ class EventGraphController extends AppController
         parent::beforeFilter();
     }
 
-    public function view($event_id = false)
+    public function view($event_id = false, $graph_id = null)
     {
         if ($event_id === false) {
             throw new MethodNotAllowedException(__('No event ID set.'));
@@ -40,12 +40,16 @@ class EventGraphController extends AppController
         }
 
         // fetch eventGraphs
+        $conditions = [
+            'EventGraph.event_id' => $event_id,
+            'EventGraph.org_id' => $org_id
+        ];
+        if (!is_null($graph_id)) {
+            $conditions['EventGraph.id'] = $graph_id;
+        }
         $eventGraphs = $this->EventGraph->find('all', array(
             'order' => 'EventGraph.timestamp DESC',
-            'conditions' => array(
-                'EventGraph.event_id' => $event_id,
-                'EventGraph.org_id' => $org_id
-            ),
+            'conditions' => $conditions,
             'contain' => array(
                 'User' => array(
                     'fields' => array(

app/Controller/EventReportsController.php

@@ -0,0 +1,349 @@
+<?php
+
+App::uses('AppController', 'Controller');
+
+class EventReportsController extends AppController
+{
+    public $components = array(
+        'Security',
+        'AdminCrud',
+        'RequestHandler'
+    );
+
+    public $paginate = array(
+        'limit' => 60,
+        'order' => array(
+                'EventReport.event_id' => 'ASC',
+                'EventReport.name' => 'ASC'
+        ),
+        'recursive' => -1,
+        'contain' => array(
+            'SharingGroup' => array('fields' => array('id', 'name', 'uuid')),
+            'Event' => array(
+                'fields' =>  array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.info', 'Event.user_id', 'Event.date'),
+                'Orgc' => array('fields' => array('Orgc.id', 'Orgc.name')),
+                'Org' => array('fields' => array('Org.id', 'Org.name'))
+            )
+        )
+    );
+
+    public function add($eventId = false)
+    {
+        if ($this->request->is('get') && $this->_isRest()) {
+            return $this->RestResponse->describe('EventReports', 'add', false, $this->response->type());
+        }
+        if ($eventId === false) {
+            throw new MethodNotAllowedException(__('No event ID set.'));
+        }
+        $event = $this->__canModifyReport($eventId);
+        if ($this->request->is('post') || $this->request->is('put')) {
+            if (!isset($this->request->data['EventReport'])) {
+                $this->request->data['EventReport'] = $this->request->data;
+            }
+            $report = $this->request->data;
+            $errors = $this->EventReport->addReport($this->Auth->user(), $report, $eventId);
+            $redirectTarget = array('controller' => 'events', 'action' => 'view', $eventId);
+            if (!empty($errors)) {
+                return $this->__getFailResponseBasedOnContext($errors, array(), 'add', $this->EventReport->id, $redirectTarget);
+            } else {
+                $successMessage = __('Report saved.');
+                $report = $this->EventReport->simpleFetchById($this->Auth->user(), $this->EventReport->id);
+                return $this->__getSuccessResponseBasedOnContext($successMessage, $report, 'add', false, $redirectTarget);
+            }
+        }
+        $this->set('event_id', $eventId);
+        $this->set('action', 'add');
+        $this->__injectDistributionLevelToViewContext();
+        $this->__injectSharingGroupsDataToViewContext();
+    }
+
+
+    public function view($reportId, $ajax=false)
+    {
+        $report = $this->EventReport->simpleFetchById($this->Auth->user(), $reportId);
+        if ($this->_isRest()) {
+            return $this->RestResponse->viewData($report, $this->response->type());
+        }
+        $this->set('ajax', $ajax);
+        $this->set('id', $reportId);
+        $this->set('report', $report);
+        $this->__injectDistributionLevelToViewContext();
+        $this->__injectPermissionsToViewContext($this->Auth->user(), $report);
+    }
+
+    public function getProxyMISPElements($reportId)
+    {
+        if (!$this->_isRest()) {
+            throw new MethodNotAllowedException(__('This function can only be reached via the API.'));
+        }
+        $report = $this->EventReport->simpleFetchById($this->Auth->user(), $reportId);
+        $proxyMISPElements = $this->EventReport->getProxyMISPElements($this->Auth->user(), $report['EventReport']['event_id']);
+        return $this->RestResponse->viewData($proxyMISPElements, $this->response->type());
+    }
+
+    public function viewSummary($reportId)
+    {
+        $report = $this->EventReport->simpleFetchById($this->Auth->user(), $reportId);
+        $this->set('id', $reportId);
+        $this->set('report', $report);
+        $this->__injectDistributionLevelToViewContext();
+        $this->__injectPermissionsToViewContext($this->Auth->user(), $report);
+    }
+
+    public function edit($id)
+    {
+        $savedReport = $this->EventReport->fetchIfAuthorized($this->Auth->user(), $id, 'edit', $throwErrors=true, $full=true);
+        if ($this->request->is('post') || $this->request->is('put')) {
+            $newReport = $this->request->data;
+            $newReport = $this->__applyDataFromSavedReport($newReport, $savedReport);
+            $errors = $this->EventReport->editReport($this->Auth->user(), $newReport, $savedReport['EventReport']['event_id']);
+            $redirectTarget = array('controller' => 'eventReports', 'action' => 'view', $id);
+            if (!empty($errors)) {
+                return $this->__getFailResponseBasedOnContext($validationErrors, array(), 'edit', $id, $redirectTarget);
+            } else {
+                $successMessage = __('Report saved.');
+                $report = $this->EventReport->simpleFetchById($this->Auth->user(), $this->EventReport->id);
+                return $this->__getSuccessResponseBasedOnContext($successMessage, $report, 'edit', $id, $redirectTarget);
+            }
+        } else {
+            $this->request->data = $savedReport;
+        }
+
+        $this->set('id', $savedReport['EventReport']['id']);
+        $this->set('event_id', $savedReport['EventReport']['event_id']);
+        $this->set('action', 'edit');
+        $this->__injectDistributionLevelToViewContext();
+        $this->__injectSharingGroupsDataToViewContext();
+        $this->render('add');
+    }
+
+    public function delete($id, $hard=false)
+    {
+        $report = $this->EventReport->fetchIfAuthorized($this->Auth->user(), $id, 'delete', $throwErrors=true, $full=false);
+        if ($this->request->is('post')) {
+            $errors = $this->EventReport->deleteReport($this->Auth->user(), $report, $hard=$hard);
+            $redirectTarget = $this->referer();
+            if (empty($errors)) {
+                $successMessage = __('Event Report %s %s deleted', $id, $hard ? __('hard') : __('soft'));
+                $report = $hard ? null : $this->EventReport->simpleFetchById($this->Auth->user(), $id);
+                return $this->__getSuccessResponseBasedOnContext($successMessage, $report, 'delete', $id, $redirectTarget);
+            } else {
+                $errorMessage = __('Event Report %s could not be %s deleted.%sReasons: %s', $id, $hard ? __('hard') : __('soft'), PHP_EOL, json_encode($errors));
+                return $this->__getFailResponseBasedOnContext($errorMessage, array(), 'edit', $id, $redirectTarget);
+            }
+        } else {
+            if (!$this->request->is('ajax')) {
+                throw new MethodNotAllowedException(__('This function can only be reached via AJAX.'));
+            } else {
+                $this->layout = 'ajax';
+                $this->set('report', $report);
+                $this->render('ajax/delete');
+            }
+        }
+    }
+
+    public function restore($id)
+    {
+        $report = $this->EventReport->fetchIfAuthorized($this->Auth->user(), $id, 'edit', $throwErrors=true, $full=false);
+        if ($this->request->is('post')) {
+            $errors = $this->EventReport->restoreReport($this->Auth->user(), $id);
+            $redirectTarget = $this->referer();
+            if (empty($errors)) {
+                $successMessage = __('Event Report %s restored', $id);
+                $report = $this->EventReport->simpleFetchById($this->Auth->user(), $id);
+                return $this->__getSuccessResponseBasedOnContext($successMessage, $report, 'restore', $id, $redirectTarget);
+            } else {
+                $errorMessage = __('Event Report %s could not be %s restored.%sReasons: %s', $id, PHP_EOL, json_encode($errors));
+                return $this->__getFailResponseBasedOnContext($errorMessage, array(), 'restore', $id, $redirectTarget);
+            }
+        } else {
+            if (!$this->request->is('ajax')) {
+                throw new MethodNotAllowedException(__('This function can only be reached via AJAX.'));
+            } else {
+                $this->layout = 'ajax';
+                $this->set('report', $report);
+            }
+        }
+    }
+
+    public function index()
+    {
+        $filters = $this->IndexFilter->harvestParameters(['event_id', 'value', 'context', 'index_for_event', 'extended_event']);
+        $filters['embedded_view']  = $this->request->is('ajax');
+        $compiledConditions = $this->__generateIndexConditions($filters);
+        if ($this->_isRest()) {
+            $reports = $this->EventReport->find('all', [
+                'recursive' => -1,
+                'conditions' => $compiledConditions,
+                'contain' => $this->EventReport->defaultContain,
+            ]);
+            return $this->RestResponse->viewData($reports, $this->response->type());
+        } else {
+            $this->paginate['conditions']['AND'][] = $compiledConditions;
+            $reports = $this->paginate();
+            $this->set('reports', $reports);
+            $this->__injectIndexVariablesToViewContext($filters);
+            if (!empty($filters['index_for_event'])) {
+                $this->set('extendedEvent', !empty($filters['extended_event']));
+                $this->render('ajax/indexForEvent');
+            }
+        }
+    }
+
+    private function __generateIndexConditions($filters = [])
+    {
+        $aclConditions = $this->EventReport->buildACLConditions($this->Auth->user());
+        $eventConditions = [];
+        if (!empty($filters['event_id'])) {
+            $extendingEventIds = [];
+            if (!empty($filters['extended_event'])) {
+                $extendingEventIds = $this->EventReport->Event->getExtendingEventIdsFromEvent($this->Auth->user(), $filters['event_id']);
+            }
+            $eventConditions = ['EventReport.event_id' => array_merge([$filters['event_id']], $extendingEventIds)];
+        }
+
+        $contextConditions = [];
+        if (empty($filters['context'])) {
+            $filters['context'] = 'default';
+        }
+        if ($filters['context'] == 'deleted') {
+            $contextConditions['EventReport.deleted'] = true;
+        } elseif ($filters['context'] == 'default') {
+            $contextConditions['EventReport.deleted'] = false;
+        }
+        $searchConditions = [];
+        if (empty($filters['value'])) {
+            $filters['value'] = '';
+        } else {
+            $searchall = '%' . strtolower($filters['value']) . '%';
+            $searchConditions = array(
+                'OR' => array(
+                    'LOWER(EventReport.name) LIKE' => $searchall,
+                    'LOWER(EventReport.content) LIKE' => $searchall,
+                    'EventReport.id' => $searchall,
+                    'EventReport.uuid' => $searchall
+                )
+            );
+        }
+        $compiledConditions = [
+            'AND' => [
+                $aclConditions,
+                $eventConditions,
+                $contextConditions,
+                $searchConditions,
+            ]
+        ];
+        return $compiledConditions;
+    }
+
+    private function __getSuccessResponseBasedOnContext($message, $data = null, $action = '', $id = false, $redirect = array())
+    {
+        if ($this->_isRest()) {
+            if (!is_null($data)) {
+                return $this->RestResponse->viewData($data, $this->response->type());
+            } else {
+                return $this->RestResponse->saveSuccessResponse($this->EventReport->alias, $action, $id, false, $message);
+            }
+        } elseif ($this->request->is('ajax')) {
+            return $this->RestResponse->saveSuccessResponse($this->EventReport->alias, $action, $id, false, $message, $data);
+        } else {
+            $this->Flash->success($message);
+            $this->redirect($redirect);
+        }
+        return;
+    }
+
+    private function __getFailResponseBasedOnContext($message, $data = null, $action = '', $id = false, $redirect = array())
+    {
+        if (is_array($message)) {
+            $message = implode(', ', $message);
+        }
+        if ($this->_isRest()) {
+            if (!is_null($data)) {
+                return $this->RestResponse->viewData($data, $this->response->type());
+            } else {
+                return $this->RestResponse->saveFailResponse('EventReport', $action, $id, $message, false);
+            }
+        } elseif ($this->request->is('ajax')) {
+            return $this->RestResponse->saveFailResponse('EventReport', $action, $id, $message, false, $data);
+        } else {
+            $this->Flash->error($message);
+            $this->redirect($this->referer());
+        }
+        return;
+    }
+
+    private function __injectIndexVariablesToViewContext($filters)
+    {
+        if (!empty($filters['context'])) {
+            $this->set('context', $filters['context']);
+        } else {
+            $this->set('context', 'default');
+        }
+        if (!empty($filters['event_id'])) {
+            $this->set('event_id', $filters['event_id']);
+        }
+        if (isset($filters['embedded_view'])) {
+            $this->set('embedded_view', $filters['embedded_view']);
+        } else {
+            $this->set('embedded_view', false);
+        }
+        if (!empty($filters['value'])) {
+            $this->set('searchall', $filters['value']);
+        } else {
+            $this->set('searchall', '');
+        }
+        $this->__injectDistributionLevelToViewContext();
+    }
+
+    private function __injectDistributionLevelToViewContext()
+    {
+        $distributionLevels = $this->EventReport->Event->Attribute->distributionLevels;
+        $this->set('distributionLevels', $distributionLevels);
+        $initialDistribution = 5;
+        $configuredDistribution = Configure::check('MISP.default_attribute_distribution');
+        if ($configuredDistribution != null && $configuredDistribution != 'event') {
+            $initialDistribution = $configuredDistribution;
+        }
+        $this->set('initialDistribution', $initialDistribution);
+    }
+
+    private function __injectSharingGroupsDataToViewContext()
+    {
+        $sgs = $this->EventReport->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1);
+        $this->set('sharingGroups', $sgs);
+    }
+
+    private function __injectPermissionsToViewContext($user, $report)
+    {
+        $canEdit = $this->EventReport->canEditReport($user, $report) === true;
+        $this->set('canEdit', $canEdit);
+    }
+
+    private function __canModifyReport($eventId)
+    {
+        $event = $this->EventReport->Event->fetchSimpleEvent($this->Auth->user(), $eventId, array());
+        if (empty($event)) {
+            throw new NotFoundException(__('Invalid event'));
+        }
+        if (!$this->__canModifyEvent($event)) {
+            throw new ForbiddenException(__('You do not have permission to do that.'));
+        }
+        return $event;
+    }
+
+    private function __applyDataFromSavedReport($newReport, $savedReport)
+    {
+        if (!isset($newReport['EventReport'])) {
+            $newReport = array('EventReport' => $newReport);
+        }
+        $fieldList = $this->EventReport->captureFields;
+        $ignoreFieldList = ['id', 'uuid', 'event_id', 'deleted'];
+        foreach ($fieldList as $field) {
+            if (!in_array($field, $ignoreFieldList) && isset($newReport['EventReport'][$field])) {
+                $savedReport['EventReport'][$field] = $newReport['EventReport'][$field];
+            }
+        }
+        return $savedReport;
+    }
+}

app/Controller/EventsController.php

@@ -95,32 +95,7 @@ class EventsController extends AppController
 
         // if not admin or own org, check private as well..
         if (!$this->_isSiteAdmin() && in_array($this->action, $this->paginationFunctions)) {
-            $sgids = $this->Event->cacheSgids($this->Auth->user(), true);
-            $conditions = array(
-                'AND' => array(
-                    array(
-                        "OR" => array(
-                            array(
-                                'Event.org_id' => $this->Auth->user('org_id')
-                            ),
-                            array(
-                                'AND' => array(
-                                        'Event.distribution >' => 0,
-                                        'Event.distribution <' => 4,
-                                        Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(),
-                                ),
-                            ),
-                            array(
-                                'AND' => array(
-                                        'Event.distribution' => 4,
-                                        'Event.sharing_group_id' => $sgids,
-                                        Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(),
-                                ),
-                            )
-                        )
-                    )
-                )
-            );
+            $conditions = $this->Event->createEventConditions($this->Auth->user());
             if ($this->userRole['perm_sync'] && $this->Auth->user('Server')['push_rules']) {
                 $conditions['AND'][] = $this->Event->filterRulesToConditions($this->Auth->user('Server')['push_rules']);
             }
@@ -128,62 +103,56 @@ class EventsController extends AppController
         }
     }
 
+    /**
+     * @param string $value
+     * @return array[]
+     */
     private function __filterOnAttributeValue($value)
     {
         // dissect the value
-        $pieces = explode('|', $value);
+        $pieces = explode('|', strtolower($value));
         $include = array();
         $exclude = array();
-        $includeIDs = array();
-        $excludeIDs = array();
+
         foreach ($pieces as $piece) {
-            if ($piece[0] == '!') {
-                $exclude[] =  '%' . strtolower(substr($piece, 1)) . '%';
+            if ($piece[0] === '!') {
+                $exclude[] =  '%' . substr($piece, 1) . '%';
             } else {
-                $include[] = '%' . strtolower($piece) . '%';
+                $include[] = "%$piece%";
             }
         }
+
+        $includeIDs = array();
         if (!empty($include)) {
-            // get all of the attributes that should be included
-            $includeQuery = array(
-                    'recursive' => -1,
-                    'fields' => array('id', 'event_id', 'distribution', 'value1', 'value2'),
-                    'conditions' => array(),
-            );
+            $includeConditions = [];
             foreach ($include as $i) {
-                $includeQuery['conditions']['OR'][] = array('lower(Attribute.value1) LIKE' => $i);
-                $includeQuery['conditions']['OR'][] = array('lower(Attribute.value2) LIKE' => $i);
+                $includeConditions['OR'][] = array('lower(Attribute.value1) LIKE' => $i);
+                $includeConditions['OR'][] = array('lower(Attribute.value2) LIKE' => $i);
             }
-            $includeQuery['conditions']['AND'][] = array('Attribute.deleted' => 0);
-            $includeHits = $this->Event->Attribute->find('all', $includeQuery);
 
-            // convert it into an array that uses the event ID as a key
-            foreach ($includeHits as $iH) {
-                $includeIDs[$iH['Attribute']['event_id']][] = array('attribute_id' => $iH['Attribute']['id'], 'distribution' => $iH['Attribute']['distribution']);
-            }
+            $includeIDs = array_values($this->Event->Attribute->fetchAttributes($this->Auth->user(), array(
+                'conditions' => $includeConditions,
+                'flatten' => true,
+                'event_ids' => true,
+                'list' => true,
+            )));
         }
 
+        $excludeIDs = array();
         if (!empty($exclude)) {
-            // get all of the attributes that should be excluded
-            $excludeQuery = array(
-                'recursive' => -1,
-                'fields' => array('id', 'event_id', 'distribution', 'value1', 'value2'),
-                'conditions' => array(),
-            );
+            $excludeConditions = [];
             foreach ($exclude as $e) {
-                $excludeQuery['conditions']['OR'][] = array('lower(Attribute.value1) LIKE' => $e);
-                $excludeQuery['conditions']['OR'][] = array('lower(Attribute.value2) LIKE' => $e);
+                $excludeConditions['OR'][] = array('lower(Attribute.value1) LIKE' => $e);
+                $excludeConditions['OR'][] = array('lower(Attribute.value2) LIKE' => $e);
             }
-            $excludeQuery['conditions']['AND'][] = array('Attribute.deleted' => 0);
-            $excludeHits = $this->Event->Attribute->find('all', $excludeQuery);
 
-            // convert it into an array that uses the event ID as a key
-            foreach ($excludeHits as $eH) {
-                $excludeIDs[$eH['Attribute']['event_id']][] = array('attribute_id' => $eH['Attribute']['id'], 'distribution' => $eH['Attribute']['distribution']);
-            }
+            $excludeIDs = array_values($this->Event->Attribute->fetchAttributes($this->Auth->user(), array(
+                'conditions' => $excludeConditions,
+                'flatten' => true,
+                'event_ids' => true,
+                'list' => true,
+            )));
         }
-        $includeIDs = array_keys($includeIDs);
-        $excludeIDs = array_keys($excludeIDs);
         // return -1 as the only value in includedIDs if both arrays are empty. This will mean that no events will be shown if there was no hit
         if (empty($includeIDs) && empty($excludeIDs)) {
             $includeIDs[] = -1;
@@ -191,6 +160,10 @@ class EventsController extends AppController
         return array($includeIDs, $excludeIDs);
     }
 
+    /**
+     * @param string|array $value
+     * @return array Event ID that match filter
+     */
     private function __quickFilter($value)
     {
         if (!is_array($value)) {
@@ -201,7 +174,6 @@ class EventsController extends AppController
             $values[] = '%' . strtolower($v) . '%';
         }
 
-        $result = array();
         // get all of the attributes that have a hit on the search term, in either the value or the comment field
         // This is not perfect, the search will be case insensitive, but value1 and value2 are searched separately. lower() doesn't seem to work on virtualfields
         $subconditions = array();
@@ -211,32 +183,16 @@ class EventsController extends AppController
             $subconditions[] = array('lower(Attribute.comment) LIKE' => $v);
         }
         $conditions = array(
-            'AND' => array(
-                'OR' => $subconditions,
-                'Attribute.deleted' => 0
-            )
+            'OR' => $subconditions,
         );
         $attributeHits = $this->Event->Attribute->fetchAttributes($this->Auth->user(), array(
-                'conditions' => $conditions,
-                'fields' => array('event_id', 'comment', 'distribution', 'value1', 'value2'),
-                'flatten' => 1
-        ));
-        // rearrange the data into an array where the keys are the event IDs
-        $eventsWithAttributeHits = array();
-        foreach ($attributeHits as $aH) {
-            $eventsWithAttributeHits[$aH['Attribute']['event_id']][] = $aH['Attribute'];
-        }
-
-        // Using the keys from the previously obtained ordered array, let's fetch all of the events involved
-        $events = $this->Event->find('all', array(
-                'recursive' => -1,
-                'fields' => array('id', 'distribution', 'org_id'),
-                'conditions' => array('id' => array_keys($eventsWithAttributeHits)),
+            'conditions' => $conditions,
+            'flatten' => 1,
+            'event_ids' => true,
+            'list' => true,
         ));
 
-        foreach ($events as $event) {
-            $result[] = $event['Event']['id'];
-        }
+        $result = array_values($attributeHits);
 
         // we now have a list of event IDs that match on an attribute level, and the user can see it. Let's also find all of the events that match on other criteria!
         // What is interesting here is that we no longer have to worry about the event's releasability. With attributes this was a different case,
@@ -250,9 +206,9 @@ class EventsController extends AppController
             $subconditions[] = array('lower(name) LIKE' => $v);
         }
         $tags = $this->Event->EventTag->Tag->find('all', array(
-                'conditions' => $subconditions,
-                'fields' => array('name', 'id'),
-                'contain' => array('EventTag', 'AttributeTag'),
+            'conditions' => $subconditions,
+            'fields' => array('id'),
+            'contain' => array('EventTag' => ['fields' => 'event_id'], 'AttributeTag' => ['fields' => 'event_id']),
         ));
         foreach ($tags as $tag) {
             foreach ($tag['EventTag'] as $eventTag) {
@@ -272,12 +228,13 @@ class EventsController extends AppController
         foreach ($values as $v) {
             $subconditions[] = array('lower(name) LIKE' => $v);
         }
-        $conditions = array();
         $orgs = $this->Event->Org->find('list', array(
-                'conditions' => $subconditions,
-                'recursive' => -1,
-                'fields' => array('id')
+            'conditions' => $subconditions,
+            'recursive' => -1,
+            'fields' => array('id')
         ));
+
+        $conditions = empty($result) ? [] : ['NOT' => ['id' => $result]]; // Do not include events that we already found
         foreach ($values as $v) {
             $conditions['OR'][] = array('lower(info) LIKE' => $v);
             $conditions['OR'][] = array('lower(uuid) LIKE' => $v);
@@ -285,15 +242,13 @@ class EventsController extends AppController
         if (!empty($orgs)) {
             $conditions['OR']['orgc_id'] = array_values($orgs);
         }
-        $otherEvents = $this->Event->find('all', array(
-                'recursive' => -1,
-                'fields' => array('id', 'orgc_id', 'info', 'uuid'),
-                'conditions' => $conditions,
+        $otherEvents = $this->Event->find('list', array(
+            'recursive' => -1,
+            'fields' => array('id'),
+            'conditions' => $conditions,
         ));
-        foreach ($otherEvents as $oE) {
-            if (!in_array($oE['Event']['id'], $result)) {
-                $result[] = $oE['Event']['id'];
-            }
+        foreach ($otherEvents as $eventId) {
+            $result[] = $eventId;
         }
         return $result;
     }
@@ -513,6 +468,7 @@ class EventsController extends AppController
                         $filterString = "";
                         $expectOR = false;
                         $setOR = false;
+                        $tagRules = [];
                         foreach ($pieces as $piece) {
                             if ($piece[0] == '!') {
                                 if (is_numeric(substr($piece, 1))) {
@@ -543,7 +499,7 @@ class EventsController extends AppController
                                     foreach ($block as $b) {
                                         $sqlSubQuery .= $b['EventTag']['event_id'] . ',';
                                     }
-                                    $this->paginate['conditions']['AND'][] = substr($sqlSubQuery, 0, -1) . ')';
+                                    $tagRules['AND'][] = substr($sqlSubQuery, 0, -1) . ')';
                                 }
                                 if ($filterString != "") {
                                     $filterString .= "|";
@@ -562,7 +518,6 @@ class EventsController extends AppController
                                         'fields' => array('id', 'name'),
                                         'recursive' => -1,
                                 ));
-
                                 if (empty($tagName)) {
                                     if ($filterString != "") {
                                         $filterString .= "|";
@@ -582,7 +537,7 @@ class EventsController extends AppController
                                         $setOR = true;
                                         $sqlSubQuery .= $a['EventTag']['event_id'] . ',';
                                     }
-                                    $this->paginate['conditions']['AND']['OR'][] = substr($sqlSubQuery, 0, -1) . ')';
+                                    $tagRules['OR'][] = substr($sqlSubQuery, 0, -1) . ')';
                                 }
                                 if ($filterString != "") {
                                     $filterString .= "|";
@@ -590,6 +545,7 @@ class EventsController extends AppController
                                 $filterString .= isset($tagName['Tag']['name']) ? $tagName['Tag']['name'] : $piece;
                             }
                         }
+                        $this->paginate['conditions']['AND'][] = $tagRules;
                         // If we have a list of OR-d arguments, we expect to end up with a list of allowed event IDs
                         // If we don't however, it means that none of the tags was found. To prevent displaying the entire event index in this case:
                         if ($expectOR && !$setOR) {
@@ -671,7 +627,14 @@ class EventsController extends AppController
                         $v = $filterString;
                         break;
                     case 'minimal':
-                        $this->paginate['conditions']['AND'][] = array('NOT' => array('Event.attribute_count' => 0));
+                        $tableName = $this->Event->EventReport->table;
+                        $eventReportQuery = sprintf('EXISTS (SELECT id, deleted FROM %s WHERE %s.event_id = Event.id and %s.deleted = 0)', $tableName, $tableName, $tableName);
+                        $this->paginate['conditions']['AND'][] = [
+                            'OR' => [
+                                ['Event.attribute_count >' => 0],
+                                [$eventReportQuery]
+                            ]
+                        ];
                         break;
                     default:
                         continue 2;
@@ -1028,48 +991,41 @@ class EventsController extends AppController
         $this->layout = 'ajax';
     }
 
-    /*
+    /**
      * Search for a value on an attribute level for a specific field.
-     * $attribute : (array) an attribute
-     * $fields : (array) list of keys in attribute to search in
-     * $searchValue : Values to search ( '|' is the separator)
-     * returns true on match
+     *
+     * @param array $attribute An attribute
+     * @param array $fields List of keys in attribute to search in
+     * @param string $searchValue Values to search ( '|' is the separator)
+     * @return bool Returns true on match
      */
     private function __valueInFieldAttribute($attribute, $fields, $searchValue)
     {
-        foreach ($attribute as $k => $v) { // look in attributes line
-            if (is_string($v)) {
-                foreach ($fields as $field) {
-                    if (strpos(".", $field) === false) { // check sub array after
-                        // check for key in attribut
-                        if (isset($attribute[$field])) {
-                            $temp_value = strtolower($attribute[$field]);
-                            $temp_search = strtolower($searchValue);
-                            $temp_searches = explode('|', $temp_search);
-                            foreach ($temp_searches as $s) {
-                                if (strpos($temp_value, $s) !==false) {
-                                    return true;
-                                }
-                            }
+        $searchParts = explode('|', mb_strtolower($searchValue));
+
+        foreach ($fields as $field) {
+            if (strpos($field, 'Tag') === 0) {
+                if (empty($attribute['AttributeTag'])) {
+                    continue;
+                }
+                $fieldValues = Hash::extract($attribute, 'AttributeTag.{n}.' . $field);
+                foreach ($fieldValues as $fieldValue) {
+                    $fieldValue = mb_strtolower($fieldValue);
+                    foreach ($searchParts as $s) {
+                        if (strpos($fieldValue, $s) !== false) {
+                            return true;
                         }
                     }
                 }
             } else {
-                // check for tag in attribut maybe for other thing later
-                if ($k === 'AttributeTag') {
-                    foreach ($v as $tag) {
-                        foreach ($fields as $field) {
-                            if (strpos(strtolower($field), "tag.") !== false) { // check sub array
-                                $tagKey = explode('tag.', strtolower($field))[1];
-                                if (isset($tag['Tag'][$tagKey])) {
-                                    $temp_value = strtolower($tag['Tag'][$tagKey]);
-                                    $temp_search = strtolower($searchValue);
-                                    if (strpos($temp_value, $temp_search) !==false) {
-                                        return true;
-                                    }
-                                }
-                            }
-                        }
+                $fieldValue = isset($attribute[$field]) ? $attribute[$field] : null;
+                if (empty($fieldValue)) {
+                    continue;
+                }
+                $fieldValue = mb_strtolower($fieldValue);
+                foreach ($searchParts as $s) {
+                    if (strpos($fieldValue, $s) !== false) {
+                        return true;
                     }
                 }
             }
@@ -1114,6 +1070,7 @@ class EventsController extends AppController
             $conditions['to_ids'] = $filters['toIDS'] == 2 ? 0 : 1;
         }
         $conditions['includeFeedCorrelations'] = true;
+        $conditions['includeWarninglistHits'] = true;
         if (!isset($filters['includeServerCorrelations'])) {
             $conditions['includeServerCorrelations'] = 1;
             if ($this->_isRest()) {
@@ -1125,6 +1082,7 @@ class EventsController extends AppController
         $conditions['includeAllTags'] = true;
         $conditions['includeGranularCorrelations'] = 1;
         $conditions['includeEventCorrelations'] = false;
+        $conditions['noEventReports'] = true; // event reports for view are loaded dynamically
         if (!empty($filters['includeRelatedTags'])) {
             $this->set('includeRelatedTags', 1);
             $conditions['includeRelatedTags'] = 1;
@@ -1297,10 +1255,6 @@ class EventsController extends AppController
         $warningTagConflicts = array();
         $filters = $this->_harvestParameters($filterData, $exception);
 
-        $this->loadModel('GalaxyCluster');
-        if (!$this->_isRest()) {
-            //$attack = $this->GalaxyCluster->Galaxy->constructAttackReport($event);
-        }
         $emptyEvent = (empty($event['Object']) && empty($event['Attribute']));
         $this->set('emptyEvent', $emptyEvent);
         $attributeCount = isset($event['Attribute']) ? count($event['Attribute']) : 0;
@@ -1322,6 +1276,7 @@ class EventsController extends AppController
         $this->set('object_count', $objectCount);
         // set the data for the contributors / history field
         $contributors = $this->Event->ShadowAttribute->getEventContributors($event['Event']['id']);
+        $this->set('contributors', $contributors);
         if ($this->userRole['perm_publish'] && $event['Event']['orgc_id'] == $this->Auth->user('org_id')) {
             $proposalStatus = false;
             if (isset($event['ShadowAttribute']) && !empty($event['ShadowAttribute'])) {
@@ -1520,7 +1475,6 @@ class EventsController extends AppController
             $cortex_modules = $this->Module->getEnabledModules($this->Auth->user(), false, 'Cortex');
             $this->set('cortex_modules', $cortex_modules);
         }
-        $this->set('contributors', $contributors);
         $this->set('typeGroups', array_keys($this->Event->Attribute->typeGroupings));
         $attributeUri = $this->baseurl . '/events/viewEventAttributes/' . $event['Event']['id'];
         foreach ($this->params->named as $k => $v) {
@@ -1551,7 +1505,8 @@ class EventsController extends AppController
         $this->set('advancedFilteringActive', $advancedFiltering['active'] ? 1 : 0);
         $this->set('advancedFilteringActiveRules', $advancedFiltering['activeRules']);
         $this->set('defaultFilteringRules', $this->defaultFilteringRules);
-        $this->set('mitreAttackGalaxyId', $this->Event->GalaxyCluster->Galaxy->getMitreAttackGalaxyId());
+        $this->loadModel('Galaxy');
+        $this->set('mitreAttackGalaxyId', $this->Galaxy->getMitreAttackGalaxyId());
         $this->set('modificationMapCSV', $modificationMapCSV);
         $this->set('title_for_layout', __('Event #%s', $event['Event']['id']));
     }
@@ -1566,10 +1521,11 @@ class EventsController extends AppController
             throw new NotFoundException(__('Invalid event'));
         }
 
-        if (!$this->_isRest()) {
-            $conditions['includeAllTags'] = true;
-        } else {
+        if ($this->_isRest()) {
             $conditions['includeAttachments'] = true;
+        } else {
+            $conditions['includeAllTags'] = true;
+            $conditions['noEventReports'] = true; // event reports for view are loaded dynamically
         }
         $deleted = 0;
         if (isset($this->params['named']['deleted'])) {
@@ -1611,6 +1567,7 @@ class EventsController extends AppController
             $this->set('extended', 0);
         }
         $conditions['excludeLocalTags'] = false;
+        $conditions['includeWarninglistHits'] = true;
         if (isset($this->params['named']['excludeLocalTags'])) {
             $conditions['excludeLocalTags'] = $this->params['named']['excludeLocalTags'];
         }
@@ -1629,37 +1586,32 @@ class EventsController extends AppController
             $conditions['includeServerCorrelations'] = $this->params['named']['includeServerCorrelations'];
         }
         $results = $this->Event->fetchEvent($this->Auth->user(), $conditions);
-        if (!empty($this->params['named']['includeGranularCorrelations'])) {
-            foreach ($results as $k => $event) {
-                if (!empty($event['RelatedAttribute'])) {
-                    foreach ($event['RelatedAttribute'] as $attribute_id => $relation) {
-                        foreach ($event['Attribute'] as $k2 => $attribute) {
-                            if ((int)$attribute['id'] == $attribute_id) {
-                                $results[$k]['Attribute'][$k2]['RelatedAttribute'][] = $relation;
-                                break 2;
-                            }
-                        }
-                        foreach ($event['Object'] as $k2 => $object) {
-                            foreach ($object['Attribute'] as $k3 => $attribute) {
-                                if ((int)$attribute['id'] == $attribute_id) {
-                                    $results[$k]['Object'][$k2]['Attribute'][$k3]['RelatedAttribute'][] = $relation;
-                                    break 3;
-                                }
-                            }
+        if (empty($results)) {
+            throw new NotFoundException(__('Invalid event'));
+        }
+        $event = $results[0];
+
+        // Attach related attributes to proper attribute
+        if (!empty($this->params['named']['includeGranularCorrelations']) && !empty($event['RelatedAttribute'])) {
+            foreach ($event['RelatedAttribute'] as $attribute_id => $relation) {
+                foreach ($event['Attribute'] as $k2 => $attribute) {
+                    if ((int)$attribute['id'] == $attribute_id) {
+                        $event['Attribute'][$k2]['RelatedAttribute'][] = $relation;
+                        break 2;
+                    }
+                }
+                foreach ($event['Object'] as $k2 => $object) {
+                    foreach ($object['Attribute'] as $k3 => $attribute) {
+                        if ((int)$attribute['id'] == $attribute_id) {
+                            $event['Object'][$k2]['Attribute'][$k3]['RelatedAttribute'][] = $relation;
+                            break 3;
                         }
                     }
                 }
             }
         }
-        if (empty($results)) {
-            throw new NotFoundException(__('Invalid event'));
-        }
-        $event = $results[0];
+
         $this->Event->id = $event['Event']['id'];
-        //if the current user is an org admin AND event belongs to his/her org, fetch also the event creator info
-        if ($this->userRole['perm_admin'] && !$this->_isSiteAdmin() && ($event['Org']['id'] == $this->Auth->user('org_id'))) {
-            $event['User']['email'] = $this->User->field('email', array('id' => $event['Event']['user_id']));
-        }
         if (isset($this->params['named']['searchFor']) && $this->params['named']['searchFor'] !== '') {
             $this->__applyQueryString($event, $this->params['named']['searchFor']);
         }
@@ -1672,11 +1624,11 @@ class EventsController extends AppController
 
         if ($this->_isRest()) {
             $this->set('event', $event);
-        }
-        $this->set('deleted', isset($deleted) ? ($deleted == 2 ? 0 : 1) : 0);
-        $this->set('includeRelatedTags', (!empty($this->params['named']['includeRelatedTags'])) ? 1 : 0);
-        $this->set('includeDecayScore', (!empty($this->params['named']['includeDecayScore'])) ? 1 : 0);
-        if (!$this->_isRest()) {
+        } else {
+            $this->set('deleted', isset($deleted) ? ($deleted == 2 ? 0 : 1) : 0);
+            $this->set('includeRelatedTags', (!empty($this->params['named']['includeRelatedTags'])) ? 1 : 0);
+            $this->set('includeDecayScore', (!empty($this->params['named']['includeDecayScore'])) ? 1 : 0);
+
             if ($this->_isSiteAdmin() && $event['Event']['orgc_id'] !== $this->Auth->user('org_id')) {
                 $this->Flash->info(__('You are currently logged in as a site administrator and about to edit an event not belonging to your organisation. This goes against the sharing model of MISP. Use a normal user account for day to day work.'));
             }
@@ -1766,12 +1718,13 @@ class EventsController extends AppController
         $this->redirect(array('controller' => 'events', 'action' => 'view', $eventId, true, $eventId));
     }
 
-    private function __applyQueryString(&$event, $searchFor, $filterColumnsOverwrite=false) {
+    private function __applyQueryString(&$event, $searchFor, $filterColumnsOverwrite=false)
+    {
         // filtering on specific columns is specified
         if ($filterColumnsOverwrite !== false) {
             $filterValue = array_map('trim', explode(",", $filterColumnsOverwrite));
         } else {
-            $filterColumnsOverwrite = empty(Configure::read('MISP.event_view_filter_fields')) ? 'id, uuid, value, comment, type, category, Tag.name' : Configure::read('MISP.event_view_filter_fields');
+            $filterColumnsOverwrite = Configure::read('MISP.event_view_filter_fields') ?: 'id,uuid,value,comment,type,category,Tag.name';
             $filterValue = array_map('trim', explode(",", $filterColumnsOverwrite));
             $validFilters = array('id', 'uuid', 'value', 'comment', 'type', 'category', 'Tag.name');
             foreach ($filterValue as $k => $v) {
@@ -1804,7 +1757,7 @@ class EventsController extends AppController
                     unset($event['Object'][$k]['Attribute'][$k2]);
                 }
             }
-            if (count($event['Object'][$k]['Attribute']) == 0) {
+            if (empty($event['Object'][$k]['Attribute'])) {
                 // remove object if empty
                 unset($event['Object'][$k]);
             } else {
@@ -1816,7 +1769,7 @@ class EventsController extends AppController
 
     // look in the parameters if we are doing advanced filtering or not
     private function __checkIfAdvancedFiltering($filters) {
-        $advancedFilteringActive = array_diff_key($filters, array('sort'=>0, 'direction'=>0, 'focus'=>0, 'extended'=>0, 'overrideLimit'=>0, 'filterColumnsOverwrite'=>0, 'attributeFilter'=>0, 'extended' => 0, 'page' => 0));
+        $advancedFilteringActive = array_diff_key($filters, array('sort'=>0, 'direction'=>0, 'focus'=>0, 'overrideLimit'=>0, 'filterColumnsOverwrite'=>0, 'attributeFilter'=>0, 'extended' => 0, 'page' => 0));
 
         if (count($advancedFilteringActive) > 0) {
             if (count(array_diff_key($advancedFilteringActive, array('deleted', 'includeRelatedTags', 'includeDecayScore'))) > 0) {
@@ -1963,7 +1916,7 @@ class EventsController extends AppController
                 if ($add === true && !is_numeric($add)) {
                     if ($this->_isRest()) {
                         if ($add === 'blocked') {
-                            throw new ForbiddenException(__('Event blocked by local blacklist.'));
+                            throw new ForbiddenException(__('Event blocked by local blocklist.'));
                         }
                         // REST users want to see the newly created event
                         $results = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $created_id));
@@ -1990,7 +1943,7 @@ class EventsController extends AppController
                         return $this->RestResponse->saveFailResponse('Events', 'add', false, $validationErrors, $this->response->type());
                     } else {
                         if ($add === 'blocked') {
-                            $this->Flash->error(__('A blacklist entry is blocking you from creating any events. Please contact the administration team of this instance') . (Configure::read('MISP.contact') ? ' at ' . Configure::read('MISP.contact') : '') . '.');
+                            $this->Flash->error(__('A blocklist entry is blocking you from creating any events. Please contact the administration team of this instance') . (Configure::read('MISP.contact') ? ' at ' . Configure::read('MISP.contact') : '') . '.');
                         } else {
                             $this->Flash->error(__('The event could not be saved. Please, try again.'), 'default', array(), 'error');
                         }
@@ -2093,11 +2046,19 @@ class EventsController extends AppController
                     throw new MethodNotAllowedException(__('File upload failed or file does not have the expected extension (.xml / .json).'));
                 }
                 if (isset($this->data['Event']['submittedfile'])) {
-                    if (Configure::read('MISP.take_ownership_xml_import')
-                        && (isset($this->data['Event']['takeownership']) && $this->data['Event']['takeownership'] == 1)) {
-                        $results = $this->_addMISPExportFile($ext, true, $this->data['Event']['publish']);
-                    } else {
-                        $results = $this->_addMISPExportFile($ext, false, $this->data['Event']['publish']);
+                    $isXml = strtolower($ext) === 'xml';
+                    App::uses('FileAccessTool', 'Tools');
+                    $data = (new FileAccessTool())->readFromFile($this->data['Event']['submittedfile']['tmp_name'], $this->data['Event']['submittedfile']['size']);
+                    $takeOwnership = Configure::read('MISP.take_ownership_xml_import')
+                        && (isset($this->data['Event']['takeownership']) && $this->data['Event']['takeownership'] == 1);
+
+                    try {
+                        $results = $this->Event->addMISPExportFile($this->Auth->user(), $data, $isXml, $takeOwnership, $this->data['Event']['publish']);
+                    } catch (Exception $e) {
+                        $filename = $this->data['Event']['submittedfile']['name'];
+                        $this->log("Exception during processing MISP file import '$filename': {$e->getMessage()}");
+                        $this->Flash->error(__('Could not process MISP export file. Probably file content is invalid.'));
+                        $this->redirect(['controller' => 'events', 'action' => 'add_misp_export']);
                     }
                 }
             }
@@ -2800,7 +2761,7 @@ class EventsController extends AppController
             $user = $this->Auth->user();
             $user = $this->Event->User->fillKeysToUser($user);
 
-            $success = $this->Event->sendContactEmailRouter($id, $message, $creator_only, $user, $this->_isSiteAdmin());
+            $success = $this->Event->sendContactEmailRouter($id, $message, $creator_only, $user);
             if ($success) {
                 $return_message = __('Email sent to the reporter.');
                 if ($this->_isRest()) {
@@ -2997,18 +2958,6 @@ class EventsController extends AppController
         return $this->restSearch();
     }
 
-    // Grab an event or a list of events for the event view or any of the XML exports. The returned object includes an array of events (or an array that only includes a single event if an ID was given)
-    // Included with the event are the attached attributes, shadow attributes, related events, related attribute information for the event view and the creating user's email address where appropriate
-    private function __fetchEvent($eventid = false, $idList = false, $user = false, $tags = false, $from = false, $to = false)
-    {
-        // if we come from automation, we may not be logged in - instead we used an auth key in the URL.
-        if (empty($user)) {
-            $user = $this->Auth->user();
-        }
-        $results = $this->Event->fetchEvent($user, array('eventid' => $eventid, 'idList' => $idList, 'tags' => $tags, 'from' => $from, 'to' => $to));
-        return $results;
-    }
-
     public function nids()
     {
         $this->_legacyAPIRemap(array(
@@ -3168,66 +3117,6 @@ class EventsController extends AppController
         }
     }
 
-    private function _addMISPExportFile($ext, $take_ownership = false, $publish = false)
-    {
-        App::uses('FileAccessTool', 'Tools');
-        $data = (new FileAccessTool())->readFromFile($this->data['Event']['submittedfile']['tmp_name'], $this->data['Event']['submittedfile']['size']);
-
-        if ($ext == 'xml') {
-            App::uses('Xml', 'Utility');
-            $dataArray = Xml::toArray(Xml::build($data));
-        } else {
-            $dataArray = json_decode($data, true);
-            if (isset($dataArray['response'][0])) {
-                foreach ($dataArray['response'] as $k => $temp) {
-                    $dataArray['Event'][] = $temp['Event'];
-                    unset($dataArray['response'][$k]);
-                }
-            }
-        }
-        // In case we receive an event that is not encapsulated in a response. This should never happen (unless it's a copy+paste fail),
-        // but just in case, let's clean it up anyway.
-        if (isset($dataArray['Event'])) {
-            $dataArray['response']['Event'] = $dataArray['Event'];
-            unset($dataArray['Event']);
-        }
-        if (!isset($dataArray['response']) || !isset($dataArray['response']['Event'])) {
-            throw new Exception(__('This is not a valid MISP XML file.'));
-        }
-        $dataArray = $this->Event->updateXMLArray($dataArray);
-        $results = array();
-        $validationIssues = array();
-        if (isset($dataArray['response']['Event'][0])) {
-            foreach ($dataArray['response']['Event'] as $k => $event) {
-                $result = array('info' => $event['info']);
-                if ($take_ownership) {
-                    $event['orgc_id'] = $this->Auth->user('org_id');
-                    unset($event['Orgc']);
-                }
-                $event = array('Event' => $event);
-                $created_id = 0;
-                $event['Event']['locked'] = 1;
-                $event['Event']['published'] = $publish;
-                $result['result'] = $this->Event->_add($event, true, $this->Auth->user(), '', null, false, null, $created_id, $validationIssues);
-                $result['id'] = $created_id;
-                $result['validationIssues'] = $validationIssues;
-                $results[] = $result;
-            }
-        } else {
-            $temp['Event'] = $dataArray['response']['Event'];
-            if ($take_ownership) {
-                $temp['Event']['orgc_id'] = $this->Auth->user('org_id');
-                unset($temp['Event']['Orgc']);
-            }
-            $created_id = 0;
-            $temp['Event']['locked'] = 1;
-            $temp['Event']['published'] = $publish;
-            $result = $this->Event->_add($temp, true, $this->Auth->user(), '', null, false, null, $created_id, $validationIssues);
-            $results = array(0 => array('info' => $temp['Event']['info'], 'result' => $result, 'id' => $created_id, 'validationIssues' => $validationIssues));
-        }
-        return $results;
-    }
-
     public function downloadOpenIOCEvent($key, $eventid, $enforceWarninglist = false)
     {
         // return a downloadable text file called misp.openIOC.<eventId>.ioc for individual events
@@ -3258,8 +3147,8 @@ class EventsController extends AppController
         if (empty($event)) {
             throw new NotFoundException(__('Invalid event or not authorised.'));
         }
-        $this->loadModel('Whitelist');
-        $temp = $this->Whitelist->removeWhitelistedFromArray(array($event[0]), false);
+        $this->loadModel('Allowedlist');
+        $temp = $this->Allowedlist->removeAllowedlistedFromArray(array($event[0]), false);
         $event = $temp[0];
 
         // send the event and the vars needed to check authorisation to the Component
@@ -3732,19 +3621,7 @@ class EventsController extends AppController
             $this->set('importComment', '');
             $this->set('title', 'Freetext Import Results');
             $this->loadModel('Warninglist');
-            $tldLists = $this->Warninglist->getTldLists();
-            $missingTldLists = array();
-            foreach ($tldLists as $tldList) {
-                $temp = $this->Warninglist->find('first', array(
-                    'recursive' => -1,
-                    'conditions' => array('Warninglist.name' => $tldList),
-                    'fields' => array('Warninglist.id')
-                ));
-                if (empty($temp)) {
-                    $missingTldLists[] = $tldList;
-                }
-            }
-            $this->set('missingTldLists', $missingTldLists);
+            $this->set('missingTldLists', $this->Warninglist->missingTldLists());
             $this->render('resolved_attributes');
         }
     }
@@ -3793,20 +3670,25 @@ class EventsController extends AppController
         if (!$this->userRole['perm_add']) {
             throw new MethodNotAllowedException(__('Event not found or you don\'t have permissions to create attributes'));
         }
+        if (!$this->request->is('post')) {
+            throw new MethodNotAllowedException('This endpoint requires a POST request.');
+        }
         $event = $this->Event->fetchSimpleEvent($this->Auth->user(), $id);
         if (!$event) {
             throw new NotFoundException(__('Invalid event.'));
         }
-        if ($this->request->is('post')) {
-            $this->Event->insertLock($this->Auth->user(), $id);
-            $attributes = json_decode($this->request->data['Attribute']['JsonObject'], true);
-            $default_comment = $this->request->data['Attribute']['default_comment'];
-            $proposals = !$this->__canModifyEvent($event) || (isset($this->request->data['Attribute']['force']) && $this->request->data['Attribute']['force']);
-            $flashMessage = $this->Event->processFreeTextDataRouter($this->Auth->user(), $attributes, $id, $default_comment, $proposals);
-            $this->Flash->info($flashMessage);
-            $this->redirect(array('controller' => 'events', 'action' => 'view', $id));
+
+        $this->Event->insertLock($this->Auth->user(), $id);
+        $attributes = json_decode($this->request->data['Attribute']['JsonObject'], true);
+        $default_comment = $this->request->data['Attribute']['default_comment'];
+        $proposals = !$this->__canModifyEvent($event) || (isset($this->request->data['Attribute']['force']) && $this->request->data['Attribute']['force']);
+        $flashMessage = $this->Event->processFreeTextDataRouter($this->Auth->user(), $attributes, $id, $default_comment, $proposals);
+        $this->Flash->info($flashMessage);
+
+        if ($this->request->is('ajax')) {
+            return $this->RestResponse->viewData($flashMessage, $this->response->type());
         } else {
-            throw new MethodNotAllowedException('This endpoint requires a POST request.');
+            $this->redirect(array('controller' => 'events', 'action' => 'view', $id));
         }
     }
 
@@ -4416,21 +4298,6 @@ class EventsController extends AppController
         $this->set('id', $id);
     }
 
-    public function viewEventGraph()
-    {
-        $event = $this->Event->fetchEvent($this->Auth->user(), array(
-            'eventid' => $id
-        ));
-        if (empty($event)) {
-            throw new MethodNotAllowedException(__('Invalid Event.'));
-        }
-        $this->set('event', $event[0]);
-        $this->set('scope', 'event');
-        $this->set('id', $id);
-    }
-
-
-
     /*
         public function deleteNode($id) {
             if (!$this->request->is('post')) throw new MethodNotAllowedException(__('Only POST requests are allowed.'));
@@ -4463,16 +4330,17 @@ class EventsController extends AppController
         return new CakeResponse(array('body' => json_encode($json), 'status' => 200, 'type' => 'json'));
     }
 
-    private function genDistributionGraph($id, $type = 'event', $extended = 0) {
+    private function genDistributionGraph($id, $type = 'event', $extended = 0)
+    {
         $validTools = array('event');
         if (!in_array($type, $validTools)) {
             throw new MethodNotAllowedException(__('Invalid type.'));
         }
-        $this->loadModel('Server');
-        $this->loadModel('Organisation');
+
         App::uses('DistributionGraphTool', 'Tools');
         $grapher = new DistributionGraphTool();
 
+        $this->loadModel('Server');
         $servers = $this->Server->find('list', array(
             'fields' => array('name'),
         ));
@@ -5014,8 +4882,8 @@ class EventsController extends AppController
             throw new ForbiddenException(__('You don\'t have permission to do that.'));
         }
 
-        $resolved_data = json_decode($this->request->data['Event']['JsonObject'], true);
-        $data = json_decode($this->request->data['Event']['data'], true);
+        $resolved_data = $this->Event->jsonDecode($this->request->data['Event']['JsonObject']);
+        $data = $this->Event->jsonDecode($this->request->data['Event']['data']);
         if (!empty($data['initialObject'])) {
             $resolved_data['initialObject'] = $data['initialObject'];
         }
@@ -5023,7 +4891,12 @@ class EventsController extends AppController
         $default_comment = $this->request->data['Event']['default_comment'];
         $flashMessage = $this->Event->processModuleResultsDataRouter($this->Auth->user(), $resolved_data, $event['Event']['id'], $default_comment);
         $this->Flash->info($flashMessage);
-        $this->redirect(array('controller' => 'events', 'action' => 'view', $event['Event']['id']));
+
+        if ($this->request->is('ajax')) {
+            return $this->RestResponse->viewData($flashMessage, $this->response->type());
+        } else {
+            $this->redirect(array('controller' => 'events', 'action' => 'view', $event['Event']['id']));
+        }
     }
 
     public function importModule($module, $eventId)
@@ -5062,25 +4935,27 @@ class EventsController extends AppController
                     $this->request->data['Event']['config']['special_delimiter'] = ' ';
                 }
             }
-            foreach ($module['mispattributes']['userConfig'] as $configName => $config) {
-                if (!$fail) {
-                    if (isset($config['validation'])) {
-                        if ($config['validation'] === '0' && $config['type'] == 'String') {
-                            $validation = true;
-                        }
-                    } else {
-                        $validation = call_user_func_array(array($this->Module, $this->Module->configTypes[$config['type']]['validation']), array($this->request->data['Event']['config'][$configName]));
-                    }
-                    if ($validation !== true) {
-                        $fail = ucfirst($configName) . ': ' . $validation;
-                    } else {
-                        if (isset($config['regex']) && !empty($config['regex'])) {
-                            $fail = preg_match($config['regex'], $this->request->data['Event']['config'][$configName]) ? false : ucfirst($configName) . ': ' . 'Invalid setting' . ($config['errorMessage'] ? ' - ' . $config['errorMessage'] : '');
-                            if (!empty($fail)) {
-                                $modulePayload['config'][$configName] = $this->request->data['Event']['config'][$configName];
+            if (isset($module['mispattributes']['userConfig'])) {
+                foreach ($module['mispattributes']['userConfig'] as $configName => $config) {
+                    if (!$fail) {
+                        if (isset($config['validation'])) {
+                            if ($config['validation'] === '0' && $config['type'] == 'String') {
+                                $validation = true;
                             }
                         } else {
-                            $modulePayload['config'][$configName] = $this->request->data['Event']['config'][$configName];
+                            $validation = call_user_func_array(array($this->Module, $this->Module->configTypes[$config['type']]['validation']), array($this->request->data['Event']['config'][$configName]));
+                        }
+                        if ($validation !== true) {
+                            $fail = ucfirst($configName) . ': ' . $validation;
+                        } else {
+                            if (isset($config['regex']) && !empty($config['regex'])) {
+                                $fail = preg_match($config['regex'], $this->request->data['Event']['config'][$configName]) ? false : ucfirst($configName) . ': ' . 'Invalid setting' . ($config['errorMessage'] ? ' - ' . $config['errorMessage'] : '');
+                                if (!empty($fail)) {
+                                    $modulePayload['config'][$configName] = $this->request->data['Event']['config'][$configName];
+                                }
+                            } else {
+                                $modulePayload['config'][$configName] = $this->request->data['Event']['config'][$configName];
+                            }
                         }
                     }
                 }
@@ -5192,13 +5067,15 @@ class EventsController extends AppController
                     $this->set('importComment', $importComment);
                     $this->render($render_name);
                 }
-            } else {
+            }
+            if ($fail) {
                 $this->Flash->error($fail);
             }
         }
         $this->set('configTypes', $this->Module->configTypes);
         $this->set('module', $module);
         $this->set('eventId', $eventId);
+        $this->set('event', $event);
     }
 
     public function exportModule($module, $id, $standard = false)
@@ -5626,7 +5503,7 @@ class EventsController extends AppController
             'recursive' => -1
         ));
         $count = 0;
-        $this->Event->skipBlacklist = true;
+        $this->Event->skipBlocklist = true;
         foreach ($eventIds as $eventId => $eventUuid) {
             $result = $this->Event->Attribute->find('first', array(
                 'conditions' => array('Attribute.event_id' => $eventId),
@@ -5638,7 +5515,7 @@ class EventsController extends AppController
                 $count++;
             }
         }
-        $this->Event->skipBlacklist = null;
+        $this->Event->skipBlocklist = null;
         $message = __('%s event(s) deleted.', $count);
         if ($this->_isRest()) {
             return $this->RestResponse->viewData($message, $this->response->type());
@@ -5647,4 +5524,107 @@ class EventsController extends AppController
             $this->redirect($this->referer());
         }
     }
+
+    public function restoreDeletedEvents($force = false)
+    {
+        $startDate = '2020-07-31 00:00:00';
+        $this->loadModel('AdminSetting');
+        $endDate = date('Y-m-d H:i:s', $this->AdminSetting->getSetting('fix_login'));
+        if (empty($endDate)) {
+            $endDate = date('Y-m-d H:i:s', time());
+        }
+        $this->loadModel('Log');
+        $redis = $this->Event->setupRedis();
+        if ($force || ($redis && !$redis->exists('misp:event_recovery'))) {
+            $deleted_events = $this->Log->findDeletedEvents(['created BETWEEN ? AND ?' => [$startDate, $endDate]]);
+            $redis->set('misp:event_recovery', json_encode($deleted_events));
+            $redis->expire('misp:event_recovery', 600);
+        } else {
+            $deleted_events = json_decode($redis->get('misp:event_recovery'), true);
+        }
+        if ($this->_isRest()) {
+            return $this->RestResponse->viewData($deleted_events, 'json');
+        } else {
+            $this->set('data', $deleted_events);
+        }
+    }
+
+    public function recoverEvent($id, $mock = false)
+    {
+        if ($mock || !Configure::read('MISP.background_jobs')) {
+            if ($this->request->is('post')) {
+                $this->loadModel('Log');
+                $result = $this->Log->recoverDeletedEvent($id, $mock);
+                if ($mock) {
+                    $message = __('Recovery simulation complete. Event #%s can be recovered using %s log entries.', $id, $result);
+                } else {
+                    $message = __('Recovery complete. Event #%s recovered, using %s log entries.', $id, $result);
+                }
+                if ($this->_isRest()) {
+                    if ($mock) {
+                        $results = $this->Log->mockLog;
+                    } else {
+                        $results = $this->Event->fetchEvent($this->Auth->user(), ['eventid' => $id]);
+                    }
+                    return $this->RestResponse->viewData($results, $this->response->type());
+                } else {
+                    $this->Flash->success($message);
+                    if (!$mock) {
+                        $this->redirect(['action' => 'restoreDeletedEvents']);
+                    }
+                }
+            } else {
+                $message = __('This action is only accessible via POST requests.');
+                if ($this->_isRest()) {
+                    return $this->RestResponse->viewData(array('message' => $message, 'error' => true), $this->response->type());
+                } else {
+                    $this->Flash->error($message);
+                }
+                $this->redirect(['action' => 'restoreDeletedEvents']);
+            }
+            $this->set('data', $this->Log->mockLog);
+        } else {
+            if ($this->request->is('post')) {
+                $job_type = 'recover_event';
+                $function = 'recoverEvent';
+                $message = __('Bootstraping recovering of event %s', $id);
+                $job = ClassRegistry::init('Job');
+                $job->create();
+                $data = array(
+                        'worker' => $this->Event->__getPrioWorkerIfPossible(),
+                        'job_type' => $job_type,
+                        'job_input' => sprintf('Event ID: %s', $id),
+                        'status' => 0,
+                        'retries' => 0,
+                        'org_id' => 0,
+                        'org' => 'ADMIN',
+                        'message' => $message
+                );
+                $job->save($data);
+                $jobId = $job->id;
+                $process_id = CakeResque::enqueue(
+                    'prio',
+                    'EventShell',
+                    array($function, $jobId, $id),
+                    true
+                );
+                $job->saveField('process_id', $process_id);
+
+                $message = __('Recover event job queued. Job ID: %s', $jobId);
+                if ($this->_isRest()) {
+                    return $this->RestResponse->viewData(array('message' => $message), $this->response->type());
+                } else {
+                    $this->Flash->success($message);
+                }
+            } else {
+                $message = __('This action is only accessible via POST requests.');
+                if ($this->_isRest()) {
+                    return $this->RestResponse->viewData(array('message' => $message, 'error' => true), $this->response->type());
+                } else {
+                    $this->Flash->error($message);
+                }
+            }
+            $this->redirect(['action' => 'restoreDeletedEvents']);
+        }
+    }
 }

app/Controller/FeedsController.php

@@ -773,6 +773,9 @@ class FeedsController extends AppController
             return $this->RestResponse->viewData($event, $this->response->type());
         }
         if (is_array($event)) {
+            $this->loadModel('Warninglist');
+            $this->Warninglist->attachWarninglistToAttributes($event['Event']['Attribute']);
+
             $this->loadModel('Event');
             $params = $this->Event->rearrangeEventForView($event, $this->passedArgs, $all);
             $this->params->params['paging'] = array('Feed' => $params);

app/Controller/GalaxyClustersController.php

@@ -1,9 +1,6 @@
 <?php
 App::uses('AppController', 'Controller');
 
-/**
- * @property GalaxyCluster $GalaxyCluster
- */
 class GalaxyClustersController extends AppController
 {
     public $components = array('Session', 'RequestHandler');
@@ -16,6 +13,17 @@ class GalaxyClustersController extends AppController
                 'GalaxyCluster.value' => 'ASC'
             ),
             'contain' => array(
+                'Tag' => array(
+                    'fields' => array('Tag.id'),
+                    /*
+                    'EventTag' => array(
+                        'fields' => array('EventTag.event_id')
+                    ),
+                    'AttributeTag' => array(
+                        'fields' => array('AttributeTag.event_id', 'AttributeTag.attribute_id')
+                    )
+                    */
+                ),
                 'GalaxyElement' => array(
                     'conditions' => array('GalaxyElement.key' => 'synonyms'),
                     'fields' => array('value')
@@ -26,6 +34,7 @@ class GalaxyClustersController extends AppController
     public function index($id)
     {
         $filters = $this->IndexFilter->harvestParameters(array('context', 'searchall'));
+        $contextConditions = array();
         if (empty($filters['context'])) {
             $filters['context'] = 'all';
         }
@@ -33,46 +42,33 @@ class GalaxyClustersController extends AppController
         $this->set('searchall', isset($filters['searchall']) ? $filters['searchall'] : '');
         $this->paginate['conditions'] = array('GalaxyCluster.galaxy_id' => $id);
         if (isset($filters['searchall']) && strlen($filters['searchall']) > 0) {
-            $search = '%' . strtolower($filters['searchall']) . '%';
             $synonym_hits = $this->GalaxyCluster->GalaxyElement->find(
                 'list',
                 array(
                     'recursive' => -1,
                     'conditions' => array(
-                        'LOWER(GalaxyElement.value) LIKE' => $search,
-                        'GalaxyElement.key' => 'synonyms',
-                    ),
-                    'fields' => array(
-                        'GalaxyElement.galaxy_cluster_id',
-                    )
-                )
+                        'LOWER(GalaxyElement.value) LIKE' => '%' . strtolower($filters['searchall']) . '%',
+                        'GalaxyElement.key' => 'synonyms' ),
+                        'fields' => array(
+                            'GalaxyElement.galaxy_cluster_id')
+                        )
             );
-            $this->paginate['conditions'] = array(
-                "AND" => array(
+            $this->paginate['conditions'] =
+                array("AND" => array(
                     'OR' => array(
-                        "LOWER(GalaxyCluster.value) LIKE" => $search,
-                        "LOWER(GalaxyCluster.description) LIKE" => $search,
+                        "LOWER(GalaxyCluster.value) LIKE" => '%'. strtolower($filters['searchall']) .'%',
+                        "LOWER(GalaxyCluster.description) LIKE" => '%'. strtolower($filters['searchall']) .'%',
                         "GalaxyCluster.id" => array_values($synonym_hits)
                     ),
                     "GalaxyCluster.galaxy_id" => $id
-                )
-            );
-            $this->set('passedArgsArray', array('all' => $filters['searchall']));
+                    ));
+            $this->set('passedArgsArray', array('all'=>$filters['searchall']));
         }
         $clusters = $this->paginate();
+        $sgs = $this->GalaxyCluster->Tag->EventTag->Event->SharingGroup->fetchAllAuthorised($this->Auth->user());
         foreach ($clusters as $k => $cluster) {
-            $tag = $this->GalaxyCluster->Tag->find('first', array(
-                'conditions' => array(
-                    'LOWER(name)' => strtolower($cluster['GalaxyCluster']['tag_name']),
-                ),
-                'fields' => array('id'),
-                'recursive' => -1,
-                'contain' => array('EventTag.event_id')
-            ));
-
-            if ($tag) {
-                $clusters[$k]['GalaxyCluster']['event_count'] = $this->GalaxyCluster->Tag->EventTag->countForTag($tag, $this->Auth->user());
-                $clusters[$k]['Tag'] = $tag['Tag'];
+            if (!empty($cluster['Tag']['id'])) {
+                $clusters[$k]['GalaxyCluster']['event_count'] = $this->GalaxyCluster->Tag->EventTag->countForTag($cluster['Tag']['id'], $this->Auth->user(), $sgs);
             } else {
                 $clusters[$k]['GalaxyCluster']['event_count'] = 0;
             }
@@ -80,6 +76,7 @@ class GalaxyClustersController extends AppController
         $tagIds = array();
         $sightings = array();
         if (!empty($clusters)) {
+            $galaxyType = $clusters[0]['GalaxyCluster']['type'];
             foreach ($clusters as $k => $v) {
                 $clusters[$k]['event_ids'] = array();
                 if (!empty($v['Tag'])) {
@@ -141,16 +138,17 @@ class GalaxyClustersController extends AppController
             'conditions' => $conditions
         ));
         if (!empty($cluster)) {
-            $tag = $this->GalaxyCluster->Tag->find('first', array(
+            $this->loadModel('Tag');
+            $tag = $this->Tag->find('first', array(
                 'conditions' => array(
                     'LOWER(name)' => strtolower($cluster['GalaxyCluster']['tag_name']),
                 ),
                 'fields' => array('id'),
                 'recursive' => -1,
-                'contain' => array('EventTag.event_id')
+                'contain' => array('EventTag.tag_id')
             ));
             if (!empty($tag)) {
-                $cluster['GalaxyCluster']['tag_count'] = $this->GalaxyCluster->Tag->EventTag->countForTag($tag, $this->Auth->user());
+                $cluster['GalaxyCluster']['tag_count'] = count($tag['EventTag']);
                 $cluster['GalaxyCluster']['tag_id'] = $tag['Tag']['id'];
             }
         } else {

app/Controller/LogsController.php

@@ -404,7 +404,7 @@ class LogsController extends AppController
             $this->set('actions', $actions);
 
             // combobox for models
-            $models = array('Attribute', 'Event', 'EventBlacklist', 'EventTag', 'Feed', 'DecayingModel', 'MispObject', 'Organisation', 'Post', 'Regexp', 'Role', 'Server', 'ShadowAttribute', 'SharingGroup', 'Tag', 'Task', 'Taxonomy', 'Template', 'Thread', 'User', 'Whitelist');
+            $models = array('Attribute', 'Event', 'EventBlocklist', 'EventTag', 'Feed', 'DecayingModel', 'MispObject', 'Organisation', 'Post', 'Regexp', 'Role', 'Server', 'ShadowAttribute', 'SharingGroup', 'Tag', 'Task', 'Taxonomy', 'Template', 'Thread', 'User', 'Allowedlist');
             $models = array('' => 'ALL') + $this->_arrayToValuesIndexArray($models);
             $this->set('models', $models);
             $this->set('actionDefinitions', $this->{$this->defaultModel}->actionDefinitions);

app/Controller/ObjectsController.php

@@ -87,6 +87,15 @@ class ObjectsController extends AppController
             $similar_object_similarity_amount[$obj['Attribute']['object_id']] = $obj[0]['similarity_amount'];
         }
 
+        if (isset($this->request->data['Attribute'])) {
+            foreach ($this->request->data['Attribute'] as &$attribute) {
+                $validation = $this->MispObject->Attribute->validateAttribute($attribute);
+                if ($validation !== true) {
+                    $attribute['validation'] = $validation;
+                }
+            }
+        }
+
         $this->set('distributionLevels', $this->MispObject->Attribute->distributionLevels);
         $this->set('action', $action);
         $this->set('template', $template);

app/Controller/OrgBlocklistsController.php

@@ -1,9 +1,9 @@
 <?php
 App::uses('AppController', 'Controller');
 
-class OrgBlacklistsController extends AppController
+class OrgBlocklistsController extends AppController
 {
-    public $components = array('Session', 'RequestHandler', 'BlackList');
+    public $components = array('Session', 'RequestHandler', 'BlockList');
 
     public function beforeFilter()
     {
@@ -11,8 +11,8 @@ class OrgBlacklistsController extends AppController
         if (!$this->_isSiteAdmin()) {
             $this->redirect('/');
         }
-        if (Configure::check('MISP.enableOrgBlacklisting') && !Configure::read('MISP.enableOrgBlacklisting') !== false) {
-            $this->Flash->info(__('Organisation Blacklisting is not currently enabled on this instance.'));
+        if (Configure::check('MISP.enableOrgBlocklisting') && !Configure::read('MISP.enableOrgBlocklisting') !== false) {
+            $this->Flash->info(__('Organisation BlockListing is not currently enabled on this instance.'));
             $this->redirect('/');
         }
     }
@@ -21,27 +21,27 @@ class OrgBlacklistsController extends AppController
             'limit' => 60,
             'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page.
             'order' => array(
-                    'OrgBlacklist.created' => 'DESC'
+                    'OrgBlocklist.created' => 'DESC'
             ),
     );
 
     public function index()
     {
-        return $this->BlackList->index($this->_isRest());
+        return $this->BlockList->index($this->_isRest());
     }
 
     public function add()
     {
-        return $this->BlackList->add($this->_isRest());
+        return $this->BlockList->add($this->_isRest());
     }
 
     public function edit($id)
     {
-        return $this->BlackList->edit($this->_isRest(), $id);
+        return $this->BlockList->edit($this->_isRest(), $id);
     }
 
     public function delete($id)
     {
-        return $this->BlackList->delete($this->_isRest(), $id);
+        return $this->BlockList->delete($this->_isRest(), $id);
     }
 }

app/Controller/OrganisationsController.php

@@ -1,6 +1,9 @@
 <?php
 App::uses('AppController', 'Controller');
 
+/**
+ * @property Organisation $Organisation
+ */
 class OrganisationsController extends AppController
 {
     public $components = array('Session', 'RequestHandler');
@@ -87,6 +90,10 @@ class OrganisationsController extends AppController
         if ($this->_isRest()) {
             return $this->RestResponse->viewData($orgs, $this->response->type());
         } else {
+            foreach ($orgs as &$org) {
+                $org['Organisation']['country_code'] = $this->Organisation->getCountryCode($org['Organisation']['nationality']);
+            }
+
             $this->set('named', $this->params['named']);
             $this->set('scope', $scope);
             $this->set('orgs', $orgs);
@@ -152,7 +159,8 @@ class OrganisationsController extends AppController
                 }
             }
         }
-        $this->set('countries', $this->_arrayToValuesIndexArray($this->Organisation->countries));
+        $countries = array_merge(['' => __('Not specified')], $this->_arrayToValuesIndexArray($this->Organisation->getCountries()));
+        $this->set('countries', $countries);
     }
 
     public function admin_edit($id)
@@ -230,7 +238,17 @@ class OrganisationsController extends AppController
             $this->Organisation->read(null, $id);
             $this->request->data = $this->Organisation->data;
         }
-        $this->set('countries', $this->_arrayToValuesIndexArray($this->Organisation->countries));
+
+        $countries = array_merge(['' => __('Not specified')], $this->_arrayToValuesIndexArray($this->Organisation->getCountries()));
+        if (!empty($this->Organisation->data['Organisation']['nationality'])) {
+            $currentCountry = $this->Organisation->data['Organisation']['nationality'];
+            if (!isset($countries[$currentCountry])) {
+                // Append old country name to list to keep backward compatibility
+                $countries[$currentCountry] = $currentCountry;
+            }
+        }
+
+        $this->set('countries', $countries);
         $this->set('orgId', $id);
         if (is_array($this->request->data['Organisation']['restricted_to_domain'])) {
             $this->request->data['Organisation']['restricted_to_domain'] = implode("\n", $this->request->data['Organisation']['restricted_to_domain']);
@@ -355,28 +373,13 @@ class OrganisationsController extends AppController
             $org['Organisation']['user_count'] = $this->Organisation->User->getMembersCount($org['Organisation']['id']);
             return $this->RestResponse->viewData($org, $this->response->type());
         } else {
+            $org['Organisation']['country_code'] = $this->Organisation->getCountryCode($org['Organisation']['nationality']);
             $this->set('fullAccess', $fullAccess);
             $this->set('org', $org);
             $this->set('id', $id);
         }
     }
 
-    public function landingpage($id)
-    {
-        $this->Organisation->id = $id;
-        if (!$this->Organisation->exists()) {
-            throw new NotFoundException(__('Invalid organisation'));
-        }
-        $org = $this->Organisation->find('first', array('conditions' => array('id' => $id), 'fields' => array('landingpage', 'name')));
-        $landingpage = $org['Organisation']['landingpage'];
-        if (empty($landingpage)) {
-            $landingpage = __('No landing page has been created for this organisation.');
-        }
-        $this->set('landingPage', $landingpage);
-        $this->set('org', $org['Organisation']['name']);
-        $this->render('ajax/landingpage');
-    }
-
     public function fetchOrgsForSG($idList = '{}', $type)
     {
         if ($type === 'local') {

app/Controller/ServersController.php

@@ -3,6 +3,9 @@ App::uses('AppController', 'Controller');
 App::uses('Xml', 'Utility');
 App::uses('AttachmentTool', 'Tools');
 
+/**
+ * @property Server $Server
+ */
 class ServersController extends AppController
 {
     public $components = array('Security' ,'RequestHandler');   // XXX ACL component
@@ -150,25 +153,36 @@ class ServersController extends AppController
         if (!$this->_isSiteAdmin()) {
             throw new MethodNotAllowedException('You are not authorised to do that.');
         }
-        $server = $this->Server->find('first', array('conditions' => array('Server.id' => $serverId), 'recursive' => -1, 'fields' => array('Server.id', 'Server.url', 'Server.name')));
+        $server = $this->Server->find('first', array(
+            'conditions' => array('Server.id' => $serverId),
+            'recursive' => -1,
+            'fields' => array('Server.id', 'Server.url', 'Server.name'))
+        );
         if (empty($server)) {
             throw new NotFoundException('Invalid server ID.');
         }
         try {
             $event = $this->Server->previewEvent($serverId, $eventId);
         } catch (NotFoundException $e) {
-            throw new NotFoundException(__("Event '$eventId' not found."));
+            throw new NotFoundException(__("Event '%s' not found.", $eventId));
         } catch (Exception $e) {
-            $this->Flash->error(__('Download failed.') . ' ' . $e->getMessage());
+            $this->Flash->error(__('Download failed. %s', $e->getMessage()));
             $this->redirect(array('action' => 'previewIndex', $serverId));
         }
 
+        $this->loadModel('Warninglist');
+        if (isset($event['Event']['Attribute'])) {
+            $this->Warninglist->attachWarninglistToAttributes($event['Event']['Attribute']);
+        }
+        if (isset($event['Event']['ShadowAttribute'])) {
+            $this->Warninglist->attachWarninglistToAttributes($event['Event']['ShadowAttribute']);
+        }
+
         $this->loadModel('Event');
         $params = $this->Event->rearrangeEventForView($event, $this->passedArgs, $all);
         $this->params->params['paging'] = array('Server' => $params);
         $this->set('event', $event);
         $this->set('server', $server);
-        $this->loadModel('Event');
         $dataForView = array(
                 'Attribute' => array('attrDescriptions' => 'fieldDescriptions', 'distributionDescriptions' => 'distributionDescriptions', 'distributionLevels' => 'distributionLevels'),
                 'Event' => array('eventDescriptions' => 'fieldDescriptions', 'analysisLevels' => 'analysisLevels'),
@@ -788,7 +802,7 @@ class ServersController extends AppController
                 }
             }
             if ($this->_isRest()) {
-                return $this->RestResponse->saveSuccessResponse('Servers', 'push', array(sprintf(__('Push complete. %s events pushed, %s events could not be pushed.', $result[0], $result[1]))), $this->response->type());
+                return $this->RestResponse->saveSuccessResponse('Servers', 'push', __('Push complete. %s events pushed, %s events could not be pushed.', count($result[0]), count($result[1])), $this->response->type());
             } else {
                 $this->set('successes', $result[0]);
                 $this->set('fails', $result[1]);
@@ -1232,8 +1246,12 @@ class ServersController extends AppController
 
     public function getWorkers()
     {
-        $issues = 0;
-        $worker_array = $this->Server->workerDiagnostics($issues);
+        if (Configure::read('MISP.background_jobs')) {
+            $workerIssueCount = 0;
+            $worker_array = $this->Server->workerDiagnostics($workerIssueCount);
+        } else {
+            $worker_array = [__('Background jobs not enabled')];
+        }
         return $this->RestResponse->viewData($worker_array);
     }
 
@@ -1471,6 +1489,18 @@ class ServersController extends AppController
         }
     }
 
+    public function killAllWorkers($force = false)
+    {
+        if (!$this->request->is('post')) {
+            throw new MethodNotAllowedException();
+        }
+        $this->Server->killAllWorkers($this->Auth->user(), $force);
+        if ($this->_isRest()) {
+            return $this->RestResponse->saveSuccessResponse('Server', 'killAllWorkers', false, $this->response->type(), __('Killing workers.'));
+        }
+        $this->redirect(array('controller' => 'servers', 'action' => 'serverSettings', 'workers'));
+    }
+
     public function restartWorkers()
     {
         if (!$this->_isSiteAdmin() || !$this->request->is('post')) {

app/Controller/SightingsController.php

@@ -3,6 +3,7 @@ App::uses('AppController', 'Controller');
 
 /**
  * @property Sighting $Sighting
+ * @property Event $Event
  */
 class SightingsController extends AppController
 {
@@ -326,28 +327,28 @@ class SightingsController extends AppController
                 throw new MethodNotAllowedException('Invalid object.');
             }
             $eventIds = array();
-            foreach ($object as $k => $v) {
+            foreach ($object as $v) {
                 $eventIds[] = $v['Attribute']['event_id'];
             }
-            $events = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $eventIds, 'metadata' => true));
+            $events = $this->Event->fetchSimpleEvents($this->Auth->user(), ['conditions' => ['id' => $eventIds]]);
         } else {
             $attribute_id = false;
             // let's set the context to event here, since we reuse the variable later on for some additional lookups.
             // Passing $context = 'org' could have interesting results otherwise...
-            $context = 'event';
-            $events = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $id, 'metadata' => true));
+            $events = $this->Event->fetchSimpleEvents($this->Auth->user(), ['conditions' => ['id' => $id]]);
         }
         if (empty($events)) {
             throw new MethodNotAllowedException('Invalid object.');
         }
-        $results = array();
         $raw = array();
         foreach ($events as $event) {
             $raw = array_merge($raw, $this->Sighting->attachToEvent($event, $this->Auth->user(), $attribute_id));
         }
+        $results = array();
         foreach ($raw as $sighting) {
             $results[$sighting['type']][date('Ymd', $sighting['date_sighting'])][] = $sighting;
         }
+        unset($raw);
         $dataPoints = array();
         $startDate = date('Ymd');
         $range = date('Ymd', $this->Sighting->getMaximumRange());

app/Controller/TagsController.php

@@ -2,9 +2,6 @@
 
 App::uses('AppController', 'Controller');
 
-/**
- * @property Tag $Tag
- */
 class TagsController extends AppController
 {
     public $components = array('Security' ,'RequestHandler');
@@ -30,6 +27,12 @@ class TagsController extends AppController
 
     public $helpers = array('TextColour');
 
+    public function beforeFilter()
+    {
+        parent::beforeFilter();
+        $this->Security->unlockedActions[] = 'search';
+    }
+
     public function index($favouritesOnly = false)
     {
         $this->loadModel('Attribute');
@@ -74,31 +77,31 @@ class TagsController extends AppController
         }
         if ($this->_isRest()) {
             unset($this->paginate['limit']);
+            unset($this->paginate['contain']['EventTag']);
+            unset($this->paginate['contain']['AttributeTag']);
             $paginated = $this->Tag->find('all', $this->paginate);
         } else {
             $paginated = $this->paginate();
         }
         $tagList = array();
         $csv = array();
+        $sgs = $this->Tag->EventTag->Event->SharingGroup->fetchAllAuthorised($this->Auth->user());
         foreach ($paginated as $k => $tag) {
             $tagList[] = $tag['Tag']['id'];
-            $paginated[$k]['Tag']['count'] = $this->Tag->EventTag->countForTag($tag, $this->Auth->user());
-            $paginated[$k]['Tag']['attribute_count'] = $this->Tag->AttributeTag->countForTag($tag, $this->Auth->user());
-
+            $paginated[$k]['Tag']['count'] = $this->Tag->EventTag->countForTag($tag['Tag']['id'], $this->Auth->user(), $sgs);
             if (!$this->_isRest()) {
                 $paginated[$k]['event_ids'] = array();
+                $paginated[$k]['attribute_ids'] = array();
                 foreach ($paginated[$k]['EventTag'] as $et) {
                     $paginated[$k]['event_ids'][] = $et['event_id'];
                 }
-                $paginated[$k]['attribute_ids'] = array();
+                unset($paginated[$k]['EventTag']);
                 foreach ($paginated[$k]['AttributeTag'] as $at) {
                     $paginated[$k]['attribute_ids'][] = $at['attribute_id'];
                 }
+                unset($paginated[$k]['AttributeTag']);
             }
-
-            unset($paginated[$k]['EventTag']);
-            unset($paginated[$k]['AttributeTag']);
-
+            $paginated[$k]['Tag']['attribute_count'] = $this->Tag->AttributeTag->countForTag($tag['Tag']['id'], $this->Auth->user(), $sgs);
             if (!empty($tag['FavouriteTag'])) {
                 foreach ($tag['FavouriteTag'] as $ft) {
                     if ($ft['user_id'] == $this->Auth->user('id')) {
@@ -398,13 +401,58 @@ class TagsController extends AppController
             if (empty($tag['EventTag'])) {
                 $tag['Tag']['count'] = 0;
             } else {
-                $tag['Tag']['count'] = $this->Tag->EventTag->countForTag($tag, $this->Auth->user());
+                $eventIDs = array();
+                foreach ($tag['EventTag'] as $eventTag) {
+                    $eventIDs[] = $eventTag['event_id'];
+                }
+                $conditions = array('Event.id' => $eventIDs);
+                if (!$this->_isSiteAdmin()) {
+                    $conditions = array_merge(
+                        $conditions,
+                        array('OR' => array(
+                                array('AND' => array(
+                                        array('Event.distribution >' => 0),
+                                        array('Event.published =' => 1)
+                                )),
+                                array('Event.orgc_id' => $this->Auth->user('org_id'))
+                        ))
+                );
+                }
+                $events = $this->Tag->EventTag->Event->find('all', array(
+                        'fields' => array('Event.id', 'Event.distribution', 'Event.orgc_id'),
+                        'conditions' => $conditions
+                ));
+                $tag['Tag']['count'] = count($events);
             }
             unset($tag['EventTag']);
             if (empty($tag['AttributeTag'])) {
                 $tag['Tag']['attribute_count'] = 0;
             } else {
-                $tag['Tag']['attribute_count'] = $this->Tag->AttributeTag->countForTag($tag, $this->Auth->user());;
+                $attributeIDs = array();
+                foreach ($tag['AttributeTag'] as $attributeTag) {
+                    $attributeIDs[] = $attributeTag['attribute_id'];
+                }
+                $conditions = array('Attribute.id' => $attributeIDs);
+                if (!$this->_isSiteAdmin()) {
+                    $conditions = array_merge(
+                        $conditions,
+                        array('OR' => array(
+                            array('AND' => array(
+                                array('Attribute.deleted =' => 0),
+                                array('Attribute.distribution >' => 0),
+                                array('Event.distribution >' => 0),
+                                array('Event.published =' => 1)
+                            )),
+                            array('Event.orgc_id' => $this->Auth->user('org_id'))
+                        ))
+                    );
+                }
+                $attributes = $this->Tag->AttributeTag->Attribute->find('all', array(
+                    'fields'     => array('Attribute.id', 'Attribute.deleted', 'Attribute.distribution', 'Event.id', 'Event.distribution', 'Event.orgc_id'),
+                    'contain'    => array('Event' => array('fields' => array('id', 'distribution', 'orgc_id'))),
+                    'conditions' => $conditions
+                ));
+                $tag['Tag']['attribute_count'] = count($attributes);
             }
             unset($tag['AttributeTag']);
             $this->set('Tag', $tag['Tag']);
@@ -882,7 +930,7 @@ class TagsController extends AppController
             if (is_numeric($tag)) {
                 $conditions = array('Tag.id' => $tag);
             } else {
-                $conditions = array('LOWER(Tag.name) LIKE' => strtolower(trim($tag)));
+                $conditions = array('Tag.name LIKE' => trim($tag));
             }
             if (empty($local)) {
                 if (!empty($this->request->data['local'])) {
@@ -1064,7 +1112,7 @@ class TagsController extends AppController
         $this->render('/Events/view_graph');
     }
 
-    public function search($tag = false)
+    public function search($tag = false, $strictTagNameOnly = false, $searchIfTagExists = true)
     {
         if (isset($this->request->data['Tag'])) {
             $this->request->data = $this->request->data['Tag'];
@@ -1077,41 +1125,59 @@ class TagsController extends AppController
         if (!is_array($tag)) {
             $tag = array($tag);
         }
-        $conditions = array();
-        foreach ($tag as $k => $t) {
-            $tag[$k] = strtolower($t);
-            $conditions['OR'][] = array('LOWER(GalaxyCluster.value)' => $tag[$k]);
-        }
-        foreach ($tag as $k => $t) {
-            $conditions['OR'][] = array('AND' => array('GalaxyElement.key' => 'synonyms', 'LOWER(GalaxyElement.value) LIKE' => $t));
-        }
         $this->loadModel('GalaxyCluster');
-        $elements = $this->GalaxyCluster->GalaxyElement->find('all', array(
-            'recursive' => -1,
-            'conditions' => $conditions,
-            'contain' => array('GalaxyCluster.tag_name')
-        ));
-        foreach ($elements as $element) {
-            $tag[] = strtolower($element['GalaxyCluster']['tag_name']);
-        }
         $conditions = array();
-        foreach ($tag as $k => $t) {
-            $conditions['OR'][] = array('LOWER(Tag.name) LIKE' => $t);
+        if (!$strictTagNameOnly) {
+            foreach ($tag as $k => $t) {
+                $tag[$k] = strtolower($t);
+                $conditions['OR'][] = array('LOWER(GalaxyCluster.value)' => $tag[$k]);
+            }
+            foreach ($tag as $k => $t) {
+                $conditions['OR'][] = array('AND' => array('GalaxyElement.key' => 'synonyms', 'LOWER(GalaxyElement.value) LIKE' => $t));
+            }
+            $elements = $this->GalaxyCluster->GalaxyElement->find('all', array(
+                'recursive' => -1,
+                'conditions' => $conditions,
+                'contain' => array('GalaxyCluster.tag_name')
+            ));
+            foreach ($elements as $element) {
+                $tag[] = strtolower($element['GalaxyCluster']['tag_name']);
+            }
+            foreach ($tag as $k => $t) {
+                $conditions['OR'][] = array('LOWER(Tag.name) LIKE' => $t);
+            }
+        } else {
+                foreach ($tag as $k => $t) {
+                    $conditions['OR'][] = array('Tag.name' => $t);
+                }
         }
         $tags = $this->Tag->find('all', array(
             'conditions' => $conditions,
             'recursive' => -1
         ));
+        if (!$searchIfTagExists && empty($tags)) {
+            $tags = [];
+            foreach ($tag as $i => $tagName) {
+                $tags[] = ['Tag' => ['name' => $tagName], 'simulatedTag' => true];
+            }
+        }
         $this->loadModel('Taxonomy');
         foreach ($tags as $k => $t) {
-            $taxonomy = $this->Taxonomy->getTaxonomyForTag($t['Tag']['name'], true);
-            if (!empty($taxonomy)) {
+            $dataFound = false;
+            $taxonomy = $this->Taxonomy->getTaxonomyForTag($t['Tag']['name'], false);
+            if (!empty($taxonomy) && !empty($taxonomy['TaxonomyPredicate'][0])) {
+                $dataFound = true;
                 $tags[$k]['Taxonomy'] = $taxonomy['Taxonomy'];
+                $tags[$k]['TaxonomyPredicate'] = $taxonomy['TaxonomyPredicate'][0];
             }
             $cluster = $this->GalaxyCluster->getCluster($t['Tag']['name']);
             if (!empty($cluster)) {
+                $dataFound = true;
                 $tags[$k]['GalaxyCluster'] = $cluster['GalaxyCluster'];
             }
+            if (!$searchIfTagExists && !$dataFound && !empty($t['simulatedTag'])) {
+                unset($tags[$k]);
+            }
         }
         return $this->RestResponse->viewData($tags, $this->response->type());
     }

app/Controller/UsersController.php

@@ -1,6 +1,9 @@
 <?php
 App::uses('AppController', 'Controller');
 
+/**
+ * @property User $User
+ */
 class UsersController extends AppController
 {
     public $newkey;
@@ -759,7 +762,9 @@ class UsersController extends AppController
                             $password = isset($this->request->data['User']['password']) ? $this->request->data['User']['password'] : false;
                             $result = $this->User->initiatePasswordReset($user, true, true, $password);
                             if ($result && empty(Configure::read('MISP.disable_emailing'))) {
-                                $notification_message .= ' User notified of new credentials.';
+                                $notification_message .= ' ' . __('User notified of new credentials.');
+                            } else {
+                                $notification_message .= ' ' . __('User notification of new credentials could not be send.');
                             }
                         }
                         if ($this->_isRest()) {
@@ -770,7 +775,7 @@ class UsersController extends AppController
                             $user['User']['password'] = '******';
                             return $this->RestResponse->viewData($user, $this->response->type());
                         } else {
-                            $this->Flash->success(__('The user has been saved.' . $notification_message));
+                            $this->Flash->success(__('The user has been saved.') . $notification_message);
                             $this->redirect(array('action' => 'index'));
                         }
                     } else {
@@ -1112,7 +1117,7 @@ class UsersController extends AppController
         if ($this->request->is('post') || $this->request->is('put')) {
             $this->Bruteforce = ClassRegistry::init('Bruteforce');
             if (!empty($this->request->data['User']['email'])) {
-                if ($this->Bruteforce->isBlacklisted($_SERVER['REMOTE_ADDR'], $this->request->data['User']['email'])) {
+                if ($this->Bruteforce->isBlocklisted($_SERVER['REMOTE_ADDR'], $this->request->data['User']['email'])) {
                     $expire = Configure::check('SecureAuth.expire') ? Configure::read('SecureAuth.expire') : 300;
                     throw new ForbiddenException('You have reached the maximum number of login attempts. Please wait ' . Configure::read('SecureAuth.expire') . ' seconds and try again.');
                 }
@@ -1689,71 +1694,73 @@ class UsersController extends AppController
 
     public function email_otp()
     {
-      $user = $this->Session->read('email_otp_user');
-      if(empty($user)) {
-        $this->redirect('login');
-      }
-      $redis = $this->User->setupRedis();
-      $user_id = $user['id'];
-
-      if ($this->request->is('post') && isset($this->request->data['User']['otp'])) {
-        $stored_otp = $redis->get('misp:otp:'.$user_id);
-        if (!empty($stored_otp) && $this->request->data['User']['otp'] == $stored_otp) {
-            // we invalidate the previously generated OTP
-            $redis->delete('misp:otp:'.$user_id);
-            // We login the user with CakePHP
-            $this->Auth->login($user);
-            $this->_postlogin();
-        } else {
-            $this->Flash->error(__("The OTP is incorrect or has expired"));
+        $user = $this->Session->read('email_otp_user');
+        if (empty($user)) {
+            $this->redirect('login');
         }
-      } else {
-        // GET Request
+        $redis = $this->User->setupRedisWithException();
+        $user_id = $user['id'];
 
-        // We check for exceptions
-        $exception_list = Configure::read('Security.email_otp_exceptions');
-        if (!empty($exception_list)) {
-          $exceptions = explode(",", $exception_list);
-          foreach ($exceptions as &$exception) {
-            if ($user['email'] == trim($exception)) {
-              // We login the user with CakePHP
-              $this->Auth->login($user);
-              $this->_postlogin();
+        if ($this->request->is('post') && isset($this->request->data['User']['otp'])) {
+            $stored_otp = $redis->get('misp:otp:' . $user_id);
+            if (!empty($stored_otp) && $this->request->data['User']['otp'] == $stored_otp) {
+                // we invalidate the previously generated OTP
+                $redis->del('misp:otp:' . $user_id);
+                // We login the user with CakePHP
+                $this->Auth->login($user);
+                $this->_postlogin();
+            } else {
+                $this->Flash->error(__("The OTP is incorrect or has expired"));
             }
-          }
-        }
-        $this->loadModel('Server');
-
-        // Generating the OTP
-        $digits = !empty(Configure::read('Security.email_otp_length')) ? Configure::read('Security.email_otp_length') : $this->Server->serverSettings['Security']['email_otp_length']['value'];
-        $otp = "";
-        for ($i=0; $i<$digits; $i++) {
-          $otp.= random_int(0,9);
-        }
-        // We use Redis to cache the OTP
-        $redis->set('misp:otp:'.$user_id, $otp);
-        $validity = !empty(Configure::read('Security.email_otp_validity')) ? Configure::read('Security.email_otp_validity') : $this->Server->serverSettings['Security']['email_otp_validity']['value'];
-        $redis->expire('misp:otp:'.$user_id, (int) $validity * 60);
-
-        // Email construction
-        $body = !empty(Configure::read('Security.email_otp_text')) ? Configure::read('Security.email_otp_text') : $this->Server->serverSettings['Security']['email_otp_text']['value'];
-        $body = str_replace('$misp', Configure::read('MISP.baseurl'), $body);
-        $body = str_replace('$org', Configure::read('MISP.org'), $body);
-        $body = str_replace('$contact', Configure::read('MISP.contact'), $body);
-        $body = str_replace('$validity', $validity, $body);
-        $body = str_replace('$otp', $otp, $body);
-        $body = str_replace('$ip', $this->__getClientIP(), $body);
-        $body = str_replace('$username', $user['email'], $body);
-        $result = $this->User->sendEmail(array('User' => $user), $body, false, "[MISP] Email OTP");
-
-        if ( $result ) {
-          $this->Flash->success(__("An email containing a OTP has been sent."));
         } else {
-          $this->Flash->error(__("The email couldn't be sent, please reach out to your administrator."));
-        }
-      }
-    }
+            // GET Request
 
+            // We check for exceptions
+            $exception_list = Configure::read('Security.email_otp_exceptions');
+            if (!empty($exception_list)) {
+                $exceptions = explode(",", $exception_list);
+                foreach ($exceptions as $exception) {
+                    if ($user['email'] === trim($exception)) {
+                        // We login the user with CakePHP
+                        $this->Auth->login($user);
+                        $this->_postlogin();
+                    }
+                }
+            }
+            $this->loadModel('Server');
+
+            // Generating the OTP
+            $digits = Configure::read('Security.email_otp_length') ?: $this->Server->serverSettings['Security']['email_otp_length']['value'];
+            $otp = "";
+            for ($i = 0; $i < $digits; $i++) {
+                $otp .= random_int(0, 9);
+            }
+            // We use Redis to cache the OTP
+            $redis->set('misp:otp:' . $user_id, $otp);
+            $validity = Configure::read('Security.email_otp_validity') ?: $this->Server->serverSettings['Security']['email_otp_validity']['value'];
+            $redis->expire('misp:otp:' . $user_id, (int)$validity * 60);
+
+            // Email construction
+            $body = Configure::read('Security.email_otp_text') ?: $this->Server->serverSettings['Security']['email_otp_text']['value'];
+            $body = str_replace('$misp', Configure::read('MISP.baseurl'), $body);
+            $body = str_replace('$org', Configure::read('MISP.org'), $body);
+            $body = str_replace('$contact', Configure::read('MISP.contact'), $body);
+            $body = str_replace('$validity', $validity, $body);
+            $body = str_replace('$otp', $otp, $body);
+            $body = str_replace('$ip', $this->__getClientIP(), $body);
+            $body = str_replace('$username', $user['email'], $body);
+
+            // Fetch user that contains also PGP or S/MIME keys for e-mail encryption
+            $userForSendMail = $this->User->getUserById($user_id);
+            $result = $this->User->sendEmail($userForSendMail, $body, false, "[MISP] Email OTP");
+
+            if ($result) {
+                $this->Flash->success(__("An email containing a OTP has been sent."));
+            } else {
+                $this->Flash->error(__("The email couldn't be sent, please reach out to your administrator."));
+            }
+        }
+    }
 
     /**
     * Helper function to determine the IP of a client (proxy aware)

app/Controller/WarninglistsController.php

@@ -1,6 +1,9 @@
 <?php
 App::uses('AppController', 'Controller');
 
+/**
+ * @property Warninglist $Warninglist
+ */
 class WarninglistsController extends AppController
 {
     public $components = array('Session', 'RequestHandler');
@@ -104,9 +107,9 @@ class WarninglistsController extends AppController
             $message = __('Could not update any of the warning lists');
         } else {
             $flashType = 'success';
-            $message = __('Successfully updated ' . $successes . ' warninglists.');
+            $message = __('Successfully updated %s warninglists.', $successes);
             if ($fails != 0) {
-                $message . __(' However, could not update ') . $fails . ' warning list.'; // TODO: non-SVO languages need to be considered
+                $message .= __(' However, could not update %s warninglists.', $fails); // TODO: non-SVO languages need to be considered
             }
         }
         if ($this->_isRest()) {
@@ -186,7 +189,6 @@ class WarninglistsController extends AppController
             $this->Warninglist->regenerateWarninglistCaches($warningList['Warninglist']['id']);
         }
         if ($success) {
-            $this->Warninglist->regenerateWarninglistCaches($id);
             return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => $success . __(' warninglist(s) ') . $message)), 'status' => 200, 'type' => 'json')); // TODO: non-SVO lang considerations
         } else {
             return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => __('Warninglist(s) could not be toggled.'))), 'status' => 200, 'type' => 'json'));
@@ -267,7 +269,6 @@ class WarninglistsController extends AppController
     public function checkValue()
     {
         if ($this->request->is('post')) {
-            $warninglists = $this->Warninglist->getWarninglists(array());
             if (empty($this->request->data)) {
                 throw new NotFoundException(__('No valid data received.'));
             }
@@ -278,13 +279,14 @@ class WarninglistsController extends AppController
             if (array_key_exists('[]', $data)) {
                 $data = $data['[]'];
             }
+
             $hits = array();
+            $warninglists = $this->Warninglist->getEnabled();
             foreach ($data as $dataPoint) {
                 foreach ($warninglists as $warninglist) {
-                    $listValues = $this->Warninglist->getWarninglistEntries($warninglist['Warninglist']['id']);
-                    $listValues = array_combine($listValues, $listValues);
-                    $result = $this->Warninglist->quickCheckValue($listValues, $dataPoint, $warninglist['Warninglist']['type']);
-                    if ($result) {
+                    $values = $this->Warninglist->getFilteredEntries($warninglist);
+                    $result = $this->Warninglist->quickCheckValue($values, $dataPoint, $warninglist['Warninglist']['type']);
+                    if ($result !== false) {
                         $hits[$dataPoint][] = array('id' => $warninglist['Warninglist']['id'], 'name' => $warninglist['Warninglist']['name']);
                     }
                 }

app/Lib/Dashboard/AchievementsWidget.php

@@ -1,17 +1,6 @@
 <?php
 class AchievementsWidget
 {
-    /*
-    * Note: for this widget to display as expected, you need all icons to be accessible in your webroot. (img/custom)
-    * Icons used:
-    * - misp_event.png --> https://user-images.githubusercontent.com/1073662/87687773-6eff6d80-c786-11ea-9dcf-009a158a276c.png
-    * - misp_object.png --> https://user-images.githubusercontent.com/1073662/87687775-6f980400-c786-11ea-985c-b3c15c01d63e.png
-    * - tlp_green.png --> https://raw.githubusercontent.com/MISP/intelligence-icons/master/square_png/48/tlp_green.png
-    * - attack.png --> https://raw.githubusercontent.com/mitre-attack/attack-website/master/attack-theme/static/images/attack-logo.png
-    * - taxonomy.png --> https://raw.githubusercontent.com/MISP/intelligence-icons/master/square_png/48/taxonomy.png
-    * - galaxy.png --> https://raw.githubusercontent.com/MISP/intelligence-icons/master/square_png/48/galaxy.png
-    *
-    */
 
     public $render = 'Achievements';
     public $title = 'Achievements of my organization';
@@ -41,32 +30,32 @@ class AchievementsWidget
     public function __construct(){
         $this->badges = array(
            "events" => array(
-               "icon" => "/img/custom/misp_event.png",
+               "icon" => "/img/ach_misp_event.png",
                "title" => __("MISP is all about sharing relevant data with each other. Start by creating your first event."),
                "help_page" => "https://www.circl.lu/doc/misp/using-the-system/#creating-an-event"
            ),
            "tags" => array(
-               "icon" => "/img/custom/tlp_green.png",
+               "icon" => "/img/ach_tlp_green.png",
                "title" => __("By adding tags to your events, they can be categorized more easily."),
                "help_page" => "https://www.circl.lu/doc/misp/using-the-system/#tagging"
            ),
            "objects" => array(
-               "icon" => "/img/custom/misp_object.png",
+               "icon" => "/img/ach_misp_object.png",
                "title" => __("To enhance the structure of your events, use MISP Objects."),
                "help_page" => "https://github.com/MISP/misp-objects/blob/main/README.md"
            ),
            "taxonomies" => array(
-               "icon" => "/img/custom/taxonomy.png",
+               "icon" => "/img/ach_taxonomy.png",
                "title" => __("Make sure to speak the same language as your counterparts by using taxonomies for your tags."),
                "help_page" => "https://www.circl.lu/doc/misp/taxonomy/"
            ),
            "galaxies" => array (
-               "icon" => "/img/custom/galaxy.png",
+               "icon" => "/img/ach_galaxy.png",
                "title" => __("Go above and beyond tags and taxonomies, and start using galaxies."),
                "help_page" => "https://www.circl.lu/doc/misp/galaxy/"
            ),
            "attack" => array(
-               "icon" => "/img/custom/attack.png",
+               "icon" => "/img/ach_attack.png",
                "title" => __("Add the TTPs following the MITRE ATT&CK framework to make your events even more interesting."),
                "help_page" => "https://www.misp-project.org/2018/06/27/MISP.2.4.93.released.html"
            )

app/Lib/Dashboard/OrgsContributorsGeneric.php

@@ -7,12 +7,12 @@ class OrgsContributorsGeneric
     public $cacheLifetime = 3600;
     public $autoRefreshDelay = false;
     public $params = array (
-        'blacklist_orgs' => 'A list of organisation names to filter out',
+        'blocklist_orgs' => 'A list of organisation names to filter out',
         'timeframe' => 'Number of days considered for the query (30 by default)'
     );
     public $placeholder =
 '{
-    "blacklist_orgs": ["Orgs to filter"],
+    "blocklist_orgs": ["Orgs to filter"],
     "timeframe": "30"
 }';
 
@@ -35,7 +35,7 @@ class OrgsContributorsGeneric
         $orgs = $this->Org->find('all', array( 'conditions' => array('Organisation.local' => 1)));
         $result = array();
         foreach($orgs as $org) {
-            if(!empty($options['blacklist_orgs']) && in_array($org['Organisation']['name'], $options['blacklist_orgs'])) {
+            if(!empty($options['blocklist_orgs']) && in_array($org['Organisation']['name'], $options['blocklist_orgs'])) {
                 continue;
             }
             if ($this->filter($user, $org, $start_timestamp)) {

app/Lib/Dashboard/SharingGraphWidget.php

@@ -15,13 +15,13 @@ class SharingGraphWidget
     public $cacheLifetime = 10;
     public $autoRefreshDelay = false;
     public $params = array (
-        'blacklist_orgs' => 'A list of organisation names to filter out',
+        'blocklist_orgs' => 'A list of organisation names to filter out',
         'months' => 'Number of past months to consider for the graph'
     );
 
     public $placeholder =
 '{
-    "blacklist_orgs": ["Orgs to filter"],
+    "blocklist_orgs": ["Orgs to filter"],
     "months": "6"
 }';
 
@@ -117,8 +117,8 @@ class SharingGraphWidget
         $ghost_orgs = array(); // track orgs without any contribution
         // We start by putting all orgs_id in there:
         foreach($orgs as $org) {
-            // We check for blacklisted orgs
-            if(!empty($options['blacklist_orgs']) && in_array($org['Organisation']['name'], $options['blacklist_orgs'])) {
+            // We check for blocklisted orgs
+            if(!empty($options['blocklist_orgs']) && in_array($org['Organisation']['name'], $options['blocklist_orgs'])) {
                 unset($orgs[$offset]);
             } else {
                 $ghost_orgs[$org['Organisation']['name']] = true;

app/Lib/Export/BroExport.php

@@ -16,6 +16,7 @@ class BroExport
         'ip-src' => array('brotype' => 'ADDR', 'alternate' => array('#/#', 'SUBNET')),
         'ip-dst|port' => array('brotype' => 'ADDR', 'alternate' => array('#/#', 'SUBNET'), 'composite' => 'NONE'),
         'ip-src|port' => array('brotype' => 'ADDR', 'alternate' => array('#/#', 'SUBNET'), 'composite' => 'NONE'),
+        'email' => array('brotype' => 'EMAIL'),
         'email-src' => array('brotype' => 'EMAIL'),
         'email-dst' => array('brotype' => 'EMAIL'),
         'target-email' => array('brotype' => 'EMAIL'),
@@ -79,6 +80,7 @@ class BroExport
             array('domain|ip', 1)
         ),
         'email' => array(
+            array('email', 1),
             array('email-src', 1),
             array('email-dst', 1),
             array('target-email', 1)

app/Lib/Export/CountExport.php

@@ -0,0 +1,36 @@
+<?php
+// You can count on me. Raiders roll.
+class CountExport
+{
+    public $additional_params = array(
+        'flatten' => 1
+    );
+    private $__count = 0;
+    public $non_restrictive_export = true;
+
+    public function handler($data, $options = array())
+    {
+        if ($options['scope'] === 'Attribute') {
+            $this->__count++;
+        }
+        if ($options['scope'] === 'Event') {
+            $this->__count++;
+        }
+        return '';
+    }
+
+    public function header($options = array())
+    {
+        return '';
+    }
+
+    public function footer()
+    {
+        return $this->__count;
+    }
+
+    public function separator()
+    {
+        return "";
+    }
+}

app/Lib/Export/NidsExport.php

@@ -158,6 +158,10 @@ class NidsExport
                 case 'ip-src|port':
                     $this->ipSrcRule($ruleFormat, $item['Attribute'], $sid);
                     break;
+                case 'email':
+                    $this->emailSrcRule($ruleFormat, $item['Attribute'], $sid);
+                    $this->emailDstRule($ruleFormat, $item['Attribute'], $sid);
+                    break;
                 case 'email-src':
                     $this->emailSrcRule($ruleFormat, $item['Attribute'], $sid);
                     break;

app/Lib/Export/OpeniocExport.php

@@ -55,6 +55,7 @@ class OpeniocExport
                 'ip-src' => array('PortItem', 'PortItem/remoteIP', 'IP'),
                 'ip-dst' => array('RouteEntryItem', 'RouteEntryItem/Destination', 'IP'),
                 'hostname' => array('RouteEntryItem', 'RouteEntryItem/Destination', 'string'),
+                'email' => array('Email', 'Email/From', 'string'),
                 'email-src' => array('Email', 'Email/From', 'string'),
                 'email-dst' => array('Email', 'Email/To', 'string'),
                 'email-subject' => array('Email', 'Email/Subject', 'string'),

app/Lib/Tools/ComplexTypeTool.php

@@ -3,7 +3,7 @@ require_once __DIR__ . '/TmpFileTool.php';
 
 class ComplexTypeTool
 {
-    private $__refangRegexTable = array(
+    private static $__refangRegexTable = array(
         array(
             'from' => '/^(hxxp|hxtp|htxp|meow|h\[tt\]p)/i',
             'to' => 'http',
@@ -27,7 +27,7 @@ class ComplexTypeTool
         array(
             'from' => '/[\@]|\[at\]/',
             'to' => '@',
-            'types' => array('email-src', 'email-dst')
+            'types' => array('email', 'email-src', 'email-dst')
         ),
         array(
             'from' => '/\[:\]/',
@@ -38,9 +38,9 @@ class ComplexTypeTool
 
     private $__tlds = null;
 
-    public function refangValue($value, $type)
+    public static function refangValue($value, $type)
     {
-        foreach ($this->__refangRegexTable as $regex) {
+        foreach (self::$__refangRegexTable as $regex) {
             if (in_array($type, $regex['types'])) {
                 $value = preg_replace($regex['from'], $regex['to'], $value);
             }
@@ -248,12 +248,23 @@ class ComplexTypeTool
         128 => array('single' => array('sha512'), 'composite' => array('filename|sha512'))
     );
 
-    // algorithms to run through in order
-    private $__checks = array('Hashes', 'Email', 'IP', 'DomainOrFilename', 'SimpleRegex', 'AS', 'BTC');
+    // algorithms to run through in order, without Hashes that are checked separately
+    private $__checks = array('Email', 'IP', 'DomainOrFilename', 'SimpleRegex', 'AS', 'BTC');
 
+    /**
+     * @param string $raw_input Trimmed value
+     * @return array|false
+     */
     private function __resolveType($raw_input)
     {
-        $input = array('raw' => trim($raw_input));
+        $input = array('raw' => $raw_input);
+
+        // Check hashes before refang and port extracting, it is not necessary for hashes. This speedups parsing
+        // freetexts or CSVs with a lot of hashes.
+        $hashes = $this->__checkForHashes($input);
+        if ($hashes) {
+            return $hashes;
+        }
 
         $input = $this->__refangInput($input);
         $input = $this->__extractPort($input);
@@ -280,7 +291,7 @@ class ComplexTypeTool
         // quick filter for an @ to see if we should validate a potential e-mail address
         if (strpos($input['refanged'], '@') !== false) {
             if (filter_var($input['refanged'], FILTER_VALIDATE_EMAIL)) {
-                return array('types' => array('email-src', 'email-dst', 'target-email', 'whois-registrant-email'), 'to_ids' => true, 'default_type' => 'email-src', 'value' => $input['refanged']);
+                return array('types' => array('email', 'email-src', 'email-dst', 'target-email', 'whois-registrant-email'), 'to_ids' => true, 'default_type' => 'email-src', 'value' => $input['refanged']);
             }
         }
         return false;
@@ -320,7 +331,7 @@ class ComplexTypeTool
             if ($this->__checkForBTC($input)) {
                 $types[] = 'btc';
             }
-            return array('types' => $types, 'to_ids' => true, 'default_type' => $hash['single'][0], 'value' => $input['raw']);
+            return array('types' => $types, 'to_ids' => true, 'default_type' => $types[0], 'value' => $input['raw']);
         }
         // ssdeep has a different pattern
         if ($this->__resolveSsdeep($input['raw'])) {
@@ -347,7 +358,7 @@ class ComplexTypeTool
     private function __refangInput($input)
     {
         $input['refanged'] = $input['raw'];
-        foreach ($this->__refangRegexTable as $regex) {
+        foreach (self::$__refangRegexTable as $regex) {
             $input['refanged'] = preg_replace($regex['from'], $regex['to'], $input['refanged']);
         }
         $input['refanged'] = rtrim($input['refanged'], ".");
@@ -365,7 +376,13 @@ class ComplexTypeTool
     {
         // CVE numbers
         if (preg_match("#^cve-[0-9]{4}-[0-9]{4,9}$#i", $input['raw'])) {
-            return array('types' => array('vulnerability'), 'categories' => array('External analysis'), 'to_ids' => false, 'default_type' => 'vulnerability', 'value' => $input['raw']);
+            return [
+                'types' => ['vulnerability'],
+                'categories' => ['External analysis'],
+                'to_ids' => false,
+                'default_type' => 'vulnerability',
+                'value' => strtoupper($input['raw']), // 'CVE' must be uppercase
+            ];
         }
         // Phone numbers - for automatic recognition, needs to start with + or include dashes
         if ($input['raw'][0] === '+' || strpos($input['raw'], '-')) {
@@ -496,7 +513,7 @@ class ComplexTypeTool
     private function __resolveHash($value)
     {
         $strlen = strlen($value);
-        if (isset($this->__hexHashTypes[$strlen]) && preg_match("#[0-9a-f]{" . $strlen . "}$#i", $value)) {
+        if (isset($this->__hexHashTypes[$strlen]) && ctype_xdigit($value)) {
             return $this->__hexHashTypes[$strlen];
         }
         return false;

app/Lib/Tools/CryptGpgExtended.php

@@ -1,6 +1,16 @@
 <?php
 class CryptGpgExtended extends Crypt_GPG
 {
+    public function __construct(array $options = array())
+    {
+        if (!method_exists($this, '_prepareInput')) {
+            $reflector = new \ReflectionClass('Crypt_GPG');
+            $classPath = $reflector->getFileName();
+            throw new Exception("Crypt_GPG class from '$classPath' is too old, at least version 1.6.1 is required.");
+        }
+        parent::__construct($options);
+    }
+
     /**
      * Export the smallest public key possible from the keyring.
      *
@@ -61,4 +71,91 @@ class CryptGpgExtended extends Crypt_GPG
 
         return $keyData;
     }
+
+    /**
+     * Return key info without importing it.
+     *
+     * @param string $key
+     * @return Crypt_GPG_Key[]
+     * @throws Crypt_GPG_Exception
+     * @throws Crypt_GPG_InvalidOperationException
+     */
+    public function keyInfo($key)
+    {
+        $input = $this->_prepareInput($key, false, false);
+
+        $output = '';
+        $this->engine->reset();
+        $this->engine->setInput($input);
+        $this->engine->setOutput($output);
+        $this->engine->setOperation('--import', ['--import-options', 'show-only', '--with-colons']);
+        $this->engine->run();
+
+        $keys   = array();
+        $key    = null; // current key
+        $subKey = null; // current sub-key
+
+        foreach (explode(PHP_EOL, $output) as $line) {
+            $lineExp = explode(':', $line);
+
+            if ($lineExp[0] === 'pub') {
+                // new primary key means last key should be added to the array
+                if ($key !== null) {
+                    $keys[] = $key;
+                }
+
+                $key = new Crypt_GPG_Key();
+
+                $subKey = Crypt_GPG_SubKey::parse($line);
+                $key->addSubKey($subKey);
+
+            } elseif ($lineExp[0] === 'sub') {
+                $subKey = Crypt_GPG_SubKey::parse($line);
+                $key->addSubKey($subKey);
+
+            } elseif ($lineExp[0] === 'fpr') {
+                $fingerprint = $lineExp[9];
+
+                // set current sub-key fingerprint
+                $subKey->setFingerprint($fingerprint);
+
+            } elseif ($lineExp[0] === 'uid') {
+                $string = stripcslashes($lineExp[9]); // as per documentation
+                $userId = new Crypt_GPG_UserId($string);
+
+                if ($lineExp[1] === 'r') {
+                    $userId->setRevoked(true);
+                }
+
+                $key->addUserId($userId);
+            }
+        }
+
+        // add last key
+        if ($key !== null) {
+            $keys[] = $key;
+        }
+
+        return $keys;
+    }
+
+    /**
+     * @param string $key
+     * @return string
+     * @throws Crypt_GPG_Exception
+     * @throws Crypt_GPG_InvalidOperationException
+     */
+    public function enarmor($key)
+    {
+        $input = $this->_prepareInput($key, false, false);
+
+        $armored = '';
+        $this->engine->reset();
+        $this->engine->setInput($input);
+        $this->engine->setOutput($armored);
+        $this->engine->setOperation('--enarmor');
+        $this->engine->run();
+
+        return $armored;
+    }
 }

app/Lib/Tools/DistributionGraphTool.php

@@ -1,223 +1,221 @@
 <?php
-    class DistributionGraphTool
+class DistributionGraphTool
+{
+    private $__user = false;
+    private $__json = array();
+    /** @var Event */
+    private $__eventModel;
+    /** @var Organisation */
+    private $__organisationModel;
+
+    public function construct(Event $eventModel, array $servers, array $user, $extended_view=0)
     {
-        private $__user = false;
-        private $__json = array();
-        private $__eventModel = false;
+        $this->__eventModel = $eventModel;
+        $this->__serverList = $servers;
+        $this->__organisationModel = $eventModel->Orgc;
+        $this->__user = $user;
+        $this->__json = array();
+        $this->__extended_view = $extended_view;
 
-        public function construct($eventModel, $servers, $user, $extended_view=0)
-        {
-            $this->__eventModel = $eventModel;
-            $this->__serverList = $servers;
-            $this->__organisationModel = $eventModel->Orgc;
-            $this->__user = $user;
-            $this->__json = array();
-            $this->__extended_view = $extended_view;
+        // construct distribution info
+        $this->__json['distributionInfo'] = array();
+        $sgs = $this->__eventModel->SharingGroup->fetchAllAuthorised($this->__user, 'simplified', 1);
+        $this->__json['allSharingGroup'] = h($sgs);
+        $distributionLevels = $this->__eventModel->distributionLevels;
+        foreach ($distributionLevels as $key => $value) {
+            $this->__json['distributionInfo'][$key] = array('key' => h($value), 'desc' => h($this->__eventModel->distributionDescriptions[$key]['formdesc']), 'value' => h($key));
+        }
+        $this->__json['distributionInfo'][5] = ""; // inherit event. Will be deleted afterward
 
-            // construct distribution info
-            $this->__json['distributionInfo'] = array();
-            $sgs = $this->__eventModel->SharingGroup->fetchAllAuthorised($this->__user, 'simplified', 1);
-            $this->__json['allSharingGroup'] = h($sgs);
-            $distributionLevels = $this->__eventModel->distributionLevels;
-            foreach ($distributionLevels as $key => $value) {
-                $this->__json['distributionInfo'][$key] = array('key' => h($value), 'desc' => h($this->__eventModel->distributionDescriptions[$key]['formdesc']), 'value' => h($key));
+        return true;
+    }
+
+    private function __fetchAndAddDistributionInfo($elem)
+    {
+        $distributionLevel = $elem['distribution'];
+
+        if ($distributionLevel == 5) { // inherit -> convert it to the event distribution level
+            $elem['distribution'] = $this->__eventDistribution;
+            $this->__fetchAndAddDistributionInfo($elem);
+        } elseif ($distributionLevel == 4) { // sharing group
+            if (isset($elem['SharingGroup'])) {
+                $sg_name = $elem['SharingGroup']['name'];
+                $this->__addAdditionalDistributionInfo($distributionLevel, $sg_name);
+            } elseif ($this->__eventDistribution == 4) { // event is distributed for sg
+                $sg_name = $this->__eventSharingGroupName;
+                $this->__addAdditionalDistributionInfo($distributionLevel, $sg_name);
             }
-            $this->__json['distributionInfo'][5] = ""; // inherit event. Will be deleted afterward
+        } else {
+            return false;
+        }
+        return true;
+    }
 
-            return true;
+    private function __addAdditionalDistributionInfo($distributionLevel, $data)
+    {
+        if (empty($this->__json['additionalDistributionInfo'][$distributionLevel])) {
+            $this->__json['additionalDistributionInfo'][$distributionLevel] = array();
+        }
+        $this->__json['additionalDistributionInfo'][$distributionLevel][h($data)] = 0; // set-alike
+        if ($distributionLevel == 4) {
+            if (!isset($this->__json['sharingGroupRepartition'][h($data)])) {
+                $this->__json['sharingGroupRepartition'][h($data)] = 0;
+            }
+            $this->__json['sharingGroupRepartition'][h($data)]++;
+        }
+    }
+
+    private function __addOtherDistributionInfo()
+    {
+        // all comm
+        $this->__addAdditionalDistributionInfo(3, "This community"); // add current community
+        $this->__addAdditionalDistributionInfo(3, "All other communities"); // add current community
+
+        // connected
+        $servers = $this->__serverList;
+        $this->__addAdditionalDistributionInfo(2, "This community"); // add current community
+        foreach ($servers as $server) {
+            $this->__addAdditionalDistributionInfo(2, $server);
         }
 
-        private function __extract_sharing_groups_names($sharingArray)
-        {
-            return $sharingArray['name'];
-        }
-
-        private function __fetchAndAddDistributionInfo($elem)
-        {
-            $distributionLevel = $elem['distribution'];
-
-            if ($distributionLevel == 5) { // inherit -> convert it to the event distribution level
-                $elem['distribution'] = $this->__eventDistribution;
-                $this->__fetchAndAddDistributionInfo($elem);
-            } elseif ($distributionLevel == 4) { // sharing group
-                if (isset($elem['SharingGroup'])) {
-                    $sg_name = $this->__extract_sharing_groups_names($elem['SharingGroup']);
-                    $this->__addAdditionalDistributionInfo($distributionLevel, $sg_name);
-                } elseif ($this->__eventDistribution == 4) { // event is distributed for sg
-                    $sg_name = $this->__eventSharingGroupName;
-                    $this->__addAdditionalDistributionInfo($distributionLevel, $sg_name);
-                }
-            } else {
-                return false;
-            }
-            return true;
-        }
-
-        private function __addAdditionalDistributionInfo($distributionLevel, $data)
-        {
-            if (empty($this->__json['additionalDistributionInfo'][$distributionLevel])) {
-                $this->__json['additionalDistributionInfo'][$distributionLevel] = array();
-            }
-            $this->__json['additionalDistributionInfo'][$distributionLevel][h($data)] = 0; // set-alike
-            if ($distributionLevel == 4) {
-                if (!isset($this->__json['sharingGroupRepartition'][h($data)])) {
-                    $this->__json['sharingGroupRepartition'][h($data)] = 0;
-                }
-                $this->__json['sharingGroupRepartition'][h($data)]++;
+        // community
+        $orgs = $this->__organisationModel->find('list', array(
+            'fields' => array('name'),
+            'conditions' => array('local' => true)
+        ));
+        $thisOrg = $this->__user['Organisation']['name'];
+        $this->__addAdditionalDistributionInfo(1, $thisOrg); // add current community
+        foreach ($orgs as $org) {
+            if ($thisOrg != $org) {
+                $this->__addAdditionalDistributionInfo(1, $org);
             }
         }
 
-        private function __addOtherDistributionInfo()
-        {
-            // all comm
-            $this->__addAdditionalDistributionInfo(3, "This community"); // add current community
-            $this->__addAdditionalDistributionInfo(3, "All other communities"); // add current community
-
-            // connected
-            $servers = $this->__serverList;
-            $this->__addAdditionalDistributionInfo(2, "This community"); // add current community
-            foreach ($servers as $server) {
-                $this->__addAdditionalDistributionInfo(2, $server);
-            }
-
-            // community
-            $orgs = $this->__organisationModel->find('list', array(
-                'fields' => array('name'),
-                'conditions' => array('local' => true)
-            ));
-            $thisOrg = $this->__user['Organisation']['name'];
-            $this->__addAdditionalDistributionInfo(1, $thisOrg); // add current community
-            foreach ($orgs as $org) {
-                if ($thisOrg != $org) {
-                    $this->__addAdditionalDistributionInfo(1, $org);
-                }
-            }
-
-            // org only
-            $thisOrg = $this->__user['Organisation']['name'];
-            $this->__addAdditionalDistributionInfo(0, $thisOrg); // add current community
-        }
-
-        private function __get_event($id)
-        {
-            $fullevent = $this->__eventModel->fetchEvent($this->__user, array('eventid' => $id, 'flatten' => 0, 'includeTagRelations' => 1, 'extended' => $this->__extended_view));
-            $event = array();
-            if (empty($fullevent)) {
-                return $event;
-            }
-
-            $fullevent = $fullevent[0];
-            if (!empty($fullevent['Object'])) {
-                $event['Object'] = $fullevent['Object'];
-            } else {
-                $event['Object'] = array();
-            }
-
-            if (!empty($fullevent['Attribute'])) {
-                $event['Attribute'] = $fullevent['Attribute'];
-            } else {
-                $event['Attribute'] = array();
-            }
-            $event['distribution'] = $fullevent['Event']['distribution'];
-
-            if (isset($fullevent['SharingGroup'])) {
-                $event['SharingGroupName'] = $fullevent['SharingGroup']['name'];
-            } else {
-                $event['SharingGroupName'] = "?";
-            }
+        // org only
+        $thisOrg = $this->__user['Organisation']['name'];
+        $this->__addAdditionalDistributionInfo(0, $thisOrg); // add current community
+    }
 
+    /**
+     * Fetch event containing just 'Attribute', 'Object', 'SharingGroup' and 'distribution'
+     * @param int $id
+     * @return array
+     * @throws Exception
+     */
+    private function __get_event($id)
+    {
+        $fullevent = $this->__eventModel->fetchEvent($this->__user, array(
+            'eventid' => $id,
+            'flatten' => 0,
+            'noShadowAttributes' => true,
+            'noEventReports' => true,
+            'noSightings' => true,
+            'extended' => $this->__extended_view,
+        ));
+        $event = array();
+        if (empty($fullevent)) {
             return $event;
         }
 
-        public function get_distributions_graph($id)
-        {
-            $this->__json['event'] = $this->init_array_distri();
-            $this->__json['attribute'] = $this->init_array_distri();
-            $this->__json['object'] = $this->init_array_distri();
-            $this->__json['obj_attr'] = $this->init_array_distri();
-            $this->__json['additionalDistributionInfo'] = $this->init_array_distri(array());
-            $this->__json['sharingGroupRepartition'] = array();
+        $fullevent = $fullevent[0];
+        if (isset($fullevent['Object'])) {
+            $event['Object'] = $fullevent['Object'];
+        } else {
+            $event['Object'] = array();
+        }
 
-            $this->__addOtherDistributionInfo();
+        if (isset($fullevent['Attribute'])) {
+            $event['Attribute'] = $fullevent['Attribute'];
+        } else {
+            $event['Attribute'] = array();
+        }
+        $event['distribution'] = $fullevent['Event']['distribution'];
 
-            // transform set into array
-            foreach (array_keys($this->__json['additionalDistributionInfo']) as $d) {
-                $this->__json['additionalDistributionInfo'][$d] = array_keys($this->__json['additionalDistributionInfo'][$d]);
-            }
+        if (isset($fullevent['SharingGroup'])) {
+            $event['SharingGroupName'] = $fullevent['SharingGroup']['name'];
+        } else {
+            $event['SharingGroupName'] = "?";
+        }
 
-            if ($id === -1) {
-                return $this->__json;
-            }
-            $event = $this->__get_event($id);
-            $eventDist = $event['distribution'];
-            $eventSGName = $event['SharingGroupName'];
-            $this->__eventDistribution = $eventDist;
-            $this->__eventSharingGroupName = $eventSGName;
+        return $event;
+    }
 
-            if (empty($event)) {
-                return $this->__json;
-            }
+    public function get_distributions_graph($id)
+    {
+        $this->__json['event'] = $this->init_array_distri();
+        $this->__json['attribute'] = $this->init_array_distri();
+        $this->__json['object'] = $this->init_array_distri();
+        $this->__json['obj_attr'] = $this->init_array_distri();
+        $this->__json['additionalDistributionInfo'] = $this->init_array_distri(array());
+        $this->__json['sharingGroupRepartition'] = array();
 
-            if (!empty($event['Object'])) {
-                $object = $event['Object'];
-            } else {
-                $object = array();
-            }
+        $this->__addOtherDistributionInfo();
 
-            if (!empty($event['Attribute'])) {
-                $attribute = $event['Attribute'];
-            } else {
-                $attribute = array();
-            }
-
-            // extract distribution
-            foreach ($attribute as $attr) {
-                $distri = $attr['distribution'];
-                $this->__json['event'][$distri] += 1;
-                $this->__json['attribute'][$distri] += 1;
-                $this->__fetchAndAddDistributionInfo($attr);
-            }
-
-            foreach ($object as $obj) {
-                $distri = $obj['distribution'];
-                $this->__json['event'][$distri] += 1;
-                $this->__json['object'][$distri] += 1;
-                $this->__fetchAndAddDistributionInfo($obj);
-
-                $added_value = array();
-                if (!empty($obj['Attribute'])) {
-                    foreach ($obj['Attribute'] as $objAttr) {
-                        $distri = $objAttr['distribution'];
-                        $this->__json['event'][$distri] += 1;
-                        $this->__json['obj_attr'][$distri] += 1;
-                        $this->__fetchAndAddDistributionInfo($objAttr);
-                    }
-                }
-            }
-            // distribution 5 is inherit event, apply this fact on values
-            $this->__json['event'][$eventDist] += $this->__json['event'][5];
-            unset($this->__json['event'][5]);
-            $this->__json['attribute'][$eventDist] += $this->__json['attribute'][5];
-            unset($this->__json['attribute'][5]);
-            $this->__json['object'][$eventDist] += $this->__json['object'][5];
-            unset($this->__json['object'][5]);
-            $this->__json['obj_attr'][$eventDist] += $this->__json['obj_attr'][5];
-            unset($this->__json['obj_attr'][5]);
-
-            unset($this->__json['distributionInfo'][5]); // inherit event.
-
-            // transform set into array for SG (others are already done)
-            $this->__json['additionalDistributionInfo'][4] = array_keys($this->__json['additionalDistributionInfo'][4]);
+        // transform set into array
+        foreach (array_keys($this->__json['additionalDistributionInfo']) as $d) {
+            $this->__json['additionalDistributionInfo'][$d] = array_keys($this->__json['additionalDistributionInfo'][$d]);
+        }
 
+        if ($id === -1) {
+            return $this->__json;
+        }
+        $event = $this->__get_event($id);
+        if (empty($event)) {
             return $this->__json;
         }
 
-        public function init_array_distri($default=0)
-        {
-            $ret = array();
-            foreach ($this->__json['distributionInfo'] as $d => $v) {
-                $ret[h($d)] = $default;
-            }
-            return $ret;
+        $eventDist = $event['distribution'];
+        $eventSGName = $event['SharingGroupName'];
+        $this->__eventDistribution = $eventDist;
+        $this->__eventSharingGroupName = $eventSGName;
+
+        // extract distribution
+        foreach ($event['Attribute'] as $attr) {
+            $distri = $attr['distribution'];
+            $this->__json['event'][$distri] += 1;
+            $this->__json['attribute'][$distri] += 1;
+            $this->__fetchAndAddDistributionInfo($attr);
         }
+
+        foreach ($event['Object'] as $obj) {
+            $distri = $obj['distribution'];
+            $this->__json['event'][$distri] += 1;
+            $this->__json['object'][$distri] += 1;
+            $this->__fetchAndAddDistributionInfo($obj);
+
+            if (!empty($obj['Attribute'])) {
+                foreach ($obj['Attribute'] as $objAttr) {
+                    $distri = $objAttr['distribution'];
+                    $this->__json['event'][$distri] += 1;
+                    $this->__json['obj_attr'][$distri] += 1;
+                    $this->__fetchAndAddDistributionInfo($objAttr);
+                }
+            }
+        }
+        // distribution 5 is inherit event, apply this fact on values
+        $this->__json['event'][$eventDist] += $this->__json['event'][5];
+        unset($this->__json['event'][5]);
+        $this->__json['attribute'][$eventDist] += $this->__json['attribute'][5];
+        unset($this->__json['attribute'][5]);
+        $this->__json['object'][$eventDist] += $this->__json['object'][5];
+        unset($this->__json['object'][5]);
+        $this->__json['obj_attr'][$eventDist] += $this->__json['obj_attr'][5];
+        unset($this->__json['obj_attr'][5]);
+
+        unset($this->__json['distributionInfo'][5]); // inherit event.
+
+        // transform set into array for SG (others are already done)
+        $this->__json['additionalDistributionInfo'][4] = array_keys($this->__json['additionalDistributionInfo'][4]);
+
+        return $this->__json;
     }
+
+    public function init_array_distri($default=0)
+    {
+        $ret = array();
+        foreach ($this->__json['distributionInfo'] as $d => $v) {
+            $ret[h($d)] = $default;
+        }
+        return $ret;
+    }
+}

app/Lib/Tools/GpgTool.php

@@ -5,7 +5,7 @@ class GpgTool
      * @return CryptGpgExtended
      * @throws Exception
      */
-    public function initializeGpg()
+    public static function initializeGpg()
     {
         if (!class_exists('Crypt_GPG')) {
             // 'Crypt_GPG' class cannot be autoloaded, try to require from include_path.
@@ -31,6 +31,14 @@ class GpgTool
         return new CryptGpgExtended($options);
     }
 
+    /** @var CryptGpgExtended */
+    private $gpg;
+
+    public function __construct($gpg)
+    {
+        $this->gpg = $gpg;
+    }
+
     /**
      * @param string $search
      * @return array
@@ -65,9 +73,41 @@ class GpgTool
 
         $key = $response->body;
 
+        if ($this->gpg) {
+            $fetchedFingerprint = $this->validateGpgKey($key);
+            if (strtolower($fingerprint) !== strtolower($fetchedFingerprint)) {
+                throw new Exception("Requested fingerprint do not match with fetched key fingerprint ($fingerprint != $fetchedFingerprint)");
+            }
+        }
+
         return $key;
     }
 
+    /**
+     * Validates PGP key
+     * @param string $keyData
+     * @return string Primary key fingerprint
+     * @throws Exception
+     */
+    public function validateGpgKey($keyData)
+    {
+        if (!$this->gpg instanceof CryptGpgExtended) {
+            throw new InvalidArgumentException("Valid CryptGpgExtended instance required.");
+        }
+        $fetchedKeyInfo = $this->gpg->keyInfo($keyData);
+        if (empty($fetchedKeyInfo)) {
+            throw new Exception("No key found");
+        }
+        if (count($fetchedKeyInfo) !== 1) {
+            throw new Exception("Multiple keys found");
+        }
+        $primaryKey = $fetchedKeyInfo[0]->getPrimaryKey();
+        if (empty($primaryKey)) {
+            throw new Exception("No primary key found");
+        }
+        return $primaryKey->getFingerprint();
+    }
+
     /**
      * @param string $body
      * @return array
@@ -107,6 +147,91 @@ class GpgTool
         return $final;
     }
 
+    /**
+     * @see https://tools.ietf.org/html/draft-koch-openpgp-webkey-service-10
+     * @param string $email
+     * @return string
+     * @throws Exception
+     */
+    public function wkd($email)
+    {
+        if (!$this->gpg instanceof CryptGpgExtended) {
+            throw new InvalidArgumentException("Valid CryptGpgExtended instance required.");
+        }
+
+        $parts = explode('@', $email);
+        if (count($parts) !== 2) {
+            throw new InvalidArgumentException("Invalid e-mail address provided.");
+        }
+
+        list($localPart, $domain) = $parts;
+        $localPart = strtolower($localPart);
+        $localPartHash = $this->zbase32(sha1($localPart, true));
+
+        $advancedUrl = "https://openpgpkey.$domain/.well-known/openpgpkey/" . strtolower($domain) . "/hu/$localPartHash";
+        try {
+            $response = $this->keyServerLookup($advancedUrl);
+            return $this->processWkdResponse($response);
+        } catch (Exception $e) {
+            // pass, continue to direct method
+        }
+
+        $directUrl = "https://$domain/.well-known/openpgpkey/hu/$localPartHash";
+        $response = $this->keyServerLookup($directUrl);
+        return $this->processWkdResponse($response);
+    }
+
+    /**
+     * @param HttpSocketResponse $response
+     * @return string
+     * @throws Crypt_GPG_Exception
+     * @throws Crypt_GPG_InvalidOperationException
+     */
+    private function processWkdResponse(HttpSocketResponse $response)
+    {
+        if ($response->code == 404) {
+            throw new NotFoundException("Key not found");
+        } else if (!$response->isOk()) {
+            throw new Exception("Fetching the WKD failed with HTTP error {$response->code}: {$response->reasonPhrase}");
+        }
+
+        return $this->gpg->enarmor($response->body());
+    }
+
+    /**
+     * Converts data to zbase32 string.
+     *
+     * @see http://philzimmermann.com/docs/human-oriented-base-32-encoding.txt
+     * @param string $data
+     * @return string
+     */
+    private function zbase32($data)
+    {
+        $chars = 'ybndrfg8ejkmcpqxot1uwisza345h769'; // lower-case
+        $res = '';
+        $remainder = 0;
+        $remainderSize = 0;
+
+        for ($i = 0; $i < strlen($data); $i++) {
+            $b = ord($data[$i]);
+            $remainder = ($remainder << 8) | $b;
+            $remainderSize += 8;
+            while ($remainderSize > 4) {
+                $remainderSize -= 5;
+                $c = $remainder & (31 << $remainderSize);
+                $c >>= $remainderSize;
+                $res .= $chars[$c];
+            }
+        }
+        if ($remainderSize > 0) {
+            // remainderSize < 5:
+            $remainder <<= (5 - $remainderSize);
+            $c = $remainder & 31;
+            $res .= $chars[$c];
+        }
+        return $res;
+    }
+
     /**
      * @param string $uri
      * @return HttpSocketResponse

app/Lib/Tools/IOCExportTool.php

@@ -86,6 +86,7 @@ class IOCExportTool
                 'ip-src' => array('PortItem', 'PortItem/remoteIP', 'IP'),
                 'ip-dst' => array('RouteEntryItem', 'RouteEntryItem/Destination', 'IP'),
                 'hostname' => array('RouteEntryItem', 'RouteEntryItem/Destination', 'string'),
+                'email' => array('Email', 'Email/From', 'string'),
                 'email-src' => array('Email', 'Email/From', 'string'),
                 'email-dst' => array('Email', 'Email/To', 'string'),
                 'email-subject' => array('Email', 'Email/Subject', 'string'),

app/Lib/Tools/JSONConverterTool.php

@@ -53,7 +53,7 @@ class JSONConverterTool
 
     public function convert($event, $isSiteAdmin=false, $raw = false)
     {
-        $toRearrange = array('Org', 'Orgc', 'SharingGroup', 'Attribute', 'ShadowAttribute', 'RelatedAttribute', 'RelatedEvent', 'Galaxy', 'Object');
+        $toRearrange = array('Org', 'Orgc', 'SharingGroup', 'Attribute', 'ShadowAttribute', 'RelatedAttribute', 'RelatedEvent', 'Galaxy', 'Object', 'EventReport');
         foreach ($toRearrange as $object) {
             if (isset($event[$object])) {
                 $event['Event'][$object] = $event[$object];

app/Lib/Tools/SendEmail.php

@@ -366,6 +366,10 @@ class SendEmail
             throw new SendEmailException('Emailing is currently disabled on this instance.');
         }
 
+        if (!isset($user['User'])) {
+            throw new InvalidArgumentException("Invalid user model provided.");
+        }
+
         // Check if the e-mail can be encrypted
         $canEncryptGpg = isset($user['User']['gpgkey']) && !empty($user['User']['gpgkey']);
         $canEncryptSmime = isset($user['User']['certif_public']) && !empty($user['User']['certif_public']) && Configure::read('SMIME.enabled');
@@ -392,7 +396,7 @@ class SendEmail
             try {
                 $gnupgEmail = Configure::read('GnuPG.email');
                 if (empty($gnupgEmail)) {
-                    throw new Exception("Email signing is enabled but variable 'GnuPG.email' is not set.");
+                    throw new Exception("GPG email signing is enabled but variable 'GnuPG.email' is not set.");
                 }
 
                 $this->gpg->addSignKey($gnupgEmail, Configure::read('GnuPG.password'));
@@ -402,7 +406,7 @@ class SendEmail
 
                 $signed = true;
             } catch (Exception $e) {
-                throw new SendEmailException("The message could not be signed.", 0, $e);
+                throw new SendEmailException("The message could not be signed by GPG.", 0, $e);
             }
         }
 
@@ -415,11 +419,11 @@ class SendEmail
             try {
                 $fingerprint = $this->importAndValidateGpgPublicKey($user['User']['gpgkey']);
             } catch (Crypt_GPG_NoDataException $e) {
-                throw new SendEmailException("The message could not be encrypted because the provided key is invalid.", 0, $e);
+                throw new SendEmailException("The message could not be encrypted because the provided GPG key is invalid.", 0, $e);
             }
 
             if (!$fingerprint) {
-                throw new SendEmailException("The message could not be encrypted because the provided key is either expired or cannot be used for encryption.");
+                throw new SendEmailException("The message could not be encrypted because the provided GPG key is either expired or cannot be used for encryption.");
             }
 
             try {
@@ -437,7 +441,7 @@ class SendEmail
 
                 $encrypted = true;
             } catch (Exception $e) {
-                throw new SendEmailException('The message could not be encrypted.', 0, $e);
+                throw new SendEmailException('The message could not be encrypted by GPG.', 0, $e);
             }
         }
 
@@ -468,24 +472,23 @@ class SendEmail
             // Try to encrypt empty message
             $this->encryptTextBySmime($certificate, '');
         } catch (SendEmailException $e) {
-            throw new Exception('This certificate cannot be used to encrypt email.', 0, $e);
+            throw new Exception('This S/MIME certificate cannot be used to encrypt email.', 0, $e);
         }
 
         $parsed = openssl_x509_parse($certificate);
 
         if (!$parsed) {
-            throw new Exception('Could not parse certificate');
+            throw new Exception('Could not parse S/MIME certificate');
         }
 
-        // Purpose '5' should be 'smimeencrypt'
-        if (!($parsed['purposes'][5][0] === 1 && $parsed['purposes'][5][2] === 'smimeencrypt')) {
-            throw new Exception('This certificate cannot be used to encrypt email.');
+        if ($parsed['purposes'][X509_PURPOSE_SMIME_ENCRYPT][0] !== true) {
+            throw new Exception('This S/MIME certificate cannot be used to encrypt email.');
         }
 
         $now = new DateTime();
         $validToTime = new DateTime("@{$parsed['validTo_time_t']}");
         if ($validToTime <= $now) {
-            throw new Exception('This certificate is expired.');
+            throw new Exception('This S/MIME certificate expired at ' . $validToTime->format('c'));
         }
 
         return true;
@@ -512,6 +515,9 @@ class SendEmail
         // If the e-mail is sent on behalf of a user, then we want the target user to be able to respond to the sender.
         // For this reason we should also attach the public key of the sender along with the message (if applicable).
         if ($replyToUser) {
+            if (!isset($replyToUser['User']['email'])) {
+                throw new InvalidArgumentException("Invalid replyToUser model provided.");
+            }
             $email->replyTo($replyToUser['User']['email']);
             if (!empty($replyToUser['User']['gpgkey'])) {
                 $attachments['gpgkey.asc'] = $replyToUser['User']['gpgkey'];
@@ -768,6 +774,7 @@ class SendEmail
      * @param string $content
      * @return File[]
      * @throws SendEmailException
+     * @throws MethodNotAllowedException
      */
     private function createInputOutputFiles($content)
     {

app/Model/Allowedlist.php

@@ -2,9 +2,9 @@
 
 App::uses('AppModel', 'Model');
 
-class Whitelist extends AppModel
+class Allowedlist extends AppModel
 {
-    public $useTable = 'whitelist';
+    public $useTable = 'allowedlist';
 
     public $displayField = 'name';
 
@@ -17,7 +17,7 @@ class Whitelist extends AppModel
             ),
     );
 
-    public $whitelistedItems = false;
+    public $allowedlistedItems = false;
 
     public $validate = array(
         'name' => array(
@@ -26,7 +26,7 @@ class Whitelist extends AppModel
             ),
             'userdefined' => array(
                 'rule' => array('validateValue'),
-                'message' => 'Name not in the right format. Whitelist entries have to be enclosed by a valid php delimiter (which can be most non-alphanumeric / non-whitespace character). Format: "/8.8.8.8/" Please double check the name.',             //'allowEmpty' => false,
+                'message' => 'Name not in the right format. Allowedlist entries have to be enclosed by a valid php delimiter (which can be most non-alphanumeric / non-whitespace character). Format: "/8.8.8.8/" Please double check the name.',             //'allowEmpty' => false,
                 //'allowEmpty' => false,
                 //'required' => true,
                 //'last' => false, // Stop validation after this rule
@@ -56,9 +56,9 @@ class Whitelist extends AppModel
     {
         $value = $fields['name'];
 
-        $whitelist = $this->find('all', array('recursive' => 0,'fields' => 'name'));
-        foreach ($whitelist as $whitelistItem) {
-            if ($value == $whitelistItem['Whitelist']['name']) {
+        $allowedlist = $this->find('all', array('recursive' => 0,'fields' => 'name'));
+        foreach ($allowedlist as $allowedlistItem) {
+            if ($value == $allowedlistItem['Allowedlist']['name']) {
                 return false;
             }
         }
@@ -68,28 +68,28 @@ class Whitelist extends AppModel
 
     public function getBlockedValues()
     {
-        if ($this->whitelistedItems === false) {
-            $Whitelists = $this->find('all', array('fields' => array('name')));
-            $this->whitelistedItems = array();
-            foreach ($Whitelists as $item) {
-                $this->whitelistedItems[] = $item['Whitelist']['name'];
+        if ($this->allowedlistedItems === false) {
+            $Allowedlists = $this->find('all', array('fields' => array('name')));
+            $this->allowedlistedItems = array();
+            foreach ($Allowedlists as $item) {
+                $this->allowedlistedItems[] = $item['Allowedlist']['name'];
             }
         }
-        return $this->whitelistedItems;
+        return $this->allowedlistedItems;
     }
 
-    public function removeWhitelistedFromArray($data, $isAttributeArray)
+    public function removeAllowedlistedFromArray($data, $isAttributeArray)
     {
-        // Let's get all of the values that will be blocked by the whitelist
-        $whitelists = $this->getBlockedValues();
-        // if we don't have any whitelist items in the db, don't loop through each attribute
-        if (!empty($whitelists)) {
+        // Let's get all of the values that will be blocked by the allowedlist
+        $allowedlists = $this->getBlockedValues();
+        // if we don't have any allowedlist items in the db, don't loop through each attribute
+        if (!empty($allowedlists)) {
             // if $isAttributeArray, we know that we have just an array of attributes
             if ($isAttributeArray) {
-                // loop through each attribute and unset the ones that are whitelisted
+                // loop through each attribute and unset the ones that are allowedlisted
                 foreach ($data as $k => $attribute) {
-                    // loop through each whitelist item and run a preg match against the attribute value. If it matches, unset the attribute
-                    foreach ($whitelists as $wlitem) {
+                    // loop through each allowedlist item and run a preg match against the attribute value. If it matches, unset the attribute
+                    foreach ($allowedlists as $wlitem) {
                         if (preg_match($wlitem, $attribute['Attribute']['value'])) {
                             unset($data[$k]);
                         }
@@ -100,10 +100,10 @@ class Whitelist extends AppModel
                 // if !$isAttributeArray, we know that we have an array of events that we need to parse through
                 foreach ($data as $ke => $event) {
                     if (isset($event['Attribute'])) {
-                        // loop through each attribute and unset the ones that are whitelisted
+                        // loop through each attribute and unset the ones that are allowedlisted
                         foreach ($event['Attribute'] as $k => $attribute) {
-                            // loop through each whitelist item and run a preg match against the attribute value. If it matches, unset the attribute
-                            foreach ($whitelists as $wlitem) {
+                            // loop through each allowedlist item and run a preg match against the attribute value. If it matches, unset the attribute
+                            foreach ($allowedlists as $wlitem) {
                                 if (preg_match($wlitem, $attribute['value'])) {
                                     unset($data[$ke]['Attribute'][$k]);
                                 }
@@ -117,14 +117,14 @@ class Whitelist extends AppModel
         return $data;
     }
 
-    // A simplified whitelist removal, for when we just want to throw values against the list instead of attributes / events
-    public function removeWhitelistedValuesFromArray($data)
+    // A simplified allowedlist removal, for when we just want to throw values against the list instead of attributes / events
+    public function removeAllowedlistedValuesFromArray($data)
     {
-        $whitelists = $this->getBlockedValues();
-        // if we don't have any whitelist items in the db, don't loop through each attribute
-        if (!empty($whitelists)) {
+        $allowedlists = $this->getBlockedValues();
+        // if we don't have any allowedlist items in the db, don't loop through each attribute
+        if (!empty($allowedlists)) {
             foreach ($data as $k => $value) {
-                foreach ($whitelists as $wlitem) {
+                foreach ($allowedlists as $wlitem) {
                     if (preg_match($wlitem, $value)) {
                         unset($data[$k]);
                     }

app/Model/AppModel.php

@@ -85,7 +85,8 @@ class AppModel extends Model
         33 => false, 34 => false, 35 => false, 36 => false, 37 => false, 38 => false,
         39 => false, 40 => false, 41 => false, 42 => false, 43 => false, 44 => false,
         45 => false, 46 => false, 47 => false, 48 => false, 49 => false, 50 => false,
-        51 => false, 52 => false, 53 => false, 54 => false, 55 => false,
+        51 => false, 52 => false, 53 => false, 54 => false, 55 => false, 56 => false,
+        57 => false, 58 => false, 59 => false
     );
 
     public $advanced_updates_description = array(
@@ -196,16 +197,16 @@ class AppModel extends Model
                 $this->Sighting->deleteAll(array('NOT' => array('Sighting.type' => array(0, 1, 2))));
                 break;
             case '2.4.71':
-                $this->OrgBlacklist = Classregistry::init('OrgBlacklist');
+                $this->OrgBlocklist = Classregistry::init('OrgBlocklist');
                 $values = array(
                     array('org_uuid' => '58d38339-7b24-4386-b4b4-4c0f950d210f', 'org_name' => 'Setec Astrononomy', 'comment' => 'default example'),
                     array('org_uuid' => '58d38326-eda8-443a-9fa8-4e12950d210f', 'org_name' => 'Acme Finance', 'comment' => 'default example')
                 );
                 foreach ($values as $value) {
-                    $found = $this->OrgBlacklist->find('first', array('conditions' => array('org_uuid' => $value['org_uuid']), 'recursive' => -1));
+                    $found = $this->OrgBlocklist->find('first', array('conditions' => array('org_uuid' => $value['org_uuid']), 'recursive' => -1));
                     if (empty($found)) {
-                        $this->OrgBlacklist->create();
-                        $this->OrgBlacklist->save($value);
+                        $this->OrgBlocklist->create();
+                        $this->OrgBlocklist->save($value);
                     }
                 }
                 $dbUpdateSuccess = $this->updateDatabase($command);
@@ -1382,18 +1383,18 @@ class AppModel extends Model
                 break;
             case 51:
                 $sqlArray[] = "ALTER TABLE `feeds` ADD `orgc_id` int(11) NOT NULL DEFAULT 0";
-                $this->__addIndex('feeds', 'orgc_id');
+                $indexArray[] = array('feeds', 'orgc_id');
                 break;
             case 52:
                 if (!empty($this->query("SHOW COLUMNS FROM `admin_settings` LIKE 'key';"))) {
                     $sqlArray[] = "ALTER TABLE admin_settings CHANGE `key` `setting` varchar(255) COLLATE utf8_bin NOT NULL;";
-                    $this->__addIndex('admin_settings', 'setting');
+                    $indexArray[] = array('admin_settings', 'setting');
                 }
                 break;
             case 53:
                 if (!empty($this->query("SHOW COLUMNS FROM `user_settings` LIKE 'key';"))) {
                     $sqlArray[] = "ALTER TABLE user_settings CHANGE `key` `setting` varchar(255) COLLATE utf8_bin NOT NULL;";
-                    $this->__addIndex('user_settings', 'setting');
+                    $indexArray[] = array('user_settings', 'setting');
                 }
                 break;
             case 54:
@@ -1407,6 +1408,34 @@ class AppModel extends Model
                 $this->__dropIndex('correlations', 'sharing_group_id');
                 $this->__dropIndex('correlations', 'a_sharing_group_id');
                 break;
+            case 56:
+                //rename tables
+                $sqlArray[] = "RENAME TABLE `org_blacklists` TO `org_blocklists`;";
+                $sqlArray[] = "RENAME TABLE `event_blacklists` TO `event_blocklists`;";
+                $sqlArray[] = "RENAME TABLE `whitelist` TO `allowedlist`;";
+                break;
+            case 57:
+                $sqlArray[] = sprintf("INSERT INTO `admin_settings` (`setting`, `value`) VALUES ('fix_login', %s);", time());
+                break;
+            case 58:
+                $sqlArray[] = "ALTER TABLE `warninglists` MODIFY COLUMN `warninglist_entry_count` int(11) unsigned NOT NULL DEFAULT 0;";
+                break;
+            case 59:
+                $sqlArray[] = "CREATE TABLE IF NOT EXISTS event_reports (
+                    `id` int(11) NOT NULL AUTO_INCREMENT,
+                    `uuid` varchar(40) COLLATE utf8_bin NOT NULL ,
+                    `event_id` int(11) NOT NULL,
+                    `name` varchar(255) NOT NULL,
+                    `content` text,
+                    `distribution` tinyint(4) NOT NULL DEFAULT 0,
+                    `sharing_group_id` int(11),
+                    `timestamp` int(11) NOT NULL,
+                    `deleted` tinyint(1) NOT NULL DEFAULT 0,
+                    PRIMARY KEY (id),
+                    CONSTRAINT u_uuid UNIQUE (uuid),
+                    INDEX `name` (`name`)
+                ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;";
+                break;
             case 'fixNonEmptySharingGroupID':
                 $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
                 $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
@@ -1555,7 +1584,7 @@ class AppModel extends Model
                             break;
                         }
                     } else {
-                        $logMessage['change'] = $logMessage['change'] . PHP_EOL . __('However, as this error is whitelisted, the update went through.');
+                        $logMessage['change'] = $logMessage['change'] . PHP_EOL . __('However, as this error is allowed, the update went through.');
                     }
                     $this->Log->save($logMessage);
                 }
@@ -1721,14 +1750,6 @@ class AppModel extends Model
         }
     }
 
-    public function checkMISPVersion()
-    {
-        $file = new File(ROOT . DS . 'VERSION.json', true);
-        $version_array = json_decode($file->read(), true);
-        $file->close();
-        return $version_array;
-    }
-
     public function getPythonVersion()
     {
         if (!empty(Configure::read('MISP.python_bin'))) {
@@ -1878,9 +1899,7 @@ class AppModel extends Model
                     $workerType = '';
                     if (isset($workerDiagnostic['update']['ok']) && $workerDiagnostic['update']['ok']) {
                         $workerType = 'update';
-                    } elseif (isset($workerDiagnostic['prio']['ok']) && $workerDiagnostic['prio']['ok']) {
-                        $workerType = 'prio';
-                    } else { // no worker running, doing inline update
+                    } else { // update worker not running, doing the update inline
                         return $this->runUpdates($verbose, false);
                     }
                     $this->Job->create();
@@ -2406,19 +2425,6 @@ class AppModel extends Model
         $this->elasticSearchClient = $client;
     }
 
-    public function checkVersionRequirements($versionString, $minVersion)
-    {
-        $version = explode('.', $versionString);
-        $minVersion = explode('.', $minVersion);
-        if (count($version) > $minVersion) {
-            return true;
-        }
-        if (count($version) == 1) {
-            return $minVersion <= $version;
-        }
-        return ($version[0] >= $minVersion[0] && $version[1] >= $minVersion[1] && $version[2] >= $minVersion[2]);
-    }
-
     // generate a generic subquery - options needs to include conditions
     public function subQueryGenerator($model, $options, $lookupKey, $negation = false)
     {
@@ -2556,35 +2562,67 @@ class AppModel extends Model
         return $HttpSocket;
     }
 
-    public function setupSyncRequest($server)
+    /**
+     * @param array $server
+     * @return array[]
+     * @throws JsonException
+     */
+    protected function setupSyncRequest(array $server)
     {
+        $version = implode('.', $this->checkMISPVersion());
         $request = array(
-                'header' => array(
-                        'Authorization' => $server['Server']['authkey'],
-                        'Accept' => 'application/json',
-                        'Content-Type' => 'application/json'
-                )
+            'header' => array(
+                'Authorization' => $server['Server']['authkey'],
+                'Accept' => 'application/json',
+                'Content-Type' => 'application/json',
+                'MISP-version' => $version,
+            )
         );
-        $request = $this->addHeaders($request);
-        return $request;
-    }
 
-    public function addHeaders($request)
-    {
-        $version = $this->checkMISPVersion();
-        $version = implode('.', $version);
-        try {
-            $commit = trim(shell_exec('git log --pretty="%H" -n1 HEAD'));
-        } catch (Exception $e) {
-            $commit = false;
-        }
-        $request['header']['MISP-version'] = $version;
+        $commit = $this->checkMIPSCommit();
         if ($commit) {
             $request['header']['commit'] = $commit;
         }
+        $request['header']['User-Agent'] = 'MISP ' . $version . (empty($commit) ? '' : ' - #' . $commit);
         return $request;
     }
 
+    /**
+     * Returns MISP version from VERSION.json file as array with major, minor and hotfix keys.
+     *
+     * @return array
+     * @throws JsonException
+     */
+    public function checkMISPVersion()
+    {
+        static $versionArray;
+        if ($versionArray === null) {
+            $file = new File(ROOT . DS . 'VERSION.json', true);
+            $versionArray = $this->jsonDecode($file->read());
+            $file->close();
+        }
+        return $versionArray;
+    }
+
+    /**
+     * Returns MISP commit hash.
+     *
+     * @return false|string
+     */
+    protected function checkMIPSCommit()
+    {
+        static $commit;
+        if ($commit === null) {
+            $commit = shell_exec('git log --pretty="%H" -n1 HEAD');
+            if ($commit) {
+                $commit = trim($commit);
+            } else {
+                $commit = false;
+            }
+        }
+        return $commit;
+    }
+
     // take filters in the {"OR" => [foo], "NOT" => [bar]} format along with conditions and set the conditions
     public function generic_add_filter($conditions, &$filter, $keys)
     {

app/Model/Attribute.php

@@ -7,9 +7,11 @@ App::uses('FinancialTool', 'Tools');
 App::uses('RandomTool', 'Tools');
 App::uses('AttachmentTool', 'Tools');
 App::uses('TmpFileTool', 'Tools');
+App::uses('ComplexTypeTool', 'Tools');
 
 /**
  * @property Event $Event
+ * @property AttributeTag $AttributeTag
  */
 class Attribute extends AppModel
 {
@@ -96,16 +98,16 @@ class Attribute extends AppModel
             'Payload delivery' => array(
                     'desc' => __('Information about how the malware is delivered'),
                     'formdesc' => __('Information about the way the malware payload is initially delivered, for example information about the email or web-site, vulnerability used, originating IP etc. Malware sample itself should be attached here.'),
-                    'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash','filename|impfuzzy', 'filename|pehash', 'mac-address', 'mac-eui-64', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'hostname', 'domain', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'email-body', 'url', 'user-agent', 'AS', 'pattern-in-file', 'pattern-in-traffic', 'stix2-pattern', 'yara', 'sigma', 'mime-type', 'attachment', 'malware-sample', 'link', 'malware-type', 'comment', 'text', 'hex', 'vulnerability', 'weakness', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'other', 'hostname|port', 'email-dst-display-name', 'email-src-display-name', 'email-header', 'email-reply-to', 'email-x-mailer', 'email-mime-boundary', 'email-thread-index', 'email-message-id', 'mobile-application-id', 'chrome-extension-id', 'whois-registrant-email', 'anonymised')
+                    'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash','filename|impfuzzy', 'filename|pehash', 'mac-address', 'mac-eui-64', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'hostname', 'domain', 'email', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'email-body', 'url', 'user-agent', 'AS', 'pattern-in-file', 'pattern-in-traffic', 'filename-pattern', 'stix2-pattern', 'yara', 'sigma', 'mime-type', 'attachment', 'malware-sample', 'link', 'malware-type', 'comment', 'text', 'hex', 'vulnerability', 'cpe', 'weakness', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'other', 'hostname|port', 'email-dst-display-name', 'email-src-display-name', 'email-header', 'email-reply-to', 'email-x-mailer', 'email-mime-boundary', 'email-thread-index', 'email-message-id', 'mobile-application-id', 'chrome-extension-id', 'whois-registrant-email', 'anonymised')
                     ),
             'Artifacts dropped' => array(
                     'desc' => __('Any artifact (files, registry keys etc.) dropped by the malware or other modifications to the system'),
-                    'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'vhash', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy','filename|pehash', 'regkey', 'regkey|value', 'pattern-in-file', 'pattern-in-memory','pdb', 'stix2-pattern', 'yara', 'sigma', 'attachment', 'malware-sample', 'named pipe', 'mutex', 'windows-scheduled-task', 'windows-service-name', 'windows-service-displayname', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'cookie', 'gene', 'kusto-query', 'mime-type', 'anonymised')
+                    'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy','filename|pehash', 'regkey', 'regkey|value', 'pattern-in-file', 'pattern-in-memory', 'filename-pattern', 'pdb', 'stix2-pattern', 'yara', 'sigma', 'attachment', 'malware-sample', 'named pipe', 'mutex', 'windows-scheduled-task', 'windows-service-name', 'windows-service-displayname', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'cookie', 'gene', 'kusto-query', 'mime-type', 'anonymised', 'pgp-public-key', 'pgp-private-key')
                     ),
             'Payload installation' => array(
                     'desc' => __('Info on where the malware gets installed in the system'),
                     'formdesc' => __('Location where the payload was placed in the system and the way it was installed. For example, a filename|md5 type attribute can be added here like this: c:\\windows\\system32\\malicious.exe|41d8cd98f00b204e9800998ecf8427e.'),
-                    'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy', 'filename|pehash', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'stix2-pattern', 'yara', 'sigma', 'vulnerability', 'weakness', 'attachment', 'malware-sample', 'malware-type', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'mobile-application-id', 'chrome-extension-id', 'other', 'mime-type', 'anonymised')
+                    'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy', 'filename|pehash', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'filename-pattern', 'stix2-pattern', 'yara', 'sigma', 'vulnerability', 'cpe','weakness', 'attachment', 'malware-sample', 'malware-type', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'mobile-application-id', 'chrome-extension-id', 'other', 'mime-type', 'anonymised')
                     ),
             'Persistence mechanism' => array(
                     'desc' => __('Mechanisms used by the malware to start at boot'),
@@ -114,7 +116,7 @@ class Attribute extends AppModel
                     ),
             'Network activity' => array(
                     'desc' => __('Information about network traffic generated by the malware'),
-                    'types' => array('ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'port', 'hostname', 'domain', 'domain|ip', 'mac-address', 'mac-eui-64', 'email-dst', 'email-src', 'eppn', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'pattern-in-file', 'stix2-pattern', 'pattern-in-traffic', 'attachment', 'comment', 'text', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'other', 'hex', 'cookie', 'hostname|port', 'bro', 'zeek', 'anonymised', 'community-id', 'email-subject')
+                    'types' => array('ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'port', 'hostname', 'domain', 'domain|ip', 'mac-address', 'mac-eui-64', 'email', 'email-dst', 'email-src', 'eppn', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'pattern-in-file', 'filename-pattern','stix2-pattern', 'pattern-in-traffic', 'attachment', 'comment', 'text', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'other', 'hex', 'cookie', 'hostname|port', 'bro', 'zeek', 'anonymised', 'community-id', 'email-subject')
                     ),
             'Payload type' => array(
                     'desc' => __('Information about the final payload(s)'),
@@ -123,12 +125,12 @@ class Attribute extends AppModel
                     ),
             'Attribution' => array(
                     'desc' => __('Identification of the group, organisation, or country behind the attack'),
-                    'types' => array('threat-actor', 'campaign-name', 'campaign-id', 'whois-registrant-phone', 'whois-registrant-email', 'whois-registrant-name', 'whois-registrant-org', 'whois-registrar', 'whois-creation-date','comment', 'text', 'x509-fingerprint-sha1','x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'dns-soa-email', 'anonymised')
+                    'types' => array('threat-actor', 'campaign-name', 'campaign-id', 'whois-registrant-phone', 'whois-registrant-email', 'whois-registrant-name', 'whois-registrant-org', 'whois-registrar', 'whois-creation-date','comment', 'text', 'x509-fingerprint-sha1','x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'dns-soa-email', 'anonymised', 'email')
                     ),
             'External analysis' => array(
                     'desc' => __('Any other result from additional analysis of the malware like tools output'),
                     'formdesc' => __('Any other result from additional analysis of the malware like tools output Examples: pdf-parser output, automated sandbox analysis, reverse engineering report.'),
-                    'types' => array('md5', 'sha1', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'mac-address', 'mac-eui-64', 'hostname', 'domain', 'domain|ip', 'url', 'user-agent', 'regkey', 'regkey|value', 'AS', 'snort', 'bro', 'zeek', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'vulnerability', 'weakness', 'attachment', 'malware-sample', 'link', 'comment', 'text', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'github-repository', 'other', 'cortex', 'anonymised', 'community-id')
+                    'types' => array('md5', 'sha1', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'mac-address', 'mac-eui-64', 'hostname', 'domain', 'domain|ip', 'url', 'user-agent', 'regkey', 'regkey|value', 'AS', 'snort', 'bro', 'zeek', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'filename-pattern','vulnerability', 'cpe', 'weakness', 'attachment', 'malware-sample', 'link', 'comment', 'text', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'github-repository', 'other', 'cortex', 'anonymised', 'community-id')
                     ),
             'Financial fraud' => array(
                     'desc' => __('Financial Fraud indicators'),
@@ -142,15 +144,15 @@ class Attribute extends AppModel
             'Social network' => array(
                     'desc' => __('Social networks and platforms'),
                     // email-src and email-dst or should we go with a new email type that is neither / both?
-                    'types' => array('github-username', 'github-repository', 'github-organisation', 'jabber-id', 'twitter-id', 'email-src', 'email-dst', 'eppn','comment', 'text', 'other', 'whois-registrant-email', 'anonymised')
+                    'types' => array('github-username', 'github-repository', 'github-organisation', 'jabber-id', 'twitter-id', 'email', 'email-src', 'email-dst', 'eppn','comment', 'text', 'other', 'whois-registrant-email', 'anonymised', 'pgp-public-key', 'pgp-private-key')
             ),
             'Person' => array(
                     'desc' => __('A human being - natural person'),
-                    'types' => array('first-name', 'middle-name', 'last-name', 'date-of-birth', 'place-of-birth', 'gender', 'passport-number', 'passport-country', 'passport-expiration', 'redress-number', 'nationality', 'visa-number', 'issue-date-of-the-visa', 'primary-residence', 'country-of-residence', 'special-service-request', 'frequent-flyer-number', 'travel-details', 'payment-details', 'place-port-of-original-embarkation', 'place-port-of-clearance', 'place-port-of-onward-foreign-destination', 'passenger-name-record-locator-number', 'comment', 'text', 'other', 'phone-number', 'identity-card-number', 'anonymised')
+                    'types' => array('first-name', 'middle-name', 'last-name', 'date-of-birth', 'place-of-birth', 'gender', 'passport-number', 'passport-country', 'passport-expiration', 'redress-number', 'nationality', 'visa-number', 'issue-date-of-the-visa', 'primary-residence', 'country-of-residence', 'special-service-request', 'frequent-flyer-number', 'travel-details', 'payment-details', 'place-port-of-original-embarkation', 'place-port-of-clearance', 'place-port-of-onward-foreign-destination', 'passenger-name-record-locator-number', 'comment', 'text', 'other', 'phone-number', 'identity-card-number', 'anonymised', 'email', 'pgp-public-key', 'pgp-private-key')
             ),
             'Other' => array(
                     'desc' => __('Attributes that are not part of any other category or are meant to be used as a component in MISP objects in the future'),
-                    'types' => array('comment', 'text', 'other', 'size-in-bytes', 'counter', 'datetime', 'cpe', 'port', 'float', 'hex', 'phone-number', 'boolean', 'anonymised')
+                    'types' => array('comment', 'text', 'other', 'size-in-bytes', 'counter', 'datetime', 'cpe', 'port', 'float', 'hex', 'phone-number', 'boolean', 'anonymised', 'pgp-public-key', 'pgp-private-key')
                     )
         );
 
@@ -171,9 +173,10 @@ class Attribute extends AppModel
             'hostname' => array('desc' => __('A full host/dnsname of an attacker'), 'formdesc' => __("A full host/dnsname of an attacker. Also set the IDS flag on when this hostname is hardcoded in malware"), 'default_category' => 'Network activity', 'to_ids' => 1),
             'domain' => array('desc' => __('A domain name used in the malware'), 'formdesc' => __("A domain name used in the malware. Use this instead of hostname when the upper domain is important or can be used to create links between events."), 'default_category' => 'Network activity', 'to_ids' => 1),
             'domain|ip' => array('desc' => __('A domain name and its IP address (as found in DNS lookup) separated by a |'),'formdesc' => __("A domain name and its IP address (as found in DNS lookup) separated by a | (no spaces)"), 'default_category' => 'Network activity', 'to_ids' => 1),
-            'email-src' => array('desc' => __("The email address used to send the malware."), 'default_category' => 'Payload delivery', 'to_ids' => 1),
+            'email' => array('desc' => ('An e-mail address'), 'default_category' => 'Social network', 'to_ids' => 1),
+            'email-src' => array('desc' => __("The source email address. Used to describe the sender when describing an e-mail."), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'eppn' => array('desc' => __("eduPersonPrincipalName - eppn - the NetId of the person for the purposes of inter-institutional authentication. Should be stored in the form of user@univ.edu, where univ.edu is the name of the local security domain."), 'default_category' => 'Network activity', 'to_ids' => 1),
-            'email-dst' => array('desc' => __("A recipient email address"), 'formdesc' => __("A recipient email address that is not related to your constituency."), 'default_category' => 'Network activity', 'to_ids' => 1),
+            'email-dst' => array('desc' => __("The destination email address. Used to describe the recipient when describing an e-mail."), 'default_category' => 'Network activity', 'to_ids' => 1),
             'email-subject' => array('desc' => __("The subject of the email"), 'default_category' => 'Payload delivery', 'to_ids' => 0),
             'email-attachment' => array('desc' => __("File name of the email attachment."), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'email-body' => array('desc' => __('Email body'), 'default_category' => 'Payload delivery', 'to_ids' => 0),
@@ -195,6 +198,9 @@ class Attribute extends AppModel
             'pattern-in-file' => array('desc' => __('Pattern in file that identifies the malware'), 'default_category' => 'Payload installation', 'to_ids' => 1),
             'pattern-in-traffic' => array('desc' => __('Pattern in network traffic that identifies the malware'), 'default_category' => 'Network activity', 'to_ids' => 1),
             'pattern-in-memory' => array('desc' => __('Pattern in memory dump that identifies the malware'), 'default_category' => 'Payload installation', 'to_ids' => 1),
+            'pattern-filename' => array('desc' => __('A pattern in the name of a file'), 'default_category' => 'Payload installation', 'to_ids' => 1),
+            'pgp-public-key' => array('desc' => __('A PGP public key'), 'default_category' => 'Person', 'to_ids' => 0),
+            'pgp-private-key' => array('desc' => __('A PGP private key'), 'default_category' => 'Person', 'to_ids' => 0),
             'yara' => array('desc' => __('Yara signature'), 'default_category' => 'Payload installation', 'to_ids' => 1),
             'stix2-pattern' => array('desc' => __('STIX 2 pattern'), 'default_category' => 'Payload installation', 'to_ids' => 1),
             'sigma' => array('desc' => __('Sigma - Generic Signature Format for SIEM Systems'), 'default_category' => 'Payload installation', 'to_ids' => 1),
@@ -204,6 +210,7 @@ class Attribute extends AppModel
             'identity-card-number' => array('desc' => __('Identity card number'), 'default_category' => 'Person', 'to_ids' => 0),
             'cookie' => array('desc' => __('HTTP cookie as often stored on the user web client. This can include authentication cookie or session cookie.'), 'default_category' => 'Network activity', 'to_ids' => 0),
             'vulnerability' => array('desc' => __('A reference to the vulnerability used in the exploit'), 'default_category' => 'External analysis', 'to_ids' => 0),
+            'cpe' => array('desc' => __('Common Platform Enumeration - structured naming scheme for information technology systems, software, and packages.'), 'default_category' => 'External analysis', 'to_ids' => 0),
             'weakness' => array('desc'=> __('A reference to the weakness used in the exploit'), 'default_category' => 'External analysis', 'to_ids' => 0),
             'attachment' => array('desc' => __('Attachment with external information'), 'formdesc' => __("Please upload files using the <em>Upload Attachment</em> button."), 'default_category' => 'External analysis', 'to_ids' => 0),
             'malware-sample' => array('desc' => __('Attachment containing encrypted malware sample'), 'formdesc' => __("Please upload files using the <em>Upload Attachment</em> button."), 'default_category' => 'Payload delivery', 'to_ids' => 1),
@@ -240,6 +247,7 @@ class Attribute extends AppModel
             'vhash' => array('desc' => __('A VirusTotal checksum'), 'formdesc' => __("You are encouraged to use filename|vhash instead. A checksum from VirusTotal, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'ssdeep' => array('desc' => __('A checksum in ssdeep format'), 'formdesc' => __("You are encouraged to use filename|ssdeep instead. A checksum in the SSDeep format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'imphash' => array('desc' => __('Import hash - a hash created based on the imports in the sample.'), 'formdesc' => __("You are encouraged to use filename|imphash instead. A hash created based on the imports in the sample, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
+            'telfhash' => array('desc' => __('telfhash is symbol hash for ELF files, just like imphash is imports hash for PE files.'), 'formdesc' => __("You are encouraged to use a file object with telfash"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'pehash' => array('desc' => __('PEhash - a hash calculated based of certain pieces of a PE executable file'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'impfuzzy' => array('desc' => __('A fuzzy hash of import table of Portable Executable format'), 'formdesc' => __("You are encouraged to use filename|impfuzzy instead. A fuzzy hash created based on the imports in the sample, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'sha224' => array('desc' => __('A checksum in sha-224 format'), 'formdesc' => __("You are encouraged to use filename|sha224 instead. A checksum in sha224 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
@@ -289,7 +297,6 @@ class Attribute extends AppModel
             'size-in-bytes' => array('desc' => __('Size expressed in bytes'), 'default_category' => 'Other', 'to_ids' => 0),
             'counter' => array('desc' => __('An integer counter, generally to be used in objects'), 'default_category' => 'Other', 'to_ids' => 0),
             'datetime' => array('desc' => __('Datetime in the ISO 8601 format'), 'default_category' => 'Other', 'to_ids' => 0),
-            'cpe' => array('desc' => __('Common platform enumeration'), 'default_category' => 'Other', 'to_ids' => 0),
             'port' => array('desc' => __('Port number'), 'default_category' => 'Network activity', 'to_ids' => 0),
             'ip-dst|port' => array('desc' => __('IP destination and port number separated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1),
             'ip-src|port' => array('desc' => __('IP source and port number separated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1),
@@ -375,7 +382,8 @@ class Attribute extends AppModel
 
     public $primaryOnlyCorrelatingTypes = array(
         'ip-src|port',
-        'ip-dst|port'
+        'ip-dst|port',
+        'hostname|port',
     );
 
     public $captureFields = array(
@@ -420,6 +428,7 @@ class Attribute extends AppModel
     public $validFormats = array(
         'attack-sightings' => array('json', 'AttackSightingsExport', 'json'),
         'cache' => array('txt', 'CacheExport', 'cache'),
+        'count' => array('txt', 'CountExport', 'txt'),
         'csv' => array('csv', 'CsvExport', 'csv'),
         'hashes' => array('txt', 'HashesExport', 'txt'),
         'json' => array('json', 'JsonExport', 'json'),
@@ -452,6 +461,7 @@ class Attribute extends AppModel
             'authentihash' => 'Payload delivery',
             'vhash' => 'Payload delivery',
             'imphash' => 'Payload delivery',
+            'telfhash' => 'Payload delivery',
             'impfuzzy'=> 'Payload delivery',
             'pehash' => 'Payload delivery',
             'cdhash' => 'Payload delivery',
@@ -474,6 +484,7 @@ class Attribute extends AppModel
             'hassh-md5' => 'Network activity',
             'hasshserver-md5' => 'Network activity',
             'link' => 'External analysis',
+            'email' => 'Social network',
             'email-src' => 'Payload delivery',
             'email-dst' => 'Payload delivery',
             'text' => 'Other',
@@ -490,7 +501,7 @@ class Attribute extends AppModel
     // whilst filenames and hashes are file related attribute types
     // This helps generate quick filtering for the event view, but we may reuse this and enhance it in the future for other uses (such as the API?)
     public $typeGroupings = array(
-        'file' => array('attachment', 'pattern-in-file', 'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|pehash', 'malware-sample', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'x509-fingerprint-md5'),
+        'file' => array('attachment', 'pattern-in-file', 'filename-pattern', 'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|pehash', 'malware-sample', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'x509-fingerprint-md5'),
         'network' => array('ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port', 'mac-address', 'mac-eui-64', 'hostname', 'hostname|port', 'domain', 'domain|ip', 'email-dst', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'bro', 'zeek',  'pattern-in-traffic', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256','ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'community-id'),
         'financial' => array('btc', 'xmr', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn', 'phone-number')
     );
@@ -542,8 +553,8 @@ class Attribute extends AppModel
         ),
         'uuid' => array(
             'uuid' => array(
-                'rule' => array('custom', '/^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$/'),
-                'message' => 'Please provide a valid UUID'
+                'rule' => 'uuid',
+                'message' => 'Please provide a valid RFC 4122 UUID'
             ),
             'unique' => array(
                 'rule' => 'isUnique',
@@ -842,8 +853,7 @@ class Attribute extends AppModel
             return false;
         }
         App::uses('ComplexTypeTool', 'Tools');
-        $this->complexTypeTool = new ComplexTypeTool();
-        $this->data['Attribute']['value'] = $this->complexTypeTool->refangValue($this->data['Attribute']['value'], $this->data['Attribute']['type']);
+        $this->data['Attribute']['value'] = ComplexTypeTool::refangValue($this->data['Attribute']['value'], $this->data['Attribute']['type']);
 
         if (!empty($this->data['Attribute']['object_id']) && empty($this->data['Attribute']['object_relation'])) {
             $this->validationErrors['type'] = ['Object attribute sent, but no object_relation set.'];
@@ -865,6 +875,8 @@ class Attribute extends AppModel
         // generate UUID if it doesn't exist
         if (empty($this->data['Attribute']['uuid'])) {
             $this->data['Attribute']['uuid'] = CakeText::uuid();
+        } else {
+            $this->data['Attribute']['uuid'] = strtolower($this->data['Attribute']['uuid']);
         }
         // generate timestamp if it doesn't exist
         if (empty($this->data['Attribute']['timestamp'])) {
@@ -911,10 +923,6 @@ class Attribute extends AppModel
             $this->data['Attribute']['sharing_group_id'] = 0;
         }
         // return true, otherwise the object cannot be saved
-
-        if ($this->data['Attribute']['type'] == 'float' && $this->data['Attribute']['value'] == 0) {
-            $this->data['Attribute']['value'] = '0.0';
-        }
         return true;
     }
 
@@ -922,8 +930,7 @@ class Attribute extends AppModel
     {
         $compositeTypes = $this->getCompositeTypes();
         if (in_array($this->data['Attribute']['type'], $compositeTypes)) {
-            $pieces = explode('|', $fields['value']);
-            if (2 != count($pieces)) {
+            if (substr_count($fields['value'], '|') !== 1) {
                 return false;
             }
         }
@@ -1030,6 +1037,7 @@ class Attribute extends AppModel
         'authentihash' => 64,
         'md5' => 32,
         'imphash' => 32,
+        'telfhash' => 70,
         'sha1' => 40,
         'git-commit-id' => 40,
         'x509-fingerprint-md5' => 32,
@@ -1058,6 +1066,7 @@ class Attribute extends AppModel
         switch ($type) {
             case 'md5':
             case 'imphash':
+            case 'telfhash':
             case 'sha1':
             case 'sha224':
             case 'sha256':
@@ -1077,13 +1086,12 @@ class Attribute extends AppModel
             case 'x509-fingerprint-sha256':
             case 'x509-fingerprint-sha1':
             case 'git-commit-id':
-                $length = $this->__hexHashLengths[$type];
-                if (preg_match("#^[0-9a-f]{" . $length . "}$#", $value)) {
-                    $returnValue = true;
+                if ($this->isHashValid($type, $value)) {
+                    return true;
                 } else {
-                    $returnValue = __('Checksum has an invalid length or format (expected: %s hexadecimal characters). Please double check the value or select type "other".', $length);
+                    $length = $this->__hexHashLengths[$type];
+                    return __('Checksum has an invalid length or format (expected: %s hexadecimal characters). Please double check the value or select type "other".', $length);
                 }
-                break;
             case 'tlsh':
                 if (preg_match("#^[0-9a-f]{35,}$#", $value)) {
                     $returnValue = true;
@@ -1092,27 +1100,24 @@ class Attribute extends AppModel
                 }
                 break;
             case 'pehash':
-                if (preg_match("#^[0-9a-f]{40}$#", $value)) {
+                if ($this->isHashValid('pehash', $value)) {
                     $returnValue = true;
                 } else {
                     $returnValue = __('The input doesn\'t match the expected sha1 format (expected: 40 hexadecimal characters). Keep in mind that MISP currently only supports SHA1 for PEhashes, if you would like to get the support extended to other hash types, make sure to create a github ticket about it at https://github.com/MISP/MISP!');
                 }
                 break;
             case 'ssdeep':
-                if (substr_count($value, ':') == 2) {
+                if (substr_count($value, ':') === 2) {
                     $parts = explode(':', $value);
-                    if (is_numeric($parts[0])) {
-                        $returnValue = true;
+                    if ($this->isPositiveInteger($parts[0])) {
+                        return true;
                     }
                 }
-                if (!$returnValue) {
-                    $returnValue = __('Invalid SSDeep hash. The format has to be blocksize:hash:hash');
-                }
-                break;
+                return __('Invalid SSDeep hash. The format has to be blocksize:hash:hash');
             case 'impfuzzy':
-                if (substr_count($value, ':') == 2) {
+                if (substr_count($value, ':') === 2) {
                     $parts = explode(':', $value);
-                    if (is_numeric($parts[0])) {
+                    if ($this->isPositiveInteger($parts[0])) {
                         $returnValue = true;
                     }
                 }
@@ -1131,7 +1136,7 @@ class Attribute extends AppModel
                 if (preg_match("#(OPTIONS|GET|HEAD|POST|PUT|DELETE|TRACE|CONNECT|PROPFIND|PROPPATCH|MKCOL|COPY|MOVE|LOCK|UNLOCK|VERSION-CONTROL|REPORT|CHECKOUT|CHECKIN|UNCHECKOUT|MKWORKSPACE|UPDATE|LABEL|MERGE|BASELINE-CONTROL|MKACTIVITY|ORDERPATCH|ACL|PATCH|SEARCH)#", $value)) {
                     $returnValue = true;
                 } else {
-                    $returnValue = 'Unknown HTTP method.';
+                    $returnValue = __('Unknown HTTP method.');
                 }
                 break;
             case 'filename|pehash':
@@ -1172,7 +1177,7 @@ class Attribute extends AppModel
                     $value = $composite[1];
                     if (substr_count($value, ':') == 2) {
                         $parts = explode(':', $value);
-                        if (is_numeric($parts[0])) {
+                        if ($this->isPositiveInteger($parts[0])) {
                             $returnValue = true;
                         }
                     }
@@ -1189,7 +1194,7 @@ class Attribute extends AppModel
                 }
                 break;
             case 'filename|vhash':
-                if (preg_match("#^.+\|[a-zA-Z0-9]+$#", $value)) {
+                if (preg_match('#^.+\|[a-zA-Z0-9&!="]+$#', $value)) {
                     $returnValue = true;
                 } else {
                     $returnValue = __('Checksum has an invalid length or format (expected: filename|string characters). Please double check the value or select type "other".');
@@ -1199,17 +1204,17 @@ class Attribute extends AppModel
             case 'ip-dst':
                 if (strpos($value, '/') !== false) {
                     $parts = explode("/", $value);
-                    if (count($parts) !== 2 || intval($parts[1]) != $parts[1] || $parts[1] < 0) {
+                    if (count($parts) !== 2 || !$this->isPositiveInteger($parts[1])) {
                         return __('Invalid CIDR notation value found.');
                     }
 
                     if (filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) {
                         if ($parts[1] > 32) {
-                            return __('Invalid CIDR notation value found.');
+                            return __('Invalid CIDR notation value found, for IPv4 must be lower or equal 32.');
                         }
                     } else if (filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) {
                         if ($parts[1] > 128) {
-                            return __('Invalid CIDR notation value found.');
+                            return __('Invalid CIDR notation value found, for IPv6 must be lower or equal 128.');
                         }
                     } else {
                         return __('IP address has an invalid format.');
@@ -1221,7 +1226,7 @@ class Attribute extends AppModel
 
             case 'port':
                 if (!$this->isPortValid($value)) {
-                    $returnValue = __('Port numbers have to be positive integers between 1 and 65535.');
+                    $returnValue = __('Port numbers have to be integers between 1 and 65535.');
                 } else {
                     $returnValue = true;
                 }
@@ -1229,17 +1234,20 @@ class Attribute extends AppModel
             case 'ip-dst|port':
             case 'ip-src|port':
                 $parts = explode('|', $value);
-                if (filter_var($parts[0], FILTER_VALIDATE_IP) && $this->isPortValid($parts[1])) {
-                    $returnValue = true;
+                if (!filter_var($parts[0], FILTER_VALIDATE_IP)) {
+                    return __('IP address has an invalid format.');
                 }
-                break;
+                if (!$this->isPortValid($parts[1])) {
+                    return __('Port numbers have to be integers between 1 and 65535.');
+                }
+                return true;
             case 'mac-address':
-                if (preg_match('/^([a-fA-F0-9]{2}[:|\-| |\.]?){6}$/', $value) == 1) {
+                if (preg_match('/^([a-fA-F0-9]{2}[:]?){6}$/', $value)) {
                     $returnValue = true;
                 }
                 break;
             case 'mac-eui-64':
-                if (preg_match('/^([a-fA-F0-9]{2}[:|\-| |\.]?){8}$/', $value) == 1) {
+                if (preg_match('/^([a-fA-F0-9]{2}[:]?){8}$/', $value)) {
                     $returnValue = true;
                 }
                 break;
@@ -1248,15 +1256,18 @@ class Attribute extends AppModel
                 if ($this->isDomainValid($value)) {
                     $returnValue = true;
                 } else {
-                    $returnValue = ucfirst($type) . __(' name has an invalid format. Please double check the value or select type "other".');
+                    $returnValue = __('%s has an invalid format. Please double check the value or select type "other".', ucfirst($type));
                 }
                 break;
             case 'hostname|port':
                 $parts = explode('|', $value);
-                if ($this->isDomainValid($parts[0]) && $this->isPortValid($parts[1])) {
-                    $returnValue = true;
+                if (!$this->isDomainValid($parts[0])) {
+                    return __('Hostname has an invalid format.');
                 }
-                break;
+                if (!$this->isPortValid($parts[1])) {
+                    return __('Port numbers have to be integers between 1 and 65535.');
+                }
+                return true;
             case 'domain|ip':
                 if (preg_match("#^[A-Z0-9.\-_]+\.[A-Z0-9\-]{2,}\|.*$#i", $value)) {
                     $parts = explode('|', $value);
@@ -1269,6 +1280,7 @@ class Attribute extends AppModel
                     $returnValue = __('Domain name has an invalid format.');
                 }
                 break;
+            case 'email':
             case 'email-src':
             case 'eppn':
             case 'email-dst':
@@ -1284,7 +1296,6 @@ class Attribute extends AppModel
                 }
                 break;
             case 'vulnerability':
-                $value = str_replace('–', '-', $value);
                 if (preg_match("#^(CVE-)[0-9]{4}(-)[0-9]{4,}$#", $value)) {
                     $returnValue = true;
                 } else {
@@ -1292,7 +1303,6 @@ class Attribute extends AppModel
                 }
                 break;
             case 'weakness':
-                $value = str_replace('–', '-', $value);
                 if (preg_match("#^(CWE-)[0-9]{1,}$#", $value)) {
                     $returnValue = true;
                 } else {
@@ -1313,7 +1323,6 @@ class Attribute extends AppModel
                 }
                 break;
             case 'mutex':
-            case 'AS':
             case 'snort':
             case 'bro':
             case 'zeek':
@@ -1322,6 +1331,9 @@ class Attribute extends AppModel
             case 'pattern-in-file':
             case 'pattern-in-traffic':
             case 'pattern-in-memory':
+            case 'filename-pattern':
+            case 'pgp-public-key':
+            case 'pgp-private-key':
             case 'yara':
             case 'stix2-pattern':
             case 'sigma':
@@ -1332,6 +1344,16 @@ class Attribute extends AppModel
             case 'cookie':
             case 'attachment':
             case 'malware-sample':
+            case 'comment':
+            case 'text':
+            case 'other':
+            case 'cpe':
+            case 'email-attachment':
+            case 'email-body':
+            case 'email-header':
+            case 'first-name':
+            case 'middle-name':
+            case 'last-name':
                 $returnValue = true;
                 break;
             case 'link':
@@ -1340,18 +1362,8 @@ class Attribute extends AppModel
                     $returnValue = true;
                 }
                 break;
-            case 'comment':
-            case 'text':
-            case 'other':
-            case 'email-attachment':
-            case 'email-body':
-                $returnValue = true;
-                break;
             case 'hex':
-                if (preg_match("/^[0-9a-f]*$/i", $value)) {
-                    $returnValue = true;
-                }
-                break;
+                return ctype_xdigit($value);
             case 'target-user':
             case 'campaign-name':
             case 'campaign-id':
@@ -1371,13 +1383,10 @@ class Attribute extends AppModel
             case 'filename':
             case 'pdb':
             case 'windows-scheduled-task':
-      case 'whois-registrant-name':
-      case 'whois-registrant-org':
+            case 'whois-registrant-name':
+            case 'whois-registrant-org':
             case 'whois-registrar':
             case 'whois-creation-date':
-            case 'first-name':
-            case 'middle-name':
-            case 'last-name':
             case 'date-of-birth':
             case 'place-of-birth':
             case 'gender':
@@ -1408,18 +1417,13 @@ class Attribute extends AppModel
             case 'github-username':
             case 'github-repository':
             case 'github-organisation':
-            case 'cpe':
             case 'twitter-id':
             case 'chrome-extension-id':
             case 'mobile-application-id':
-                // no newline
-                if (!preg_match("#\n#", $value)) {
-                    $returnValue = true;
+                if (strpos($value, "\n") !== false) {
+                    return __('Value must not contain new line character.');
                 }
-                break;
-            case 'email-header':
-                $returnValue = true;
-                break;
+                return true;
             case 'datetime':
                 try {
                     new DateTime($value);
@@ -1430,12 +1434,10 @@ class Attribute extends AppModel
                 break;
             case 'size-in-bytes':
             case 'counter':
-                if (!is_numeric($value) || $value < 0) {
-                    $returnValue = __('The value has to be a number greater or equal 0.');
-                } else {
-                    $returnValue = true;
+                if ($this->isPositiveInteger($value)) {
+                    return true;
                 }
-                break;
+                return __('The value has to be a whole number greater or equal 0.');
             case 'targeted-threat-index':
                 if (!is_numeric($value) || $value < 0 || $value > 10) {
                     $returnValue = __('The value has to be a number between 0 and 10.');
@@ -1448,11 +1450,15 @@ class Attribute extends AppModel
             case 'btc':
             case 'dash':
             case 'xmr':
-            case 'vhash':
                 if (preg_match('/^[a-zA-Z0-9]+$/', $value)) {
                     $returnValue = true;
                 }
                 break;
+            case 'vhash':
+                if (preg_match('/^[a-zA-Z0-9&!="]+$/', $value)) {
+                    $returnValue = true;
+                }
+                break;
             case 'bin':
             case 'cc-number':
             case 'bank-account-nr':
@@ -1469,15 +1475,17 @@ class Attribute extends AppModel
                 $returnValue = (json_last_error() == JSON_ERROR_NONE);
                 break;
             case 'float':
-                $value = floatval($value);
-                if (is_float($value)) {
-                    $returnValue = true;
-                }
-                break;
+                return is_numeric($value);
             case 'boolean':
                 if ($value == 1 || $value == 0) {
                     $returnValue = true;
                 }
+                break;
+            case 'AS':
+                if ($this->isPositiveInteger($value) && $value <= 4294967295) {
+                    return true;
+                }
+                return __('AS number have to be integers between 1 and 4294967295');
         }
         return $returnValue;
     }
@@ -1507,9 +1515,11 @@ class Attribute extends AppModel
             case 'authentihash':
             case 'vhash':
             case 'imphash':
+            case 'telfhash':
             case 'tlsh':
             case 'anonymised':
             case 'cdhash':
+            case 'email':
             case 'email-src':
             case 'email-dst':
             case 'target-email':
@@ -1565,8 +1575,12 @@ class Attribute extends AppModel
                 $value = $pieces[0] . '|' . strtolower($pieces[1]);
                 break;
             case 'http-method':
-                $value = strtoupper($value);
-                break;
+            case 'hex':
+                return strtoupper($value);
+            case 'vulnerability':
+            case 'weakness':
+                $value = str_replace('–', '-', $value);
+                return strtoupper($value);
             case 'cc-number':
             case 'bin':
                 $value = preg_replace('/[^0-9]+/', '', $value);
@@ -1585,12 +1599,8 @@ class Attribute extends AppModel
                 $value = preg_replace('/\(0\)/', '', $value);
                 $value = preg_replace('/[^\+0-9]+/', '', $value);
                 break;
-            case 'url':
-                $value = preg_replace('/^hxxp/i', 'http', $value);
-                $value = preg_replace('/\[\.\]/', '.', $value);
-                break;
-      case 'x509-fingerprint-md5':
-      case 'x509-fingerprint-sha256':
+            case 'x509-fingerprint-md5':
+            case 'x509-fingerprint-sha256':
             case 'x509-fingerprint-sha1':
                 $value = str_replace(':', '', $value);
                 $value = strtolower($value);
@@ -1635,19 +1645,14 @@ class Attribute extends AppModel
                         $parts[0] = inet_ntop(inet_pton($parts[0]));
                     }
                     return $parts[0] . '|' . $parts[1];
-                break;
             case 'mac-address':
             case 'mac-eui-64':
-                $value = str_replace(array('.', ':', '-', ' '), '', $value);
+                $value = str_replace(array('.', ':', '-', ' '), '', strtolower($value));
                 $value = wordwrap($value, 2, ':', true);
                 break;
             case 'hostname|port':
                 $value = strtolower($value);
-                str_replace(':', '|', $value);
-                break;
-            case 'hex':
-                $value = strtoupper($value);
-                break;
+                return str_replace(':', '|', $value);
             case 'boolean':
                 if ('true' == trim(strtolower($value))) {
                     $value = 1;
@@ -1664,6 +1669,17 @@ class Attribute extends AppModel
                     // silently skip. Rejection will be done in runValidation()
                 }
                 break;
+            case 'AS':
+                if (strtoupper(substr($value, 0, 2)) === 'AS') {
+                    $value = substr($value, 2); // remove 'AS'
+                }
+                if (strpos($value, '.') !== false) { // maybe value is in asdot notation
+                    $parts = explode('.', $value);
+                    if ($this->isPositiveInteger($parts[0]) && $this->isPositiveInteger($parts[1])) {
+                        return $parts[0] * 65536 + $parts[1];
+                    }
+                }
+                break;
         }
         return $value;
     }
@@ -2443,7 +2459,7 @@ class Attribute extends AppModel
         $rules = array();
         foreach ($eventIds as $event) {
             $conditions['AND'] = array('Attribute.to_ids' => 1, "Event.published" => 1, 'Attribute.event_id' => $event['Event']['id']);
-            $valid_types = array('ip-dst', 'ip-src', 'ip-dst|port', 'ip-src|port', 'eppn', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'domain', 'domain|ip', 'hostname', 'url', 'user-agent', 'snort');
+            $valid_types = array('ip-dst', 'ip-src', 'ip-dst|port', 'ip-src|port', 'eppn', 'email', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'domain', 'domain|ip', 'hostname', 'url', 'user-agent', 'snort');
             $conditions['AND']['Attribute.type'] = $valid_types;
             if (!empty($type)) {
                 $conditions['AND'][] = array('Attribute.type' => $type);
@@ -2770,8 +2786,8 @@ class Attribute extends AppModel
                 }
                 $conditions['AND'][] = $temp;
             }
-            $this->Whitelist = ClassRegistry::init('Whitelist');
-            $this->whitelist = $this->Whitelist->getBlockedValues();
+            $this->Allowedlist = ClassRegistry::init('Allowedlist');
+            $this->allowedlist = $this->Allowedlist->getBlockedValues();
             $instanceString = 'MISP';
             if (Configure::read('MISP.host_org_id') && Configure::read('MISP.host_org_id') > 0) {
                 $this->Event->Orgc->id = Configure::read('MISP.host_org_id');
@@ -2782,7 +2798,7 @@ class Attribute extends AppModel
             $mispTypes = $export->getMispTypes($type);
             foreach ($mispTypes as $mispType) {
                 $conditions['AND']['Attribute.type'] = $mispType[0];
-                $intel = array_merge($intel, $this->__bro($user, $conditions, $mispType[1], $export, $this->whitelist, $instanceString, $enforceWarninglist));
+                $intel = array_merge($intel, $this->__bro($user, $conditions, $mispType[1], $export, $this->allowedlist, $instanceString, $enforceWarninglist));
             }
         }
         natsort($intel);
@@ -2793,7 +2809,7 @@ class Attribute extends AppModel
         return $intel;
     }
 
-    private function __bro($user, $conditions, $valueField, $export, $whitelist, $instanceString, $enforceWarninglist)
+    private function __bro($user, $conditions, $valueField, $export, $allowedlist, $instanceString, $enforceWarninglist)
     {
         $attributes = $this->fetchAttributes(
             $user,
@@ -2811,7 +2827,7 @@ class Attribute extends AppModel
         $orgs = $this->Event->Orgc->find('list', array(
                 'fields' => array('Orgc.id', 'Orgc.name')
         ));
-        return $export->export($attributes, $orgs, $valueField, $whitelist, $instanceString);
+        return $export->export($attributes, $orgs, $valueField, $allowedlist, $instanceString);
     }
 
     public function generateCorrelation($jobId = false, $startPercentage = 0, $eventId = false, $attributeId = false)
@@ -3122,7 +3138,6 @@ class Attribute extends AppModel
                 'value' => $value,
         );
         if ($element['complex']) {
-            App::uses('ComplexTypeTool', 'Tools');
             $complexTypeTool = new ComplexTypeTool();
             $result = $complexTypeTool->checkComplexRouter($value, ucfirst($element['type']));
             if (isset($result['multi'])) {
@@ -3359,8 +3374,8 @@ class Attribute extends AppModel
             $params['limit'] = $options['limit'];
         }
         if (
-            Configure::read('MISP.proposals_block_attributes') &&
-            !empty($options['allow_proposal_blocking'])
+            !empty($options['allow_proposal_blocking']) &&
+            Configure::read('MISP.proposals_block_attributes')
         ) {
             $this->bindModel(array('hasMany' => array('ShadowAttribute' => array('foreignKey' => 'old_id'))));
             $proposalRestriction =  array(
@@ -3454,9 +3469,8 @@ class Attribute extends AppModel
             return $results;
         }
 
-        if (($options['enforceWarninglist'] || $options['includeWarninglistHits']) && !isset($this->warninglists)) {
+        if (($options['enforceWarninglist'] || $options['includeWarninglistHits']) && !isset($this->Warninglist)) {
             $this->Warninglist = ClassRegistry::init('Warninglist');
-            $this->warninglists = $this->Warninglist->fetchForEventView();
         }
         if (empty($params['limit'])) {
             $loopLimit = 50000;
@@ -3526,14 +3540,15 @@ class Attribute extends AppModel
             $results = array_values($results);
             $proposals_block_attributes = Configure::read('MISP.proposals_block_attributes');
             foreach ($results as $key => $attribute) {
+                if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttribute($attribute['Attribute'])) {
+                    unset($results[$key]); // Remove attribute that match any enabled warninglists
+                    continue;
+                }
                 if (!empty($options['includeEventTags'])) {
                     $results = $this->__attachEventTagsToAttributes($eventTags, $results, $key, $options);
                 }
-                if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttributes($this->warninglists, $attribute['Attribute'])) {
-                    continue;
-                }
                 if ($options['includeWarninglistHits']) {
-                    $results[$key]['Attribute'] = $this->Warninglist->simpleCheckForWarning($results[$key]['Attribute'], $this->warninglists, true);
+                    $results[$key]['Attribute'] = $this->Warninglist->checkForWarning($results[$key]['Attribute']);
                 }
                 if (!empty($options['includeAttributeUuid']) || !empty($options['includeEventUuid'])) {
                     $results[$key]['Attribute']['event_uuid'] = $results[$key]['Event']['uuid'];
@@ -3687,7 +3702,12 @@ class Attribute extends AppModel
         return $validTypes;
     }
 
-    public function validateAttribute($attribute, $context = true)
+    /**
+     * @param $attribute
+     * @param bool $context
+     * @return array|true
+     */
+    public function validateAttribute(array $attribute, $context = true)
     {
         $this->set($attribute);
         if (!$context) {
@@ -4106,10 +4126,7 @@ class Attribute extends AppModel
         }
         if (!empty($attribute['enforceWarninglist']) || !empty($params['enforceWarninglist'])) {
             $this->Warninglist = ClassRegistry::init('Warninglist');
-            if (empty($this->warninglists)) {
-                $this->warninglists = $this->Warninglist->fetchForEventView();
-            }
-            if (!$this->Warninglist->filterWarninglistAttributes($warninglists, $attributes[$k])) {
+            if (!$this->Warninglist->filterWarninglistAttribute($attribute)) {
                 $this->validationErrors['warninglist'] = 'Attribute could not be saved as it trips over a warninglist and enforceWarninglist is enforced.';
                 $validationErrors = $this->validationErrors['warninglist'];
                 $log->create();
@@ -4120,7 +4137,7 @@ class Attribute extends AppModel
                         'email' => $user['email'],
                         'action' => 'add',
                         'user_id' => $user['id'],
-                        'title' => 'Attribute dropped due to validation for Event ' . $eventId . ' failed: ' . $attribute_short,
+                        'title' => 'Attribute dropped due to validation for Event ' . $eventId . ' failed',
                         'change' => 'Validation errors: ' . json_encode($this->validationErrors) . ' Full Attribute: ' . json_encode($attribute),
                 ));
                 return $attribute;
@@ -4453,7 +4470,7 @@ class Attribute extends AppModel
                 'Attribute' => array(
                     'value' => array('function' => 'set_filter_value'),
                     'category' => array('function' => 'set_filter_simple_attribute'),
-                    'type' => array('function' => 'set_filter_simple_attribute'),
+                    'type' => array('function' => 'set_filter_type'),
                     'object_relation' => array('function' => 'set_filter_simple_attribute'),
                     'tags' => array('function' => 'set_filter_tags', 'pop' => true),
                     'uuid' => array('function' => 'set_filter_uuid'),
@@ -4634,7 +4651,7 @@ class Attribute extends AppModel
 
     private function __iteratedFetch($user, &$params, &$loop, TmpFileTool $tmpfile, $exportTool, $exportToolParams, &$elementCounter = 0)
     {
-        $this->Whitelist = ClassRegistry::init('Whitelist');
+        $this->Allowedlist = ClassRegistry::init('Allowedlist');
         $continue = true;
         while ($continue) {
             $results = $this->fetchAttributes($user, $params, $continue);
@@ -4643,7 +4660,7 @@ class Attribute extends AppModel
                 $results = $this->Sightingdb->attachToAttributes($results, $user);
             }
             $params['page'] += 1;
-            $results = $this->Whitelist->removeWhitelistedFromArray($results, true);
+            $results = $this->Allowedlist->removeAllowedlistedFromArray($results, true);
             $i = 0;
             $temp = '';
             foreach ($results as $attribute) {
@@ -4731,6 +4748,30 @@ class Attribute extends AppModel
      */
     private function isPortValid($value)
     {
-        return is_numeric($value) && $value >= 1 && $value <= 65535;
+        return $this->isPositiveInteger($value) && $value >= 1 && $value <= 65535;
+    }
+
+    /**
+     * @param string $type
+     * @param string $value
+     * @return bool
+     */
+    private function isHashValid($type, $value)
+    {
+        if (!isset($this->__hexHashLengths[$type])) {
+            throw new InvalidArgumentException("Invalid hash type '$type'.");
+        }
+        $length = $this->__hexHashLengths[$type];
+        return strlen($value) === $length && ctype_xdigit($value);
+    }
+
+    /**
+     * Returns true if input value is positive integer or zero.
+     * @param int|string $value
+     * @return bool
+     */
+    private function isPositiveInteger($value)
+    {
+        return (is_int($value) && $value >= 0) || ctype_digit($value);
     }
 }

app/Model/AttributeTag.php

@@ -2,7 +2,7 @@
 App::uses('AppModel', 'Model');
 
 /**
- * @property Attribute $Attribute
+ * @property Tag $Tag
  */
 class AttributeTag extends AppModel
 {
@@ -180,30 +180,11 @@ class AttributeTag extends AppModel
         }
     }
 
-    /**
-     * Count number of not deleted attributes that contains given tag for given user. Tag must contains 'AttributeTag'.
-     *
-     * @param array $tag
-     * @param array $user
-     * @return int
-     */
-    public function countForTag(array $tag, array $user)
+    public function countForTag($tag_id, $user)
     {
-        $attributeIds = [];
-        foreach ($tag['AttributeTag'] as $attributeTag) {
-            $attributeIds[] = $attributeTag['attribute_id'];
-        }
-
-        if (empty($attributeIds)) {
-            return 0;
-        }
-
-        $conditions = $this->Attribute->buildConditions($user);
-        $conditions['Attribute.id'] = $attributeIds;
-        $conditions['Attribute.deleted'] = 0;
-        return $this->Attribute->find('count', array(
-            'recursive' => 0,
-            'conditions' => $conditions,
+        return $this->find('count', array(
+            'recursive' => -1,
+            'conditions' => array('AttributeTag.tag_id' => $tag_id)
         ));
     }
 

app/Model/Behavior/RegexpBehavior.php

@@ -12,7 +12,7 @@ class RegexpBehavior extends ModelBehavior
     public $excluded_types = array('sigma', 'float');
 
     /**
-     * replace the current value according to the regexp rules, or block blacklisted regular expressions
+     * replace the current value according to the regexp rules, or block blocklisted regular expressions
      *
      * @param Model $Model
      * @param string $type

app/Model/Bruteforce.php

@@ -20,8 +20,8 @@ class Bruteforce extends AppModel
         );
         $this->save($bruteforceEntry);
         $title = 'Failed login attempt using username ' . $username . ' from IP: ' . $_SERVER['REMOTE_ADDR'] . '.';
-        if ($this->isBlacklisted($ip, $username)) {
-            $title .= 'This has tripped the bruteforce protection after  ' . $amount . ' failed attempts. The user is now blacklisted for ' . $expire . ' seconds.';
+        if ($this->isBlocklisted($ip, $username)) {
+            $title .= 'This has tripped the bruteforce protection after  ' . $amount . ' failed attempts. The user is now blocklisted for ' . $expire . ' seconds.';
         }
         $log = array(
                 'org' => 'SYSTEM',
@@ -47,7 +47,7 @@ class Bruteforce extends AppModel
         $this->query($sql);
     }
 
-    public function isBlacklisted($ip, $username)
+    public function isBlocklisted($ip, $username)
     {
         // first remove old expired rows
         $this->clean();

app/Model/Dashboard.php

@@ -14,8 +14,8 @@ class Dashboard extends AppModel
         'role_id' => 'numeric',
         'uuid' => array(
             'uuid' => array(
-                'rule' => array('custom', '/^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$/'),
-                'message' => 'Please provide a valid UUID'
+                'rule' => 'uuid',
+                'message' => 'Please provide a valid RFC 4122 UUID'
             ),
         )
     );

app/Model/DecayingModelsFormulas/Base.php

@@ -106,7 +106,7 @@ abstract class DecayingModelBase
         $flag_contain_matching_taxonomy = false;
         if (!empty($taxonomy_effective_ratios)) {
             foreach ($tags as $k => $tag) {
-                $taxonomy = explode('=', $tag['Tag']['name'])[0];
+                $taxonomy = $this->__extractTagBasename($tag['Tag']['name'])['base'];
                 if (isset($taxonomy_effective_ratios[$taxonomy])) {
                     $flag_contain_matching_taxonomy = true;
                     $base_score += $taxonomy_effective_ratios[$taxonomy] * $tag['Tag']['numerical_value'];

app/Model/Dns.php

@@ -1,41 +0,0 @@
-<?php
-
-App::uses('AppModel', 'Model');
-
-/*
- * Domain Name System related
- */
-class Dns extends AppModel
-{
-    public $useTable = false;
-
-    // Checks for a valid internet name
-    // Returns true if the name is an existing domain name, false otherwise
-    public function testipaddress($nametotest)
-    {
-        if (intval($nametotest) > 0) {
-            return true;
-        } else {
-            $ipaddress = gethostbyname($nametotest);
-            if ($ipaddress == $nametotest) {
-                return false;
-            } else {
-                return true;
-            }
-        }
-    }
-
-    // Name to IP list
-    // get all IP addresses of a certain domain name via DNS.
-    public function nametoipl($name = '')
-    {
-        if ('true' == Configure::read('MISP.dns')) {
-            if (!$ips = gethostbynamel($name)) {
-                $ips = array();
-            }
-        } else {
-            $ips = array();
-        }
-        return $ips;
-    }
-}

app/Model/EventBlacklist.php

@@ -1,50 +0,0 @@
-<?php
-App::uses('AppModel', 'Model');
-class EventBlacklist extends AppModel
-{
-    public $useTable = 'event_blacklists';
-
-    public $recursive = -1;
-
-    public $actsAs = array(
-            'SysLogLogable.SysLogLogable' => array( // TODO Audit, logable
-                    'userModel' => 'User',
-                    'userKey' => 'user_id',
-                    'change' => 'full'),
-            'Containable',
-    );
-
-    public $blacklistFields = array('event_uuid', 'comment', 'event_info', 'event_orgc');
-
-    public $blacklistTarget = 'event';
-
-    public $validate = array(
-            'event_uuid' => array(
-                    'unique' => array(
-                            'rule' => 'isUnique',
-                            'message' => 'Event already blacklisted.'
-                    ),
-                    'uuid' => array(
-                            'rule' => array('uuid'),
-                            'message' => 'Please provide a valid UUID'
-                    ),
-            )
-    );
-
-    public function beforeValidate($options = array())
-    {
-        parent::beforeValidate();
-        $schema = $this->schema();
-        if (!isset($schema['event_info'])) {
-            $this->updateDatabase('addEventBlacklistsContext');
-        }
-        $date = date('Y-m-d H:i:s');
-        if (empty($this->data['EventBlacklist']['id'])) {
-            $this->data['EventBlacklist']['date_created'] = $date;
-        }
-        if (empty($this->data['EventBlacklist']['comment'])) {
-            $this->data['EventBlacklist']['comment'] = '';
-        }
-        return true;
-    }
-}

app/Model/EventBlocklist.php

@@ -0,0 +1,63 @@
+<?php
+App::uses('AppModel', 'Model');
+
+class EventBlocklist extends AppModel
+{
+    public $useTable = 'event_blocklists';
+
+    public $recursive = -1;
+
+    public $actsAs = array(
+            'SysLogLogable.SysLogLogable' => array( // TODO Audit, logable
+                    'userModel' => 'User',
+                    'userKey' => 'user_id',
+                    'change' => 'full'),
+            'Containable',
+    );
+
+    public $blocklistFields = array('event_uuid', 'comment', 'event_info', 'event_orgc');
+
+    public $blocklistTarget = 'event';
+
+    public $validate = array(
+            'event_uuid' => array(
+                    'unique' => array(
+                            'rule' => 'isUnique',
+                            'message' => 'Event already blocklisted.'
+                    ),
+                    'uuid' => array(
+                            'rule' => 'uuid',
+                            'message' => 'Please provide a valid RFC 4122 UUID'
+                    ),
+            )
+    );
+
+    public function beforeValidate($options = array())
+    {
+        parent::beforeValidate();
+        $schema = $this->schema();
+        if (!isset($schema['event_info'])) {
+            $this->updateDatabase('addEventBlocklistsContext');
+        }
+        if (empty($this->data['EventBlocklist']['id'])) {
+            $this->data['EventBlocklist']['date_created'] = date('Y-m-d H:i:s');
+        }
+        if (empty($this->data['EventBlocklist']['comment'])) {
+            $this->data['EventBlocklist']['comment'] = '';
+        }
+        return true;
+    }
+
+    /**
+     * @param string $eventUuid
+     * @return bool
+     */
+    public function isBlocked($eventUuid)
+    {
+        $result = $this->find('first', [
+            'conditions' => ['event_uuid' => $eventUuid],
+            'fields' => ['id']
+        ]);
+        return !empty($result);
+    }
+}

app/Model/EventReport.php

@@ -0,0 +1,522 @@
+<?php
+App::uses('AppModel', 'Model');
+
+/**
+ * @property Event $Event
+ */
+class EventReport extends AppModel
+{
+    public $actsAs = array(
+        'Containable',
+        'SysLogLogable.SysLogLogable' => array(
+            'userModel' => 'User',
+            'userKey' => 'user_id',
+            'change' => 'full'
+        ),
+    );
+
+    public $validate = array(
+        'event_id' => array(
+            'numeric' => array(
+                'rule' => array('numeric')
+            )
+        ),
+        'uuid' => array(
+            'uuid' => array(
+                'rule' => 'uuid',
+                'message' => 'Please provide a valid RFC 4122 UUID'
+            ),
+            'unique' => array(
+                'rule' => 'isUnique',
+                'message' => 'The UUID provided is not unique',
+                'required' => 'create'
+            )
+        ),
+        'distribution' => array(
+            'rule' => array('inList', array('0', '1', '2', '3', '4', '5')),
+            'message' => 'Options: Your organisation only, This community only, Connected communities, All communities, Sharing group, Inherit event',
+            'required' => true
+        )
+    );
+
+    public $captureFields = array('uuid', 'name', 'content', 'distribution', 'sharing_group_id', 'timestamp', 'deleted', 'event_id');
+    public $defaultContain = array(
+        'SharingGroup' => array('fields' => array('id', 'name', 'uuid')),
+        'Event' => array(
+            'fields' =>  array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.info', 'Event.user_id', 'Event.date'),
+            'Orgc' => array('fields' => array('Orgc.id', 'Orgc.name')),
+            'Org' => array('fields' => array('Org.id', 'Org.name'))
+        )
+    );
+
+    public $belongsTo = array(
+        'Event' => array(
+            'className' => 'Event',
+            'foreignKey' => 'event_id'
+        ),
+        'SharingGroup' => array(
+            'className' => 'SharingGroup',
+            'foreignKey' => 'sharing_group_id'
+        ),
+    );
+
+    public function beforeValidate($options = array())
+    {
+        parent::beforeValidate();
+        // generate UUID if it doesn't exist
+        if (empty($this->data['EventReport']['uuid'])) {
+            $this->data['EventReport']['uuid'] = CakeText::uuid();
+        } else {
+            $this->data['EventReport']['uuid'] = strtolower($this->data['EventReport']['uuid']);
+        }
+        // generate timestamp if it doesn't exist
+        if (empty($this->data['EventReport']['timestamp'])) {
+            $date = new DateTime();
+            $this->data['EventReport']['timestamp'] = $date->getTimestamp();
+        }
+        if ($this->data['EventReport']['distribution'] != 4) {
+            $this->data['EventReport']['sharing_group_id'] = 0;
+        }
+        // Set defaults for when some of the mandatory fields don't have defaults
+        // These fields all have sane defaults either based on another field, or due to server settings
+        if (!isset($this->data['EventReport']['distribution'])) {
+            $this->data['EventReport']['distribution'] = Configure::read('MISP.default_attribute_distribution');
+            if ($this->data['EventReport']['distribution'] == 'event') {
+                $this->data['EventReport']['distribution'] = 5;
+            }
+        }
+        return true;
+    }
+    /**
+     * captureReport Gets a report then save it
+     *
+     * @param  array $user
+     * @param  array $report
+     * @param  int|string $eventId
+     * @return array Any errors preventing the capture
+     */
+    public function captureReport(array $user, array $report, $eventId)
+    {
+        $this->Log = ClassRegistry::init('Log');
+        if (!isset($report['EventReport'])) {
+            $report = ['EventReport' => $report];
+        }
+        $report['EventReport']['event_id'] = $eventId;
+        $report = $this->captureSG($user, $report);
+        $this->create();
+        $errors = $this->saveAndReturnErrors($report, ['fieldList' => $this->captureFields]);
+        if (!empty($errors)) {
+            $this->Log->createLogEntry($user, 'add', 'EventReport', 0,
+                __('Event Report dropped due to validation for Event report %s failed: %s', $report['EventReport']['uuid'], ' failed: ' . $report['EventReport']['name']),
+                __('Validation errors: %s.%sFull report: %s', json_encode($errors), PHP_EOL, json_encode($report['EventReport']))
+            );
+        }
+        return $errors;
+    }
+    
+    /**
+     * addReport Add a report
+     *
+     * @param  array $user
+     * @param  array $report
+     * @param  int|string $eventId
+     * @return array Any errors preventing the addition
+     */
+    public function addReport(array $user, array $report, $eventId)
+    {
+        $errors = $this->captureReport($user, $report, $eventId);
+        if (empty($errors)) {
+            $this->Event->unpublishEvent($eventId);
+        }
+        return $errors;
+    }
+    
+    /**
+     * editReport Edit a report
+     *
+     * @param  array $user
+     * @param  array $report
+     * @param  int|string $eventId
+     * @param  bool  $fromPull
+     * @param  bool  $nothingToChange
+     * @return array Any errors preventing the edition
+     */
+    public function editReport(array $user, array $report, $eventId, $fromPull = false, &$nothingToChange = false)
+    {
+        $errors = array();
+        if (!isset($report['EventReport']['uuid'])) {
+            $errors[] = __('Event Report doesn\'t have an UUID');
+            return $errors;
+        }
+        $report['EventReport']['event_id'] = $eventId;
+        $existingReport = $this->find('first', array(
+            'conditions' => array('EventReport.uuid' => $report['EventReport']['uuid']),
+            'recursive' => -1,
+        ));
+        if (empty($existingReport)) {
+            if ($fromPull) {
+                return $this->captureReport($user, $report, $eventId);
+            } else {
+                $errors[] = __('Event Report not found.');
+                return $errors;
+            }
+        }
+
+        if ($fromPull) {
+            if (isset($report['EventReport']['timestamp'])) {
+                if ($report['EventReport']['timestamp'] <= $existingReport['EventReport']['timestamp']) {
+                    $nothingToChange = true;
+                    return array();
+                }
+            }
+        } else {
+            unset($report['EventReport']['timestamp']);
+        }
+        $errors = $this->saveAndReturnErrors($report, ['fieldList' => $this->captureFields], $errors);
+        if (empty($errors)) {
+            $this->Event->unpublishEvent($eventId);
+        }
+        return $errors;
+    }
+
+    /**
+     * deleteReport ACL-aware method to delete the report.
+     *
+     * @param  array $user
+     * @param  int|string $id
+     * @param  bool $hard
+     * @return array Any errors preventing the deletion
+     */
+    public function deleteReport(array $user, $report, $hard=false)
+    {
+        $report = $this->fetchIfAuthorized($user, $report, 'delete', $throwErrors=true, $full=false);
+        $errors = [];
+        if ($hard) {
+            $deleted = $this->delete($report['EventReport']['id'], true);
+            if (!$deleted) {
+                $errors[] = __('Failed to delete report');
+            }
+        } else {
+            $report['EventReport']['deleted'] = true;
+            $errors = $this->saveAndReturnErrors($report, ['fieldList' => ['deleted']]);
+        }
+        if (empty($errors)) {
+            $this->Event->unpublishEvent($report['EventReport']['event_id']);
+        }
+        return $errors;
+    }
+    
+    /**
+     * restoreReport ACL-aware method to restore a report.
+     *
+     * @param  array $user
+     * @param  int|string $id
+     * @return array Any errors preventing the restoration
+     */
+    public function restoreReport(array $user, $id)
+    {
+        $report = $this->fetchIfAuthorized($user, $id, 'edit', $throwErrors=true, $full=false);
+        $report['EventReport']['deleted'] = false;
+        $errors = $this->saveAndReturnErrors($report, ['fieldList' => ['deleted']]);
+        if (empty($errors)) {
+            $this->Event->unpublishEvent($report['EventReport']['event_id']);
+        }
+        return $errors;
+    }
+
+    private function captureSG(array $user, array $report)
+    {
+        $this->Event = ClassRegistry::init('Event');
+        if (isset($report['EventReport']['distribution']) && $report['EventReport']['distribution'] == 4) {
+            $report['EventReport'] = $this->Event->__captureSGForElement($report['EventReport'], $user);
+        }
+        return $report;
+    }
+    
+    /**
+     * buildACLConditions Generate ACL conditions for viewing the report
+     *
+     * @param  array $user
+     * @return array
+     */
+    public function buildACLConditions(array $user)
+    {
+        $this->Event = ClassRegistry::init('Event');
+        $conditions = array();
+        if (!$user['Role']['perm_site_admin']) {
+            $sgids = $this->Event->cacheSgids($user, true);
+            $eventConditions = $this->Event->createEventConditions($user);
+            $conditions = array(
+                'AND' => array(
+                    $eventConditions['AND'],
+                    array(
+                        'OR' => array(
+                            'Event.org_id' => $user['org_id'],
+                            'EventReport.distribution' => array('1', '2', '3', '5'),
+                            'AND '=> array(
+                                'EventReport.distribution' => 4,
+                                'EventReport.sharing_group_id' => $sgids,
+                            )
+                        )
+                    )
+                )
+            );
+        }
+        return $conditions;
+    }
+
+    /**
+     * fetchById Simple ACL-aware method to fetch a report by Id or UUID
+     *
+     * @param  array $user
+     * @param  int|string $reportId
+     * @param  bool  $throwErrors
+     * @param  bool  $full
+     * @return array
+     */
+    public function simpleFetchById(array $user, $reportId, $throwErrors=true, $full=false)
+    {
+        if (is_numeric($reportId)) {
+            $options = array('conditions' => array("EventReport.id" => $reportId));
+        } elseif (Validation::uuid($reportId)) {
+            $options = array('conditions' => array("EventReport.uuid" => $reportId));
+        } else {
+            if ($throwErrors) {
+                throw new NotFoundException(__('Invalid report'));
+            }
+            return array();
+        }
+
+        $report = $this->fetchReports($user, $options, $full);
+        if (!empty($report)) {
+            return $report[0];
+        }
+        if ($throwErrors) {
+            throw new NotFoundException(__('Invalid report'));
+        }
+        return array();
+    }
+    
+    /**
+     * fetchReports ACL-aware method. Basically find with ACL
+     *
+     * @param  array $user
+     * @param  array $options
+     * @param  bool  $full
+     * @return array
+     */
+    public function fetchReports(array $user, array $options = array(), $full=false)
+    {
+        $params = array(
+            'conditions' => $this->buildACLConditions($user),
+            'contain' => $this->defaultContain,
+            'recursive' => -1
+        );
+        if ($full) {
+            $params['recursive'] = 1;
+        }
+        if (isset($options['fields'])) {
+            $params['fields'] = $options['fields'];
+        }
+        if (isset($options['conditions'])) {
+            $params['conditions']['AND'][] = $options['conditions'];
+        }
+        if (isset($options['group'])) {
+            $params['group'] = empty($options['group']) ? $options['group'] : false;
+        }
+        $reports = $this->find('all', $params);
+        return $reports;
+    }
+
+    /**
+     * fetchIfAuthorized Fetches a report and checks if the user has the authorization to perform the requested operation
+     *
+     * @param  array $user
+     * @param  int|string|array $report
+     * @param  mixed $authorizations the requested actions to be performed on the report
+     * @param  bool  $throwErrors Should the function throws excpetion if users is not allowed to perform the action
+     * @param  bool  $full
+     * @return array The report or an error message
+     */
+    public function fetchIfAuthorized(array $user, $report, $authorizations, $throwErrors=true, $full=false)
+    {
+        $authorizations = is_array($authorizations) ? $authorizations : array($authorizations);
+        $possibleAuthorizations = array('view', 'edit', 'delete');
+        if (!empty(array_diff($authorizations, $possibleAuthorizations))) {
+            throw new NotFoundException(__('Invalid authorization requested'));
+        }
+        if (isset($report['uuid'])) {
+            $report['EventReport'] = $report;
+        }
+        if (!isset($report['EventReport']['uuid'])) {
+            $report = $this->simpleFetchById($user, $report, $throwErrors=$throwErrors, $full=$full);
+            if (empty($report)) {
+                $message = __('Invalid report');
+                return array('authorized' => false, 'error' => $message);
+            }
+        }
+        if ($user['Role']['perm_site_admin']) {
+            return $report;
+        }
+
+        if (in_array('view', $authorizations) && count($authorizations) == 1) {
+            return $report;
+        } else {
+            if (in_array('edit', $authorizations) || in_array('delete', $authorizations)) {
+                $checkResult = $this->canEditReport($user, $report);
+                if ($checkResult !== true) {
+                    if ($throwErrors) {
+                        throw new UnauthorizedException($checkResult);
+                    }
+                    return array('authorized' => false, 'error' => $checkResult);
+                }
+            }
+            return $report;
+        }
+    }
+
+    public function canEditReport(array $user, array $report)
+    {
+        if ($user['Role']['perm_site_admin']) {
+            return true;
+        }
+        if (empty($report['Event'])) {
+            return __('Could not find associated event');
+        }
+        if ($report['Event']['orgc_id'] != $user['org_id']) {
+            return __('Only the creator organisation of the event can modify the report');
+        }
+        return true;
+    }
+    
+    public function reArrangeReport(array $report)
+    {
+        $rearrangeObjects = array('Event', 'SharingGroup');
+        if (isset($report['EventReport'])) {
+            foreach ($rearrangeObjects as $ro) {
+                if (isset($report[$ro]) && !is_null($report[$ro]['id'])) {
+                    $report['EventReport'][$ro] = $report[$ro];
+                }
+                unset($report[$ro]);
+            }
+        }
+        return $report;
+    }
+
+    /**
+     * getProxyMISPElements Extract MISP Elements from an event and make them accessible by their UUID
+     *
+     * @param array $user
+     * @param int|string $eventid
+     * @return array
+     * @throws Exception
+     */
+    public function getProxyMISPElements(array $user, $eventid)
+    {
+        $options = [
+            'noSightings' => true,
+            'sgReferenceOnly' => true,
+            'noEventReports' => true,
+            'noShadowAttributes' => true,
+        ];
+
+        $event = $this->Event->fetchEvent($user, array_merge(['eventid' => $eventid], $options));
+        if (empty($event)) {
+            throw new NotFoundException(__('Invalid Event'));
+        }
+        $event = $event[0];
+
+        if (!empty($event['Event']['extends_uuid'])) {
+            $extendedParentEvent = $this->Event->fetchEvent($user, array_merge([
+                'event_uuid' => $event['Event']['extends_uuid'],
+                'extended' => true,
+            ], $options));
+            if (!empty($extendedParentEvent)) {
+                $event = $extendedParentEvent[0];
+            }
+        }
+
+        $allTagNames = [];
+        foreach ($event['EventTag'] as $eventTag) {
+            // include just tags that belongs to requested event or its parent, not to other child
+            if ($eventTag['event_id'] == $eventid || $eventTag['event_id'] == $event['Event']['id']) {
+                $allTagNames[$eventTag['Tag']['name']] = $eventTag['Tag'];
+            }
+        }
+
+        $attributes = [];
+        foreach ($event['Attribute'] as $attribute) {
+            unset($attribute['ShadowAttribute']);
+            foreach ($attribute['AttributeTag'] as $at) {
+                $allTagNames[$at['Tag']['name']] = $at['Tag'];
+            }
+            $this->Event->Attribute->removeGalaxyClusterTags($attribute);
+            $attributes[$attribute['uuid']] = $attribute;
+        }
+
+        $objects = [];
+        $templateConditions = [];
+        foreach ($event['Object'] as $k => $object) {
+            foreach ($object['Attribute'] as &$objectAttribute) {
+                unset($objectAttribute['ShadowAttribute']);
+                $objectAttribute['object_uuid'] = $object['uuid'];
+                $attributes[$objectAttribute['uuid']] = $objectAttribute;
+
+                foreach ($objectAttribute['AttributeTag'] as $at) {
+                    $allTagNames[$at['Tag']['name']] = $at['Tag'];
+                }
+                $this->Event->Attribute->removeGalaxyClusterTags($objectAttribute);
+            }
+            $objects[$object['uuid']] = $object;
+
+            $uniqueCondition = "{$object['template_uuid']}.{$object['template_version']}";
+            if (!isset($templateConditions[$uniqueCondition])) {
+                $templateConditions[$uniqueCondition]['AND'] = [
+                    'ObjectTemplate.uuid' => $object['template_uuid'],
+                    'ObjectTemplate.version' => $object['template_version']
+                ];
+            }
+        }
+        if (!empty($templateConditions)) {
+            // Fetch object templates for event objects
+            $this->ObjectTemplate = ClassRegistry::init('ObjectTemplate');
+            $templates = $this->ObjectTemplate->find('all', array(
+                'conditions' => ['OR' => array_values($templateConditions)],
+                'recursive' => -1,
+                'contain' => array(
+                    'ObjectTemplateElement' => [
+                        'order' => ['ui-priority' => 'DESC'],
+                        'fields' => ['object_relation', 'type', 'ui-priority']
+                    ]
+                )
+            ));
+            $objectTemplates = [];
+            foreach ($templates as $template) {
+                $objectTemplates["{$template['ObjectTemplate']['uuid']}.{$template['ObjectTemplate']['version']}"] = $template;
+            }
+        } else {
+            $objectTemplates = [];
+        }
+        $this->Galaxy = ClassRegistry::init('Galaxy');
+        $allowedGalaxies = $this->Galaxy->getAllowedMatrixGalaxies();
+        $allowedGalaxies = Hash::combine($allowedGalaxies, '{n}.Galaxy.uuid', '{n}.Galaxy');
+        return [
+            'attribute' => $attributes,
+            'object' => $objects,
+            'objectTemplates' => $objectTemplates,
+            'galaxymatrix' => $allowedGalaxies,
+            'tagname' => $allTagNames
+        ];
+    }
+
+    private function saveAndReturnErrors($data, $saveOptions = [], $errors = [])
+    {
+        $saveSuccess = $this->save($data, $saveOptions);
+        if (!$saveSuccess) {
+            foreach ($this->validationErrors as $validationError) {
+                $errors[] = $validationError[0];
+            }
+        }
+        return $errors;
+    }
+}

app/Model/EventTag.php

@@ -1,9 +1,6 @@
 <?php
 App::uses('AppModel', 'Model');
 
-/**
- * @property Event $Event
- */
 class EventTag extends AppModel
 {
     public $actsAs = array('Containable');
@@ -82,37 +79,6 @@ class EventTag extends AppModel
         $this->delete($id);
     }
 
-    // take an array of tag names to be included and an array with tagnames to be excluded and find all event IDs that fit the criteria
-    public function getEventIDsFromTags($includedTags, $excludedTags)
-    {
-        $conditions = array();
-        if (!empty($includedTags)) {
-            $conditions['OR'] = array('name' => $includedTags);
-        }
-        if (!empty($excludedTags)) {
-            $conditions['NOT'] = array('name' => $excludedTags);
-        }
-        $tags = $this->Tag->find('all', array(
-            'recursive' => -1,
-            'fields' => array('id', 'name'),
-            'conditions' => $conditions
-        ));
-        $tagIDs = array();
-        foreach ($tags as $tag) {
-            $tagIDs[] = $tag['Tag']['id'];
-        }
-        $eventTags = $this->find('all', array(
-            'recursive' => -1,
-            'conditions' => array('tag_id' => $tagIDs)
-        ));
-        $eventIDs = array();
-        foreach ($eventTags as $eventTag) {
-            $eventIDs[] = $eventTag['EventTag']['event_id'];
-        }
-        $eventIDs = array_unique($eventIDs);
-        return $eventIDs;
-    }
-
     public function handleEventTag($event_id, $tag, &$nothingToChange = false)
     {
         if (empty($tag['deleted'])) {
@@ -191,29 +157,11 @@ class EventTag extends AppModel
         return $tags;
     }
 
-    /**
-     * Count number of event that contains given tag for given user. Tag must contains 'EventTag'.
-     *
-     * @param array $tag
-     * @param array $user
-     * @return int
-     */
-    public function countForTag(array $tag, array $user)
+    public function countForTag($tag_id, $user)
     {
-        $eventIds = [];
-        foreach ($tag['EventTag'] as $eventTag) {
-            $eventIds[] = $eventTag['event_id'];
-        }
-
-        if (empty($eventIds)) {
-            return 0;
-        }
-
-        $conditions = $this->Event->createEventConditions($user);
-        $conditions['Event.id'] = $eventIds;
-        return $this->Event->find('count', array(
+        return $this->find('count', array(
             'recursive' => -1,
-            'conditions' => $conditions,
+            'conditions' => array('EventTag.tag_id' => $tag_id)
         ));
     }
 

app/Model/Feed.php

@@ -163,7 +163,7 @@ class Feed extends AppModel
     /**
      * @param array $feed
      * @param HttpSocket|null $HttpSocket Null can be for local feed
-     * @return Generator|array
+     * @return Generator
      * @throws Exception
      */
     public function getCache(array $feed, HttpSocket $HttpSocket = null)
@@ -180,10 +180,7 @@ class Feed extends AppModel
         $tmpFile->write(trim($data));
         unset($data);
 
-        foreach ($tmpFile->lines() as $line) {
-            yield explode(',', rtrim($line));
-        }
-        return;
+        return $tmpFile->csv();
     }
 
     /**
@@ -250,9 +247,7 @@ class Feed extends AppModel
             $data = $this->feedGetUri($feed, $feedUrl, $HttpSocket, true);
 
             if (!$isLocal) {
-                $redis = $this->setupRedisWithException();
-                $redis->del('misp:feed_cache:' . $feed['Feed']['id']);
-                file_put_contents($feedCache, $data);
+                file_put_contents($feedCache, $data); // save to cache
             }
         }
 
@@ -333,126 +328,146 @@ class Feed extends AppModel
     /**
      * Attach correlations from cached servers or feeds.
      *
-     * @param array $objects
+     * @param array $attributes
      * @param array $user
      * @param array $event
-     * @param bool $overrideLimit Override hardcoded limit for 10 000 attribute correlations.
+     * @param bool $overrideLimit Override hardcoded limit for 10 000 correlations.
      * @param string $scope `Feed` or `Server`
      * @return array
      */
-    public function attachFeedCorrelations($objects, $user, &$event, $overrideLimit = false, $scope = 'Feed')
+    public function attachFeedCorrelations(array $attributes, array $user, array &$event, $overrideLimit = false, $scope = 'Feed')
     {
+        if (empty($attributes)) {
+            return $attributes;
+        }
+
         try {
             $redis = $this->setupRedisWithException();
         } catch (Exception $e) {
-            return $objects;
+            return $attributes;
         }
 
         $cachePrefix = 'misp:' . strtolower($scope) . '_cache:';
 
-        // Redis cache for $scope is empty.
+        // Skip if redis cache for $scope is empty.
         if ($redis->sCard($cachePrefix . 'combined') === 0) {
-            return $objects;
+            return $attributes;
         }
 
+        if (!isset($this->Attribute)) {
+            $this->Attribute = ClassRegistry::init('Attribute');
+        }
+        $compositeTypes = $this->Attribute->getCompositeTypes();
+
         $pipe = $redis->multi(Redis::PIPELINE);
-        $hashTable = array();
+        $hashTable = [];
+        $redisResultToAttributePosition = [];
 
-        $this->Event = ClassRegistry::init('Event');
-        $compositeTypes = $this->Event->Attribute->getCompositeTypes();
-
-        foreach ($objects as $k => $object) {
-            if (in_array($object['type'], $compositeTypes)) {
-                $value = explode('|', $object['value']);
-                $hashTable[$k] = md5($value[0]);
-            } else {
-                $hashTable[$k] = md5($object['value']);
+        foreach ($attributes as $k => $attribute) {
+            if (in_array($attribute['type'], $this->Attribute->nonCorrelatingTypes)) {
+                continue; // attribute type is not correlateable
+            }
+            if (!empty($attribute['disable_correlation'])) {
+                continue; // attribute correlation is disabled
+            }
+
+            if (in_array($attribute['type'], $compositeTypes)) {
+                list($value1, $value2) = explode('|', $attribute['value']);
+                $parts = [$value1];
+
+                if (!in_array($attribute['type'], $this->Attribute->primaryOnlyCorrelatingTypes)) {
+                    $parts[] = $value2;
+                }
+            } else {
+                $parts = [$attribute['value']];
+            }
+
+            foreach ($parts as $part) {
+                $md5 = md5($part);
+                $hashTable[] = $md5;
+                $redis->sismember($cachePrefix . 'combined', $md5);
+                $redisResultToAttributePosition[] = $k;
             }
-            $redis->sismember($cachePrefix . 'combined', $hashTable[$k]);
         }
+
+        if (empty($redisResultToAttributePosition)) {
+            // No attribute that can be correlated
+            $pipe->discard();
+            return $attributes;
+        }
+
         $results = $pipe->exec();
-        if (!$overrideLimit && count($objects) > 10000) {
-            foreach ($results as $k => $result) {
-                if ($result && empty($objects[$k]['disable_correlation'])) {
-                    if (isset($event['FeedCount'])) {
-                        $event['FeedCount']++;
-                    } else {
-                        $event['FeedCount'] = 1;
+
+        $hitIds = [];
+        foreach ($results as $k => $result) {
+            if ($result) {
+                $hitIds[] = $k;
+            }
+        }
+
+        if (empty($hitIds)) {
+            return $attributes; // nothing matches, skip
+        }
+
+        $hitCount = count($hitIds);
+        if (!$overrideLimit && $hitCount > 10000) {
+            $event['FeedCount'] = $hitCount;
+            foreach ($hitIds as $k) {
+                $attributes[$redisResultToAttributePosition[$k]]['FeedHit'] = true;
+            }
+            return $attributes;
+        }
+
+        $sources = $this->getCachedFeedsOrServers($user, $scope);
+        foreach ($sources as $source) {
+            $sourceId = $source[$scope]['id'];
+
+            $pipe = $redis->multi(Redis::PIPELINE);
+            foreach ($hitIds as $k) {
+                $redis->sismember($cachePrefix . $sourceId, $hashTable[$k]);
+            }
+            $sourceHits = $pipe->exec();
+            $sourceHasHit = false;
+            foreach ($sourceHits as $k => $hit) {
+                if ($hit) {
+                    if (!isset($event[$scope][$sourceId])) {
+                        $event[$scope][$sourceId] = $source[$scope];
                     }
-                    $objects[$k]['FeedHit'] = true;
+                    $attributePosition = $redisResultToAttributePosition[$hitIds[$k]];
+                    $attributes[$attributePosition][$scope][] = $source[$scope];
+                    $sourceHasHit = true;
                 }
             }
-        } else {
-            if ($scope === 'Feed') {
-                $params = array(
-                    'recursive' => -1,
-                    'fields' => array('id', 'name', 'url', 'provider', 'source_format')
-                );
-                if (!$user['Role']['perm_site_admin']) {
-                    $params['conditions'] = array('Feed.lookup_visible' => 1);
-                }
-                $sources = $this->find('all', $params);
-            } else {
-                $params = array(
-                    'recursive' => -1,
-                    'fields' => array('id', 'name', 'url', 'caching_enabled')
-                );
-                if (!$user['Role']['perm_site_admin']) {
-                    $params['conditions'] = array('Server.caching_enabled' => 1);
-                }
-                $this->Server = ClassRegistry::init('Server');
-                $sources = $this->Server->find('all', $params);
-            }
-
-            $hitIds = array();
-            foreach ($results as $k => $result) {
-                if ($result && empty($objects[$k]['disable_correlation'])) {
-                    $hitIds[] = $k;
-                }
-            }
-            foreach ($sources as $source) {
-                $sourceScopeId = $source[$scope]['id'];
-
+            // Append also exact MISP feed event UUID
+            // TODO: This can be optimised in future to do that in one pass
+            if ($sourceHasHit && ($scope === 'Server' || $source[$scope]['source_format'] === 'misp')) {
                 $pipe = $redis->multi(Redis::PIPELINE);
-                foreach ($hitIds as $k) {
-                    $redis->sismember($cachePrefix . $sourceScopeId, $hashTable[$k]);
-                }
-                $sourceHits = $pipe->exec();
-                foreach ($sourceHits as $k4 => $hit) {
-                    if ($hit) {
-                        if (!isset($event[$scope][$sourceScopeId]['id'])) {
-                            if (!isset($event[$scope][$sourceScopeId])) {
-                                $event[$scope][$sourceScopeId] = array();
-                            }
-                            $event[$scope][$sourceScopeId] = array_merge($event[$scope][$sourceScopeId], $source[$scope]);
-                        }
-                        $objects[$hitIds[$k4]][$scope][] = $source[$scope];
+                $eventUuidHitPosition = [];
+                foreach ($hitIds as $sourceHitPos => $k) {
+                    if ($sourceHits[$sourceHitPos]) {
+                        $redis->smembers($cachePrefix . 'event_uuid_lookup:' . $hashTable[$k]);
+                        $eventUuidHitPosition[] = $redisResultToAttributePosition[$k];
                     }
                 }
-                if ($scope === 'Server' || $source[$scope]['source_format'] == 'misp') {
-                    $pipe = $redis->multi(Redis::PIPELINE);
-                    $eventUuidHitPosition = array();
-                    foreach ($objects as $k => $object) {
-                        if (isset($object[$scope])) {
-                            foreach ($object[$scope] as $currentFeed) {
-                                if ($source[$scope]['id'] == $currentFeed['id']) {
-                                    $eventUuidHitPosition[] = $k;
-                                    $redis->smembers($cachePrefix . 'event_uuid_lookup:' . $hashTable[$k]);
-                                }
-                            }
-                        }
+                $mispFeedHits = $pipe->exec();
+                foreach ($mispFeedHits as $sourceHitPos => $feedUuidMatches) {
+                    if (empty($feedUuidMatches)) {
+                        continue;
                     }
-                    $mispFeedHits = $pipe->exec();
-                    foreach ($mispFeedHits as $sourcehitPos => $f) {
-                        foreach ($f as $url) {
-                            list($feedId, $eventUuid) = explode('/', $url);
-                            if (empty($event[$scope][$feedId]['event_uuids']) || !in_array($eventUuid, $event[$scope][$feedId]['event_uuids'])) {
-                                $event[$scope][$feedId]['event_uuids'][] = $eventUuid;
-                            }
-                            foreach ($objects[$eventUuidHitPosition[$sourcehitPos]][$scope] as $tempKey => $tempFeed) {
-                                if ($tempFeed['id'] == $feedId) {
-                                    $objects[$eventUuidHitPosition[$sourcehitPos]][$scope][$tempKey]['event_uuids'][] = $eventUuid;
-                                }
+                    foreach ($feedUuidMatches as $url) {
+                        list($feedId, $eventUuid) = explode('/', $url);
+                        if ($feedId != $sourceId) {
+                            continue; // just process current source, skip others
+                        }
+
+                        if (empty($event[$scope][$feedId]['event_uuids']) || !in_array($eventUuid, $event[$scope][$feedId]['event_uuids'])) {
+                            $event[$scope][$feedId]['event_uuids'][] = $eventUuid;
+                        }
+                        $attributePosition = $eventUuidHitPosition[$sourceHitPos];
+                        foreach ($attributes[$attributePosition][$scope] as $tempKey => $tempFeed) {
+                            if ($tempFeed['id'] == $feedId) {
+                                $attributes[$attributePosition][$scope][$tempKey]['event_uuids'][] = $eventUuid;
+                                break;
                             }
                         }
                     }
@@ -460,11 +475,58 @@ class Feed extends AppModel
             }
         }
 
-        if (!empty($event[$scope])) {
+        if (isset($event[$scope])) {
             $event[$scope] = array_values($event[$scope]);
         }
 
-        return $objects;
+        return $attributes;
+    }
+
+    /**
+     * Return just feeds or servers that has some data in Redis cache.
+     * @param array $user
+     * @param string $scope 'Feed' or 'Server'
+     * @return array
+     */
+    private function getCachedFeedsOrServers(array $user, $scope)
+    {
+        if ($scope === 'Feed') {
+            $params = array(
+                'recursive' => -1,
+                'fields' => array('id', 'name', 'url', 'provider', 'source_format')
+            );
+            if (!$user['Role']['perm_site_admin']) {
+                $params['conditions'] = array('Feed.lookup_visible' => 1);
+            }
+            $sources = $this->find('all', $params);
+        } else {
+            $params = array(
+                'recursive' => -1,
+                'fields' => array('id', 'name', 'url')
+            );
+            if (!$user['Role']['perm_site_admin']) {
+                $params['conditions'] = array('Server.caching_enabled' => 1);
+            }
+            $this->Server = ClassRegistry::init('Server');
+            $sources = $this->Server->find('all', $params);
+        }
+
+        try {
+            $redis = $this->setupRedisWithException();
+            $pipe = $redis->multi(Redis::PIPELINE);
+            $cachePrefix = 'misp:' . strtolower($scope) . '_cache:';
+            foreach ($sources as $source) {
+                $pipe->exists($cachePrefix . $source[$scope]['id']);
+            }
+            $results = $pipe->exec();
+            foreach ($sources as $k => $source) {
+                if (!$results[$k]) {
+                    unset($sources[$k]);
+                }
+            }
+        } catch (Exception $e) {}
+
+        return $sources;
     }
 
     public function downloadFromFeed($actions, $feed, HttpSocket $HttpSocket = null, $user, $jobId = false)
@@ -518,7 +580,6 @@ class Feed extends AppModel
     {
         $version = $this->checkMISPVersion();
         $version = implode('.', $version);
-        $commit = trim(shell_exec('git log --pretty="%H" -n1 HEAD'));
 
         $result = array(
             'header' => array(
@@ -533,6 +594,7 @@ class Feed extends AppModel
             $result['header']['Accept-Encoding'] = 'gzip';
         }
 
+        $commit = $this->checkMIPSCommit();
         if ($commit) {
             $result['header']['commit'] = $commit;
         }
@@ -1104,52 +1166,73 @@ class Feed extends AppModel
         return $results;
     }
 
-    public function attachFeedCacheTimestamps($data)
+    /**
+     * @param array $feeds
+     * @return array
+     */
+    public function attachFeedCacheTimestamps(array $feeds)
     {
-        $redis = $this->setupRedis();
-        if ($redis === false) {
-            return $data;
+        try {
+            $redis = $this->setupRedisWithException();
+        } catch (Exception $e) {
+            return $feeds;
         }
-        foreach ($data as $k => $v) {
-            $data[$k]['Feed']['cache_timestamp'] = $redis->get('misp:feed_cache_timestamp:' . $data[$k]['Feed']['id']);
+
+        $pipe = $redis->multi(Redis::PIPELINE);
+        foreach ($feeds as $feed) {
+            $pipe->get('misp:feed_cache_timestamp:' . $feed['Feed']['id']);
         }
-        return $data;
+        $result = $redis->exec();
+        foreach ($feeds as $k => $feed) {
+            $feeds[$k]['Feed']['cache_timestamp'] = $result[$k];
+        }
+        return $feeds;
     }
 
     private function __cacheFeed($feed, $redis, $jobId = false)
     {
         $HttpSocket = $this->isFeedLocal($feed) ? null : $this->__setupHttpSocket($feed);
-        if ($feed['Feed']['source_format'] == 'misp') {
+        if ($feed['Feed']['source_format'] === 'misp') {
             return $this->__cacheMISPFeed($feed, $redis, $HttpSocket, $jobId);
         } else {
             return $this->__cacheFreetextFeed($feed, $redis, $HttpSocket, $jobId);
         }
     }
 
+    /**
+     * @param array $feed
+     * @param Redis $redis
+     * @param HttpSocket|null $HttpSocket
+     * @param int|false $jobId
+     * @return bool
+     */
     private function __cacheFreetextFeed(array $feed, $redis, HttpSocket $HttpSocket = null, $jobId = false)
     {
         $feedId = $feed['Feed']['id'];
 
+        $this->jobProgress($jobId, __("Feed %s: Fetching.", $feedId));
+
         try {
             $values = $this->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format'], 'all');
         } catch (Exception $e) {
             $this->logException("Could not get freetext feed $feedId", $e);
-            $this->jobProgress($jobId, 'Could not fetch freetext feed. See error log for more details.');
+            $this->jobProgress($jobId, __('Could not fetch freetext feed. See error log for more details.'));
             return false;
         }
 
         $pipe = $redis->multi(Redis::PIPELINE);
+        $redis->del('misp:feed_cache:' . $feedId);
         foreach ($values as $k => $value) {
             $md5Value = md5($value['value']);
             $redis->sAdd('misp:feed_cache:' . $feedId, $md5Value);
             $redis->sAdd('misp:feed_cache:combined', $md5Value);
-            if ($k % 1000 == 0) {
+            if ($k % 1000 === 0) {
                 $this->jobProgress($jobId, "Feed $feedId: $k/" . count($values) . " values cached.");
                 $pipe->exec();
                 $pipe = $redis->multi(Redis::PIPELINE);
             }
         }
-        $redis->set('misp:feed_cache_timestamp:' . $feedId, time());
+        $pipe->set('misp:feed_cache_timestamp:' . $feedId, time());
         $pipe->exec();
         return true;
     }
@@ -1182,16 +1265,18 @@ class Feed extends AppModel
                     if (!in_array($attribute['type'], $this->Attribute->nonCorrelatingTypes)) {
                         if (in_array($attribute['type'], $this->Attribute->getCompositeTypes())) {
                             $value = explode('|', $attribute['value']);
-                            $redis->sAdd('misp:feed_cache:' . $feedId, md5($value[0]));
-                            $redis->sAdd('misp:feed_cache:' . $feedId, md5($value[1]));
-                            $redis->sAdd('misp:feed_cache:combined', md5($value[0]));
-                            $redis->sAdd('misp:feed_cache:combined', md5($value[1]));
-                            $redis->sAdd('misp:feed_cache:event_uuid_lookup:' . md5($value[0]), $feedId . '/' . $event['Event']['uuid']);
-                            $redis->sAdd('misp:feed_cache:event_uuid_lookup:' . md5($value[1]), $feedId . '/' . $event['Event']['uuid']);
+                            if (in_array($attribute['type'], $this->Attribute->primaryOnlyCorrelatingTypes)) {
+                                unset($value[1]);
+                            }
                         } else {
-                            $redis->sAdd('misp:feed_cache:' . $feedId, md5($attribute['value']));
-                            $redis->sAdd('misp:feed_cache:combined', md5($attribute['value']));
-                            $redis->sAdd('misp:feed_cache:event_uuid_lookup:' . md5($attribute['value']), $feedId . '/' . $event['Event']['uuid']);
+                            $value = [$attribute['value']];
+                        }
+
+                        foreach ($value as $v) {
+                            $md5 = md5($v);
+                            $redis->sAdd('misp:feed_cache:' . $feedId, $md5);
+                            $redis->sAdd('misp:feed_cache:combined', $md5);
+                            $redis->sAdd('misp:feed_cache:event_uuid_lookup:' . $md5, $feedId . '/' . $event['Event']['uuid']);
                         }
                     }
                 }
@@ -1218,10 +1303,12 @@ class Feed extends AppModel
         }
 
         $pipe = $redis->multi(Redis::PIPELINE);
+        $redis->del('misp:feed_cache:' . $feedId);
         foreach ($cache as $v) {
-            $redis->sAdd('misp:feed_cache:' . $feedId, $v[0]);
-            $redis->sAdd('misp:feed_cache:combined', $v[0]);
-            $redis->sAdd('misp:feed_cache:event_uuid_lookup:' . $v[0], $feedId . '/' . $v[1]);
+            list($hash, $eventUuid) = $v;
+            $redis->sAdd('misp:feed_cache:' . $feedId, $hash);
+            $redis->sAdd('misp:feed_cache:combined', $hash);
+            $redis->sAdd('misp:feed_cache:event_uuid_lookup:' . $hash, "$feedId/$eventUuid");
         }
         $pipe->exec();
         $this->jobProgress($jobId, "Feed $feedId: cached via quick cache.");
@@ -1233,7 +1320,7 @@ class Feed extends AppModel
         $result = true;
         if (!$this->__cacheMISPFeedCache($feed, $redis, $HttpSocket, $jobId)) {
             $result = $this->__cacheMISPFeedTraditional($feed, $redis, $HttpSocket, $jobId);
-        };
+        }
         if ($result) {
             $redis->set('misp:feed_cache_timestamp:' . $feed['Feed']['id'], time());
         }
@@ -1743,8 +1830,10 @@ class Feed extends AppModel
     private function jobProgress($jobId = null, $message = null, $progress = null)
     {
         if ($jobId) {
-            $job = ClassRegistry::init('Job');
-            $job->saveProgress($jobId, $message, $progress);
+            if (!isset($this->Job)) {
+                $this->Job = ClassRegistry::init('Job');
+            }
+            $this->Job->saveProgress($jobId, $message, $progress);
         }
     }
 
@@ -1804,13 +1893,25 @@ class Feed extends AppModel
     private function unzipFirstFile(File $zipFile)
     {
         if (!class_exists('ZipArchive')) {
-            throw new Exception('ZIP archive decompressing is not supported. ZIP support is missing in PHP.');
+            throw new Exception('ZIP archive decompressing is not supported. ZIP extension is missing in PHP.');
         }
 
         $zip = new ZipArchive();
         $result = $zip->open($zipFile->pwd());
         if ($result !== true) {
-            throw new Exception("Remote server returns ZIP file, that cannot be open (error $result)");
+            $errorCodes = [
+                ZipArchive::ER_EXISTS => 'file already exists',
+                ZipArchive::ER_INCONS => 'zip archive inconsistent',
+                ZipArchive::ER_INVAL => 'invalid argument',
+                ZipArchive::ER_MEMORY => 'malloc failure',
+                ZipArchive::ER_NOENT => 'no such file',
+                ZipArchive::ER_NOZIP => 'not a zip archive',
+                ZipArchive::ER_OPEN => 'can\'t open file',
+                ZipArchive::ER_READ => 'read error',
+                ZipArchive::ER_SEEK => 'seek error',
+            ];
+            $message = isset($errorCodes[$result]) ? $errorCodes[$result] : 'error ' . $result;
+            throw new Exception("Remote server returns ZIP file, that cannot be open ($message)");
         }
 
         if ($zip->numFiles !== 1) {
@@ -1827,7 +1928,7 @@ class Feed extends AppModel
         $destinationFile = $this->tempFileName();
         $result = copy("zip://{$zipFile->pwd()}#$filename", $destinationFile);
         if ($result === false) {
-            throw new Exception("Remote server returns ZIP file, that contains '$filename' file, that cannot be extracted.");
+            throw new Exception("Remote server returns ZIP file, that contains '$filename' file, but this file cannot be extracted.");
         }
 
         $unzipped = new File($destinationFile);

app/Model/GalaxyCluster.php

@@ -181,6 +181,10 @@ class GalaxyCluster extends AppModel
             }
         }
 
+        if (empty($clusterTagNames)) {
+            return $events;
+        }
+
         $clusters = $this->find('all', array(
             'conditions' => array('LOWER(GalaxyCluster.tag_name)' => $clusterTagNames),
             'contain' => array('Galaxy', 'GalaxyElement'),
@@ -275,7 +279,7 @@ class GalaxyCluster extends AppModel
             foreach ($tmpResults as $tmpResult) {
                 $matchingClusters = array_intersect($matchingClusters, $tmpResult);
             }
-    
+
             $clusterTags = $this->find('list', array(
                 'conditions' => array('id' => $matchingClusters),
                 'fields' => array('GalaxyCluster.tag_name'),
@@ -284,4 +288,14 @@ class GalaxyCluster extends AppModel
         }
         return array_values($clusterTags);
     }
+
+    public function getTagIdByClusterId($cluster_id)
+    {
+        $cluster = $this->find('first', [
+            'recursive' => -1,
+            'conditions' => ['GalaxyCluster.id' => $cluster_id],
+            'contain' => ['Tag']
+        ]);
+        return empty($cluster['Tag']['id']) ? false : $cluster['Tag']['id'];
+    }
 }

app/Model/Log.php

@@ -26,7 +26,7 @@ class Log extends AppModel
                     'admin_email',
                     'auth',
                     'auth_fail',
-                    'blacklisted',
+                    'blocklisted',
                     'change_pw',
                     'delete',
                     'disable',
@@ -114,7 +114,7 @@ class Log extends AppModel
                 $this->data['Log']['ip'] = $_SERVER[$ip_header];
             }
         }
-        $setEmpty = array('title' => '', 'model' => '', 'model_id' => 0, 'action' => '', 'user_id' => 0, 'change' => '', 'email' => '', 'org' => '', 'description' => '');
+        $setEmpty = array('title' => '', 'model' => '', 'model_id' => 0, 'action' => '', 'user_id' => 0, 'change' => '', 'email' => '', 'org' => '', 'description' => '', 'ip' => '');
         foreach ($setEmpty as $field => $empty) {
             if (!isset($this->data['Log'][$field]) || empty($this->data['Log'][$field])) {
                 $this->data['Log'][$field] = $empty;
@@ -384,4 +384,715 @@ class Log extends AppModel
         }
         return $list;
     }
+
+    public function changeParser($change)
+    {
+        $change = explode(',', $change);
+        $data = [];
+        foreach ($change as $entry) {
+            $entry = trim($entry);
+            $fieldName = explode(' ', $entry)[0];
+            $entry = substr($entry, strlen($fieldName));
+            $entry = trim($entry);
+            if (strpos($entry, ') => (')) {
+                list($before, $after) = explode(') => (', $entry);
+                $before = substr($before, 1);
+                $after = substr($after, 0, -1);
+                $data[$fieldName] = $after;
+            }
+        }
+        return $data;
+    }
+
+    public function findDeletedEvents($conditions)
+    {
+        $conditions['model'] = 'Event';
+        $conditions['action'] = 'delete';
+        $this->Event = ClassRegistry::init('Event');
+        $deletions = $this->find('all', [
+            'recursive' => -1,
+            'conditions' => $conditions,
+            'order' => ['Log.id']
+        ]);
+        $deleted_events = [];
+        $users = [];
+        $orgs = [];
+        $deleted_event_ids = [];
+        foreach ($deletions as $deletion_entry) {
+            if (!empty($deleted_event_ids[$deletion_entry['Log']['model_id']])) {
+                continue;
+            } else {
+                $deleted_event_ids[$deletion_entry['Log']['model_id']] = true;
+            }
+            $event = $this->Event->find('first', [
+                'conditions' => ['Event.id' => $deletion_entry['Log']['model_id']],
+                'recursive' => -1,
+                'fields' => ['Event.id']
+            ]);
+            if (!empty($event)) {
+                // event is already restored / not deleted
+                continue;
+            }
+            $temp = [
+                'event_id' => $deletion_entry['Log']['model_id'],
+                'user_id' => $deletion_entry['Log']['user_id'],
+                'created' => $deletion_entry['Log']['created']
+            ];
+            $event_creation_entry = $this->find('first', [
+                'recursive' => -1,
+                'conditions' => [
+                    'model_id' => $temp['event_id'],
+                    'model' => 'Event',
+                    'action' => 'add'
+                ]
+            ]);
+            $event = $this->changeParser($event_creation_entry['Log']['change']);
+            $temp['event_info'] = $event['info'];
+            $temp['event_org_id'] = $event['org_id'];
+            $temp['event_orgc_id'] = $event['orgc_id'];
+            $temp['event_user_id'] = $event['user_id'];
+            $temp['event_info'] = $event['info'];
+            $temp['event_created'] = $event_creation_entry['Log']['created'];
+            foreach (['org', 'orgc'] as $scope) {
+                if (empty($orgs[$temp['event_' . $scope . '_id']])) {
+                    $orgs[$temp['event_' . $scope . '_id']] = array_values($this->Event->Orgc->find('list', [
+                        'recursive' => -1,
+                        'conditions' => array('id' => $temp['event_' . $scope . '_id']),
+                        'fields' => array('id', 'name')
+                    ]))[0];
+                }
+                $temp['event_' . $scope . '_name'] = $orgs[$temp['event_' . $scope . '_id']];
+            }
+            $users[$temp['user_id']] = array_values($this->Event->User->find('list', [
+                'recursive' => -1,
+                'conditions' => array('id' => $temp['user_id']),
+                'fields' => array('id', 'email')
+            ]))[0];
+            $temp['user_name'] = $users[$temp['user_id']];
+            $users[$temp['event_user_id']] = array_values($this->Event->User->find('list', [
+                'recursive' => -1,
+                'conditions' => array('id' => $temp['event_user_id']),
+                'fields' => array('id', 'email')
+            ]))[0];
+            $temp['event_user_name'] = $users[$temp['event_user_id']];
+            $deleted_events[] = $temp;
+        }
+        return $deleted_events;
+    }
+
+    public function recoverDeletedEvent($id, $mock = false)
+    {
+        if ($mock) {
+            $this->mockRecovery = true;
+            $this->mockLog = [];
+        }
+        $objectMap = [];
+        $logEntries = [];
+        $this->__recoverDeletedEventContainer($id, $objectMap, $logEntries);
+        $this->__recoverDeletedObjects($id, $objectMap, $logEntries);
+        $this->__recoverDeletedAttributes($id, $objectMap, $logEntries);
+        $this->__recoverDeletedObjectReferences($id, $objectMap, $logEntries);
+        $this->__recoverDeletedTagConnectors($id, $objectMap, $logEntries, 'Event');
+        $this->__recoverDeletedTagConnectors($id, $objectMap, $logEntries, 'Attribute');
+        $this->__recoverDeletedProposals($id, $objectMap, $logEntries);
+        ksort($logEntries);
+        foreach ($logEntries as $logEntry) {
+            $this->{'__executeRecovery' . $logEntry['model']}($logEntry, $id);
+        }
+        return count($logEntries);
+        // order: event -> object -> attribute -> object reference -> tag -> galaxy -> shadow_attribute -> sighting
+    }
+
+    private function __recoverDeletedEventContainer($id, &$objectMap, &$logEntries)
+    {
+        $logs = $this->find('all', [
+            'recursive' => -1,
+            'conditions' => [
+                'model' => 'Event',
+                'model_id' => $id,
+                'action' => ['add', 'edit', 'publish', 'alert']
+            ]
+        ]);
+        if (empty($logs)) {
+            return;
+        }
+        foreach ($logs as $log) {
+            $logEntries[$log['Log']['id']] = [
+                'model_id' => $log['Log']['model_id'],
+                'model' => $log['Log']['model'],
+                'action' => $log['Log']['action'],
+                'data' => array_merge(
+                    $this->changeParser($log['Log']['change']),
+                    [
+                        'timestamp' => strtotime($log['Log']['created']),
+                        'id' => $log['Log']['model_id']
+                    ]
+                )
+            ];
+            $objectMap['Event'][$log['Log']['model_id']] = true;
+        }
+    }
+
+    private function __recoverDeletedObjects($id, &$objectMap, &$logEntries)
+    {
+        $logs = $this->find('all', [
+            'recursive' => -1,
+            'conditions' => [
+                'model' => 'MispObject',
+                'change LIKE ' => '%event_id () => (' . $id . ')%',
+                'action' => ['add']
+            ]
+        ]);
+        if (empty($logs)) {
+            return;
+        }
+        foreach ($logs as $log) {
+            $objectMap['MispObject'][$log['Log']['model_id']] = true;
+        }
+        $logs = $this->find('all', [
+            'recursive' => -1,
+            'conditions' => [
+                'model' => 'MispObject',
+                'model_id' => array_keys($objectMap['MispObject']),
+                'action' => ['add', 'edit', 'delete']
+            ]
+        ]);
+        foreach ($logs as $log) {
+            $logEntries[$log['Log']['id']] = [
+                'model_id' => $log['Log']['model_id'],
+                'model' => $log['Log']['model'],
+                'action' => $log['Log']['action'],
+                'data' => array_merge(
+                    $this->changeParser($log['Log']['change']),
+                    [
+                        'timestamp' => strtotime($log['Log']['created']),
+                        'id' => $log['Log']['model_id']
+                    ]
+                )
+            ];
+        }
+    }
+
+    private function __recoverDeletedObjectReferences($id, &$objectMap, &$logEntries)
+    {
+        $logs = $this->find('all', [
+            'recursive' => -1,
+            'conditions' => [
+                'model' => 'ObjectReference',
+                'change LIKE ' => '%event_id () => (' . $id . ')%',
+                'action' => ['add']
+            ]
+        ]);
+        if (empty($logs)) {
+            return;
+        }
+        foreach ($logs as $log) {
+            $objectMap['ObjectReference'][$log['Log']['model_id']] = true;
+        }
+        $logs = $this->find('all', [
+            'recursive' => -1,
+            'conditions' => [
+                'model' => 'ObjectReference',
+                'model_id' => array_keys($objectMap['ObjectReference']),
+                'action' => ['add', 'edit']
+            ]
+        ]);
+        foreach ($logs as $log) {
+            $logEntries[$log['Log']['id']] = [
+                'model_id' => $log['Log']['model_id'],
+                'model' => $log['Log']['model'],
+                'action' => $log['Log']['action'],
+                'data' => array_merge(
+                    $this->changeParser($log['Log']['change']),
+                    [
+                        'timestamp' => strtotime($log['Log']['created']),
+                        'id' => $log['Log']['model_id']
+                    ]
+                )
+            ];
+        }
+    }
+
+    private function __recoverDeletedTagConnectors($id, &$objectMap, &$logEntries, $scope)
+    {
+        if (empty($objectMap[$scope])) {
+            // example: we have no attributes, so we return
+            return;
+        }
+        $logs = $this->find('all', [
+            'recursive' => -1,
+            'conditions' => [
+                'model' => $scope,
+                'action' => ['tag', 'galaxy'],
+                'model_id' => array_keys($objectMap[$scope])
+            ]
+        ]);
+        if (empty($logs)) {
+            return;
+        }
+        foreach ($logs as $log) {
+            if ($log['Log']['action'] === 'tag') {
+                $temp = explode(' ', $log['Log']['title']);
+                $local = ($temp[1] === 'local' ? true : false);
+                $tag_id = ($local ? $temp[3] : $temp[2]);
+                $tag_id = substr($tag_id, 1, -1);
+            } else {
+                $matches = [];
+                preg_match('/\(([0-9]*)\)\s(from|to)/', $log['Log']['title'], $matches);
+                if (!isset($matches[1])) {
+                    continue;
+                }
+                $local = false;
+                $tag_id = $matches[1];
+            }
+            $logEntries[$log['Log']['id']] = [
+                'model_id' => $log['Log']['model_id'],
+                'model' => $log['Log']['model'],
+                'action' => (strpos($log['Log']['title'], 'Attached')) === false ? 'remove_tag' : 'add_tag',
+                'data' => [
+                    'tag_id' => $tag_id,
+                    'id' => $log['Log']['model_id'],
+                    'target_type' => $log['Log']['model'],
+                    'tag_type' => $log['Log']['action'],
+                    'local' => $local
+                ]
+            ];
+        }
+    }
+
+    private function __recoverDeletedAttributes($id, &$objectMap, &$logEntries)
+    {
+        $logs = $this->find('all', [
+            'recursive' => -1,
+            'conditions' => [
+                'model' => 'Attribute',
+                'title LIKE ' => '%from Event (' . $id . ')%',
+                'action' => ['add']
+            ]
+        ]);
+        if (empty($logs)) {
+            return;
+        }
+        foreach ($logs as $log) {
+            $objectMap['Attribute'][$log['Log']['model_id']] = true;
+        }
+        $logs = $this->find('all', [
+            'recursive' => -1,
+            'conditions' => [
+                'model' => 'Attribute',
+                'model_id' => array_keys($objectMap['Attribute']),
+                'action' => ['add', 'edit', 'delete']
+            ]
+        ]);
+        foreach ($logs as $log) {
+            $logEntries[$log['Log']['id']] = [
+                'model_id' => $log['Log']['model_id'],
+                'model' => $log['Log']['model'],
+                'action' => $log['Log']['action'],
+                'data' => array_merge(
+                    $this->changeParser($log['Log']['change']),
+                    [
+                        'timestamp' => strtotime($log['Log']['created']),
+                        'id' => $log['Log']['model_id']
+                    ]
+                )
+            ];
+            $objectMap['Attribute'][$log['Log']['model_id']] = true;
+        }
+    }
+
+    private function __recoverDeletedProposals($id, &$objectMap, &$logEntries)
+    {
+        $logs = $this->find('all', [
+            'recursive' => -1,
+            'conditions' => [
+                'model' => 'ShadowAttribute',
+                'title LIKE ' => '%: to Event (' . $id . '): %',
+                'action' => ['add']
+            ]
+        ]);
+        if (empty($logs)) {
+            return;
+        }
+        foreach ($logs as $log) {
+            $objectMap['ShadowAttribute'][$log['Log']['model_id']] = true;
+        }
+        $logs = $this->find('all', [
+            'recursive' => -1,
+            'conditions' => [
+                'model' => 'ShadowAttribute',
+                'model_id' => array_keys($objectMap['ShadowAttribute']),
+                'action' => ['add', 'accept', 'delete']
+            ]
+        ]);
+        foreach ($logs as $log) {
+            $logEntries[$log['Log']['id']] = [
+                'model_id' => $log['Log']['model_id'],
+                'model' => $log['Log']['model'],
+                'action' => $log['Log']['action'],
+                'data' => array_merge(
+                    $this->changeParser($log['Log']['change']),
+                    [
+                        'timestamp' => strtotime($log['Log']['created']),
+                        'id' => $log['Log']['model_id']
+                    ]
+                )
+            ];
+            $objectMap['ShadowAttribute'][$log['Log']['model_id']] = true;
+        }
+    }
+
+
+    private function __executeRecoveryEvent($logEntry, $id)
+    {
+        if (empty($this->Event)) {
+            $this->Event = ClassRegistry::init('Event');
+        }
+        if (empty($this->GalaxyCluster)) {
+            $this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
+        }
+        if (empty($this->EventBlocklist)) {
+            $this->EventBlocklist = ClassRegistry::init('EventBlocklist');
+        }
+        switch($logEntry['action']) {
+            case 'add':
+                if (!empty($this->mockRecovery)) {
+                    $this->mockLog[] = ['model' => 'Event', 'action' => 'add', 'data' => $logEntry['data']];
+                } else {
+                    $this->Event->create();
+                    $this->Event->save($logEntry['data']);
+                    $blockListEntry = $this->EventBlocklist->find('first', array(
+                        'conditions' => array('event_uuid' => $logEntry['data']['uuid']),
+                        'fields' => 'id'
+                    ));
+                    if (!empty($blockListEntry)) {
+                        $this->EventBlocklist->delete($blockListEntry['EventBlocklist']['id']);
+                    }
+                }
+                break;
+            case 'edit':
+            case 'publish':
+                $event = $this->Event->find('first', [
+                    'recursive' => -1,
+                    'conditions' => ['Event.id' => $logEntry['model_id']]
+                ]);
+                if (!empty($event)) {
+                    if ($logEntry['action'] === 'publish' || $logEntry['action'] === 'alert') {
+                        $event['Event']['published'] = 1;
+                        $event['Event']['publish_timestamp'] = strtotime($logEntry['data']['timestamp']);
+                    } else {
+                        foreach ($logEntry['data'] as $field => $value) {
+                            $event['Event'][$field] = $value;
+                        }
+                    }
+                    $this->Event->create();
+                    if (!empty($this->mockRecovery)) {
+                        $this->mockLog[] = ['model' => 'Event', 'action' => 'edit', 'data' => $event];
+                    } else {
+                        $this->Event->save($event);
+                    }
+                }
+                break;
+            case 'add_tag':
+                $tag_id = $logEntry['data']['tag_type'] === 'galaxy' ? $this->GalaxyCluster->getTagIdByClusterId($logEntry['data']['tag_id']) : $logEntry['data']['tag_id'];
+                $this->Event->EventTag->create();
+                $this->Event->create();
+                if (!empty($this->mockRecovery)) {
+                    $this->mockLog[] = ['model' => 'EventTag', 'action' => 'add', 'data' => [
+                        'tag_id' => $tag_id,
+                        'event_id' => $logEntry['data']['id'],
+                        'local' => !empty($logEntry['data']['local'])
+                    ]];
+                } else {
+                    $this->Event->EventTag->save([
+                        'tag_id' => $tag_id,
+                        'event_id' => $logEntry['data']['id'],
+                        'local' => !empty($logEntry['data']['local'])
+                    ]);
+                }
+                break;
+            case 'remove_tag':
+                $tag_id = $logEntry['data']['tag_type'] === 'galaxy' ? $this->GalaxyCluster->getTagIdByClusterId($logEntry['data']['tag_id']) : $logEntry['data']['tag_id'];
+                $et = $this->Event->EventTag->find('first', [
+                    'recursive' => -1,
+                    'conditions' => [
+                        'tag_id' => $tag_id,
+                        'event_id' => $logEntry['data']['id']
+                    ]
+                ]);
+                if (!empty($et)) {
+                    if (!empty($this->mockRecovery)) {
+                        $this->mockLog[] = ['model' => 'EventTag', 'action' => 'delete', 'data' => $et['EventTag']['id']];
+                    } else {
+                        $this->Event->EventTag->delete($et['EventTag']['id']);
+                    }
+                }
+                break;
+        }
+    }
+
+    private function __executeRecoveryAttribute($logEntry, $id)
+    {
+        if (empty($this->Attribute)) {
+            $this->Attribute = ClassRegistry::init('Attribute');
+        }
+        if (empty($this->GalaxyCluster)) {
+            $this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
+        }
+        if (!empty($logEntry['data']['value1'])) {
+            $logEntry['data']['value'] = $logEntry['data']['value1'];
+            if (!empty($logEntry['data']['value2'])) {
+                $logEntry .= '|' . $logEntry['data']['value2'];
+            }
+        }
+        switch($logEntry['action']) {
+            case 'add':
+                $logEntry['data'] = $this->Attribute->UTCToISODatetime(['Attribute' => $logEntry['data']], 'Attribute');
+                $logEntry['data'] = $logEntry['data']['Attribute'];
+                if (!empty($this->mockRecovery)) {
+                    $this->mockLog[] = ['model' => 'Attribute', 'action' => 'add', 'data' => $logEntry['data']];
+                } else {
+                    $this->Attribute->create();
+                    if (!isset($logEntry['data']['to_ids'])) {
+                        $logEntry['data']['to_ids'] = 0;
+                    }
+                    $this->Attribute->save($logEntry['data']);
+                }
+                break;
+            case 'edit':
+                $attribute = $this->Attribute->find('first', [
+                    'recursive' => -1,
+                    'conditions' => ['Attribute.id' => $logEntry['model_id']]
+                ]);
+                if (!empty($attribute)) {
+                    $logEntry['data'] = $this->Attribute->UTCToISODatetime(['Attribute' => $logEntry['data']], 'Attribute');
+                    $logEntry['data'] = $logEntry['data']['Attribute'];
+                    foreach ($logEntry['data'] as $field => $value) {
+                        $attribute['Attribute'][$field] = $value;
+                    }
+                    if (!empty($this->mockRecovery)) {
+                        $this->mockLog[] = ['model' => 'Attribute', 'action' => 'edit', 'data' => $attribute];
+                    } else {
+                        $this->Attribute->save($attribute);
+                    }
+                }
+                break;
+            case 'delete':
+                $attribute = $this->Attribute->find('first', [
+                    'recursive' => -1,
+                    'conditions' => ['Attribute.id' => $logEntry['model_id']]
+                ]);
+                if (!empty($attribute)) {
+                    $attribute['Attribute']['deleted'] = 1;
+                    $attribute['Attribute']['timestamp'] = $logEntry['data']['timestamp'];
+                    if (!empty($this->mockRecovery)) {
+                        $this->mockLog[] = ['model' => 'Attribute', 'action' => 'delete', 'data' => $attribute];
+                    } else {
+                        $this->Attribute->save($attribute);
+                    }
+                }
+                break;
+            case 'add_tag':
+                $tag_id = $logEntry['data']['tag_type'] === 'galaxy' ? $this->GalaxyCluster->getTagIdByClusterId($logEntry['data']['tag_id']) : $logEntry['data']['tag_id'];
+                if (!empty($this->mockRecovery)) {
+                    $this->mockLog[] = ['model' => 'AttributeTag', 'action' => 'add', 'data' => [
+                        'tag_id' => $tag_id,
+                        'attribute_id' => $logEntry['data']['id'],
+                        'event_id' => $id,
+                        'local' => !empty($logEntry['data']['local'])
+                    ]];
+                } else {
+                    $this->Attribute->AttributeTag->create();
+                    $this->Attribute->AttributeTag->save([
+                        'tag_id' => $tag_id,
+                        'attribute_id' => $logEntry['data']['id'],
+                        'event_id' => $id,
+                        'local' => !empty($logEntry['data']['local'])
+                    ]);
+                }
+                break;
+            case 'remove_tag':
+                $tag_id = $logEntry['data']['tag_type'] === 'galaxy' ? $this->GalaxyCluster->getTagIdByClusterId($logEntry['data']['tag_id']) : $logEntry['data']['tag_id'];
+                $at = $this->Attribute->AttributeTag->find('first', [
+                    'recursive' => -1,
+                    'conditions' => [
+                        'tag_id' => $tag_id,
+                        'attribute_id' => $logEntry['data']['id'],
+                        'event_id' => $id
+                    ]
+                ]);
+                if (!empty($at)) {
+                    if (!empty($this->mockRecovery)) {
+                        $this->mockLog[] = ['model' => 'AttributeTag', 'action' => 'delete', 'data' => $at['AttributeTag']['id']];
+                    } else {
+                        $this->Attribute->AttributeTag->delete($at['AttributeTag']['id']);
+                    }
+                }
+                break;
+        }
+    }
+
+    private function __executeRecoveryShadowAttribute($logEntry, $id)
+    {
+        if (empty($this->Attribute)) {
+            $this->Attribute = ClassRegistry::init('Attribute');
+        }
+        if (empty($this->ShadowAttribute)) {
+            $this->ShadowAttribute = ClassRegistry::init('ShadowAttribute');
+        }
+        if (!empty($logEntry['data']['value1'])) {
+            $logEntry['data']['value'] = $logEntry['data']['value1'];
+            if (!empty($logEntry['data']['value2'])) {
+                $logEntry .= '|' . $logEntry['data']['value2'];
+            }
+        }
+        switch($logEntry['action']) {
+            case 'add':
+                $logEntry['data']['value'] = $logEntry['data']['value1'];
+                if (!empty($logEntry['data']['value2'])) {
+                    $logEntry['data']['value'] .= '|' . $logEntry['data']['value2'];
+                }
+                $logEntry['data'] = $this->Attribute->UTCToISODatetime(['ShadowAttribute' => $logEntry['data']], 'ShadowAttribute');
+                if (!empty($this->mockRecovery)) {
+                    $this->mockLog[] = ['model' => 'ShadowAttribute', 'action' => 'add', 'data' => $logEntry['data']];
+                } else {
+                    $this->ShadowAttribute->create();
+                    $this->ShadowAttribute->save($logEntry['data']);
+                }
+                break;
+            case 'delete':
+                $shadow_attribute = $this->ShadowAttribute->find('first', [
+                    'recursive' => -1,
+                    'conditions' => ['ShadowAttribute.id' => $logEntry['model_id']]
+                ]);
+                if (!empty($shadow_attribute)) {
+                    $shadow_attribute['ShadowAttribute']['deleted'] = 1;
+                    $shadow_attribute['ShadowAttribute']['timestamp'] = $logEntry['data']['timestamp'];
+                    if (!empty($this->mockRecovery)) {
+                        $this->mockLog[] = ['model' => 'ShadowAttribute', 'action' => 'delete', 'data' => $attribute];
+                    } else {
+                        $this->ShadowAttribute->save($attribute);
+                    }
+                }
+                break;
+            case 'accept':
+                $shadow_attribute = $this->ShadowAttribute->find('first', [
+                    'recursive' => -1,
+                    'conditions' => ['ShadowAttribute.id' => $logEntry['model_id']]
+                ]);
+                if (!empty($shadow_attribute['ShadowAttribute']['old_id'])) {
+                    $attribute = $this->Attribute->find('first', [
+                        'conditions' => ['Attribute.id' => $shadow_attribute['ShadowAttribute']['old_id']],
+                        'recursive' => -1
+                    ]);
+                    if (!empty($shadow_attribute['ShadowAttribute']['proposal_to_delete'])) {
+                        $attribute['Attribute']['deleted'] = 1;
+                    } else {
+                        foreach(['category', 'type', 'value', 'to_ids', 'comment', 'first_seen', 'last_seen'] as $field) {
+                            if (isset($shadow_attribute['ShadowAttribute'][$field])) {
+                                $attribute['Attribute'][$field] = $shadow_attribute['ShadowAttribute'][$field];
+                            }
+                        }
+                    }
+                    $attribute['Attribute']['timestamp'] = $logEntry['data']['timestamp'];
+                    if (!empty($this->mockRecovery)) {
+                        $this->mockLog[] = ['model' => 'Attribute', 'action' => 'edit', 'data' => $attribute];
+                    } else {
+                        $this->Attribute->save($attribute);
+                    }
+                } else {
+                    $this->Attribute->create();
+                    $attribute = $shadow_attribute['ShadowAttribute'];
+                    if (isset($attribute['id'])) {
+                        unset($attribute['id']);
+                    }
+                    if (!empty($this->mockRecovery)) {
+                        $this->mockLog[] = ['model' => 'Attribute', 'action' => 'add', 'data' => $attribute];
+                    } else {
+                        $this->Attribute->save($attribute);
+                    }
+                }
+                $shadow_attribute['ShadowAttribute']['deleted'] = 1;
+                $shadow_attribute['ShadowAttribute']['timestamp'] = $logEntry['data']['timestamp'];
+                if (!empty($this->mockRecovery)) {
+                    $this->mockLog[] = ['model' => 'ShadowAttribute', 'action' => 'delete', 'data' => $shadow_attribute];
+                } else {
+                    $this->ShadowAttribute->save($shadow_attribute);
+                }
+                break;
+        }
+    }
+
+    private function __executeRecoveryObjectReference($logEntry, $id)
+    {
+        if (empty($this->ObjectReference)) {
+            $this->ObjectReference = ClassRegistry::init('ObjectReference');
+        }
+        switch($logEntry['action']) {
+            case 'add':
+                if (!empty($this->mockRecovery)) {
+                    $this->mockLog[] = ['model' => 'ObjectReference', 'action' => 'add', 'data' => $logEntry['data']];
+                } else {
+                    $this->ObjectReference->create();
+                    $this->ObjectReference->save($logEntry['data']);
+                }
+                break;
+            case 'edit':
+                $objectRef = $this->ObjectReference->find('first', [
+                    'recursive' => -1,
+                    'conditions' => ['ObjectReference.id' => $logEntry['model_id']]
+                ]);
+                if (!empty($object)) {
+                    foreach ($logEntry['data'] as $field => $value) {
+                        $object['ObjectReference'][$field] = $value;
+                    }
+                    if (!empty($this->mockRecovery)) {
+                        $this->mockLog[] = ['model' => 'ObjectReference', 'action' => 'edit', 'data' => $object];
+                    } else {
+                        $this->ObjectReference->save($object);
+                    }
+                }
+                break;
+        }
+    }
+
+    private function __executeRecoveryMispObject($logEntry)
+    {
+        if (empty($this->Attribute)) {
+            $this->Attribute = ClassRegistry::init('Attribute');
+        }
+        if (empty($this->MispObject)) {
+            $this->MispObject = ClassRegistry::init('MispObject');
+        }
+        switch($logEntry['action']) {
+            case 'add':
+                $logEntry['data'] = $this->MispObject->Attribute->UTCToISODatetime(['Object' => $logEntry['data']], 'Object');
+                $logEntry['data'] = $logEntry['data']['Object'];
+                if (!empty($this->mockRecovery)) {
+                    $this->mockLog[] = ['model' => 'MispObject', 'action' => 'add', 'data' => $logEntry['data']];
+                } else {
+                    $this->MispObject->create();
+                    $this->MispObject->save($logEntry['data']);
+                }
+                break;
+            case 'edit':
+                $logEntry['data'] = $this->MispObject->Attribute->UTCToISODatetime(['Object' => $logEntry['data']], 'Object');
+                $logEntry['data'] = $logEntry['data']['Object'];
+                $object = $this->MispObject->find('first', [
+                    'recursive' => -1,
+                    'conditions' => ['Object.id' => $logEntry['model_id']]
+                ]);
+                if (!empty($object)) {
+                    foreach ($logEntry['data'] as $field => $value) {
+                        $object['Object'][$field] = $value;
+                    }
+                    if (!empty($this->mockRecovery)) {
+                        $this->mockLog[] = ['model' => 'MispObject', 'action' => 'add', 'data' => $object];
+                    } else {
+                        $this->MispObject->save($object);
+                    }
+                }
+                break;
+        }
+    }
 }

app/Model/MispObject.php

@@ -5,6 +5,7 @@ App::uses('TmpFileTool', 'Tools');
 /**
  * @property Event $Event
  * @property SharingGroup $SharingGroup
+ * @property Attribute $Attribute
  */
 class MispObject extends AppModel
 {
@@ -59,8 +60,8 @@ class MispObject extends AppModel
     public $validate = array(
         'uuid' => array(
             'uuid' => array(
-                'rule' => array('custom', '/^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$/'),
-                'message' => 'Please provide a valid UUID'
+                'rule' => 'uuid',
+                'message' => 'Please provide a valid RFC 4122 UUID'
             ),
             'unique' => array(
                 'rule' => 'isUnique',
@@ -80,9 +81,9 @@ class MispObject extends AppModel
         )
     );
 
-    public function buildFilterConditions($user, &$params)
+    public function buildFilterConditions(&$params)
     {
-        $conditions = $this->buildConditions($user);
+        $conditions = [];
         if (isset($params['wildcard'])) {
             $temp = array();
             $options = array(
@@ -93,8 +94,6 @@ class MispObject extends AppModel
             );
             $conditions['AND'][] = array('OR' => $this->Event->set_filter_wildcard_attributes($params, $temp, $options));
         } else {
-            $attribute_conditions = array();
-            $object_conditions = array();
             if (isset($params['ignore'])) {
                 $params['to_ids'] = array(0, 1);
                 $params['published'] = array(0, 1);
@@ -392,90 +391,54 @@ class MispObject extends AppModel
         return $result;
     }
 
-    public function buildEventConditions($user, $sgids = false)
+    public function buildConditions(array $user)
     {
         if ($user['Role']['perm_site_admin']) {
-            return array();
+            return [];
         }
-        if ($sgids == false) {
-            $sgsids = $this->SharingGroup->fetchAllAuthorised($user);
-        }
-        return array(
-            'OR' => array(
-                array(
-                    'AND' => array(
-                        'Event.distribution >' => 0,
-                        'Event.distribution <' => 4,
-                        Configure::read('MISP.unpublishedprivate') ? array('Event.published' => 1) : array(),
-                    ),
-                ),
-                array(
-                    'AND' => array(
-                        'Event.sharing_group_id' => $sgids,
-                        'Event.distribution' => 4,
-                        Configure::read('MISP.unpublishedprivate') ? array('Event.published' => 1) : array(),
-                    )
-                )
-            )
-        );
-    }
 
-    public function buildConditions($user, $sgids = false)
-    {
-        $conditions = array();
-        if (!$user['Role']['perm_site_admin']) {
-            if ($sgids === false) {
-                $sgsids = $this->SharingGroup->fetchAllAuthorised($user);
-            }
-            $conditions = array(
-                'AND' => array(
-                    'OR' => array(
-                        array(
-                            'AND' => array(
-                                'Event.org_id' => $user['org_id'],
-                            )
-                        ),
-                        array(
-                            'AND' => array(
-                                $this->buildEventConditions($user, $sgids),
-                                'OR' => array(
-                                    'Object.distribution' => array('1', '2', '3', '5'),
-                                    'AND '=> array(
-                                        'Object.distribution' => 4,
-                                        'Object.sharing_group_id' => $sgsids,
-                                    )
-                                )
-                            )
-                        )
-                    )
-                )
-            );
-        }
-        return $conditions;
+        $sgids = $this->Event->cacheSgids($user, true);
+        return [
+            'AND' => [
+                'OR' => [
+                    'Event.org_id' => $user['org_id'], // if event is owned by current user org, allow access to all objects
+                    'AND' => [
+                        $this->Event->createEventConditions($user),
+                        'OR' => [
+                            'Object.distribution' => array(1, 2, 3, 5),
+                            'AND' => [
+                                'Object.distribution' => 4,
+                                'Object.sharing_group_id' => $sgids,
+                            ]
+                        ]
+                    ]
+                ]
+            ]
+        ];
     }
 
     public function fetchObjectSimple($user, $options = array())
-        {
-            $params = array(
-                'conditions' => $this->buildConditions($user),
-                'fields' => array(),
-                'recursive' => -1
-            );
-            if (isset($options['conditions'])) {
-                $params['conditions']['AND'][] = $options['conditions'];
-            }
-            if (isset($options['fields'])) {
-                $params['fields'] = $options['fields'];
-            }
-            $results = $this->find('all', array(
-                'conditions' => $params['conditions'],
-                'recursive' => -1,
-                'fields' => $params['fields'],
-                'contain' => array('Event' => array('distribution', 'id', 'user_id', 'orgc_id', 'org_id')),
-                'sort' => false
-            ));
-            return $results;
+    {
+        $params = array(
+            'conditions' => $this->buildConditions($user),
+            'fields' => array(),
+            'recursive' => -1
+        );
+        if (isset($options['conditions'])) {
+            $params['conditions']['AND'][] = $options['conditions'];
         }
+        if (isset($options['fields'])) {
+            $params['fields'] = $options['fields'];
+        }
+        $results = $this->find('all', array(
+            'conditions' => $params['conditions'],
+            'recursive' => -1,
+            'fields' => $params['fields'],
+            'contain' => array('Event' => array('distribution', 'id', 'user_id', 'orgc_id', 'org_id')),
+            'sort' => false
+        ));
+        return $results;
+    }
 
     // Method that fetches all objects
     // very flexible, it's basically a replacement for find, with the addition that it restricts access based on user
@@ -487,9 +450,9 @@ class MispObject extends AppModel
     //     group
     public function fetchObjects($user, $options = array())
     {
-        $sgsids = $this->SharingGroup->fetchAllAuthorised($user);
         $attributeConditions = array();
         if (!$user['Role']['perm_site_admin']) {
+            $sgids = $this->Event->cacheSgids($user, true);
             $attributeConditions = array(
                 'OR' => array(
                     array(
@@ -500,7 +463,7 @@ class MispObject extends AppModel
                             'Attribute.distribution' => array(1, 2, 3, 5),
                             array(
                                 'Attribute.distribution' => 4,
-                                'Attribute.sharing_group_id' => $sgsids
+                                'Attribute.sharing_group_id' => $sgids,
                             )
                         )
                     )
@@ -581,20 +544,16 @@ class MispObject extends AppModel
                 $params['page'] = $options['page'];
             }
         }
-        if (Configure::read('MISP.unpublishedprivate') && !$user['Role']['perm_site_admin']) {
-            $params['conditions']['AND'][] = array('OR' => array('Event.published' => 1, 'Event.orgc_id' => $user['org_id']));
-        }
         $results = $this->find('all', $params);
-        if ($options['enforceWarninglist']) {
+        if ($options['enforceWarninglist'] && !isset($this->Warninglist)) {
             $this->Warninglist = ClassRegistry::init('Warninglist');
-            $warninglists = $this->Warninglist->fetchForEventView();
         }
         $results = array_values($results);
         $proposals_block_attributes = Configure::read('MISP.proposals_block_attributes');
         if (empty($options['metadata'])) {
-            foreach ($results as $key => $objects) {
-                foreach ($objects as $key2 => $attribute) {
-                    if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttributes($warninglists, $attribute['Attribute'], $this->Warninglist)) {
+            foreach ($results as $key => $object) {
+                foreach ($object['Attribute'] as $key2 => $attribute) {
+                    if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttribute($attribute['Attribute'])) {
                         unset($results[$key][$key2]);
                         continue;
                     }
@@ -606,9 +565,9 @@ class MispObject extends AppModel
                         }
                     }
                     if ($options['withAttachments']) {
-                        if ($this->typeIsAttachment($attribute['Attribute']['type'])) {
-                            $encodedFile = $this->base64EncodeAttachment($attribute['Attribute']);
-                            $results[$key][$key2]['Attribute']['data'] = $encodedFile;
+                        if ($this->Attribute->typeIsAttachment($attribute['type'])) {
+                            $encodedFile = $this->Attribute->base64EncodeAttachment($attribute);
+                            $results[$key]['Attribute'][$key2]['data'] = $encodedFile;
                         }
                     }
                 }
@@ -1411,25 +1370,25 @@ class MispObject extends AppModel
         $subqueryElements = $this->Event->harvestSubqueryElements($filters);
         $filters = $this->Event->addFiltersFromSubqueryElements($filters, $subqueryElements);
         $filters = $this->Event->addFiltersFromUserSettings($user, $filters);
-        $conditions = $this->buildFilterConditions($user, $filters);
+        $conditions = $this->buildFilterConditions($filters);
         $params = array(
-                'conditions' => $conditions,
-                'fields' => array('Attribute.*', 'Event.org_id', 'Event.distribution', 'Object.*'),
-                'withAttachments' => !empty($filters['withAttachments']) ? $filters['withAttachments'] : 0,
-                'enforceWarninglist' => !empty($filters['enforceWarninglist']) ? $filters['enforceWarninglist'] : 0,
-                'includeAllTags' => !empty($filters['includeAllTags']) ? $filters['includeAllTags'] : 0,
-                'includeEventUuid' => !empty($filters['includeEventUuid']) ? $filters['includeEventUuid'] : 0,
-                'includeEventTags' => !empty($filters['includeEventTags']) ? $filters['includeEventTags'] : 0,
-                'includeProposals' => !empty($filters['includeProposals']) ? $filters['includeProposals'] : 0,
-                'includeWarninglistHits' => !empty($filters['includeWarninglistHits']) ? $filters['includeWarninglistHits'] : 0,
-                'includeContext' => !empty($filters['includeContext']) ? $filters['includeContext'] : 0,
-                'includeSightings' => !empty($filters['includeSightings']) ? $filters['includeSightings'] : 0,
-                'includeSightingdb' => !empty($filters['includeSightingdb']) ? $filters['includeSightingdb'] : 0,
-                'includeCorrelations' => !empty($filters['includeCorrelations']) ? $filters['includeCorrelations'] : 0,
-                'includeDecayScore' => !empty($filters['includeDecayScore']) ? $filters['includeDecayScore'] : 0,
-                'includeFullModel' => !empty($filters['includeFullModel']) ? $filters['includeFullModel'] : 0,
-                'allow_proposal_blocking' => !empty($filters['allow_proposal_blocking']) ? $filters['allow_proposal_blocking'] : 0,
-                'metadata' => !empty($filters['metadata']) ? $filters['metadata'] : 0,
+            'conditions' => $conditions,
+            'fields' => array('Attribute.*', 'Event.org_id', 'Event.distribution', 'Object.*'),
+            'withAttachments' => !empty($filters['withAttachments']) ? $filters['withAttachments'] : 0,
+            'enforceWarninglist' => !empty($filters['enforceWarninglist']) ? $filters['enforceWarninglist'] : 0,
+            'includeAllTags' => !empty($filters['includeAllTags']) ? $filters['includeAllTags'] : 0,
+            'includeEventUuid' => !empty($filters['includeEventUuid']) ? $filters['includeEventUuid'] : 0,
+            'includeEventTags' => !empty($filters['includeEventTags']) ? $filters['includeEventTags'] : 0,
+            'includeProposals' => !empty($filters['includeProposals']) ? $filters['includeProposals'] : 0,
+            'includeWarninglistHits' => !empty($filters['includeWarninglistHits']) ? $filters['includeWarninglistHits'] : 0,
+            'includeContext' => !empty($filters['includeContext']) ? $filters['includeContext'] : 0,
+            'includeSightings' => !empty($filters['includeSightings']) ? $filters['includeSightings'] : 0,
+            'includeSightingdb' => !empty($filters['includeSightingdb']) ? $filters['includeSightingdb'] : 0,
+            'includeCorrelations' => !empty($filters['includeCorrelations']) ? $filters['includeCorrelations'] : 0,
+            'includeDecayScore' => !empty($filters['includeDecayScore']) ? $filters['includeDecayScore'] : 0,
+            'includeFullModel' => !empty($filters['includeFullModel']) ? $filters['includeFullModel'] : 0,
+            'allow_proposal_blocking' => !empty($filters['allow_proposal_blocking']) ? $filters['allow_proposal_blocking'] : 0,
+            'metadata' => !empty($filters['metadata']) ? $filters['metadata'] : 0,
         );
         if (!empty($filters['attackGalaxy'])) {
             $params['attackGalaxy'] = $filters['attackGalaxy'];
@@ -1505,7 +1464,7 @@ class MispObject extends AppModel
         $continue = true;
         while ($continue) {
             $temp = '';
-            $this->Whitelist = ClassRegistry::init('Whitelist');
+            $this->Allowedlist = ClassRegistry::init('Allowedlist');
             $results = $this->fetchObjects($user, $params, $continue);
             if (empty($results)) {
                 $loop = false;
@@ -1519,7 +1478,7 @@ class MispObject extends AppModel
                 $results = $this->Sightingdb->attachToObjects($results, $user);
             }
             $params['page'] += 1;
-            $results = $this->Whitelist->removeWhitelistedFromArray($results, true);
+            $results = $this->Allowedlist->removeAllowedlistedFromArray($results, true);
             $results = array_values($results);
             $i = 0;
             foreach ($results as $object) {

app/Model/Module.php

@@ -145,12 +145,18 @@ class Module extends AppModel
         return $modules;
     }
 
+    /**
+     * @param string $name
+     * @param string $type
+     * @return array|string
+     */
     public function getEnabledModule($name, $type)
     {
+        if (!isset($this->__typeToFamily[$type])) {
+            throw new InvalidArgumentException("Invalid type '$type'.");
+        }
         $moduleFamily = $this->__typeToFamily[$type];
-        $url = $this->__getModuleServer($moduleFamily);
         $modules = $this->getModules($type, $moduleFamily);
-        $module = false;
         if (!Configure::read('Plugin.' . $moduleFamily . '_' . $name . '_enabled')) {
             return 'The requested module is not enabled.';
         }
@@ -175,10 +181,22 @@ class Module extends AppModel
         if (!Configure::read('Plugin.' . $moduleFamily . '_services_enable')) {
             return false;
         }
-        $this->Server = ClassRegistry::init('Server');
-        $url = Configure::read('Plugin.' . $moduleFamily . '_services_url') ? Configure::read('Plugin.' . $moduleFamily . '_services_url') : $this->Server->serverSettings['Plugin'][$moduleFamily . '_services_url']['value'];
-        $port = Configure::read('Plugin.' . $moduleFamily . '_services_port') ? Configure::read('Plugin.' . $moduleFamily . '_services_port') : $this->Server->serverSettings['Plugin'][$moduleFamily . '_services_port']['value'];
-        return $url . ':' . $port;
+
+        $url = Configure::read('Plugin.' . $moduleFamily . '_services_url');
+        $port = Configure::read('Plugin.' . $moduleFamily . '_services_port');
+
+        if (empty($url) || empty($port)) {
+            // Load default values
+            $this->Server = ClassRegistry::init('Server');
+            if (empty($url)) {
+                $url = $this->Server->serverSettings['Plugin'][$moduleFamily . '_services_url']['value'];
+            }
+            if (empty($port)) {
+                $port = $this->Server->serverSettings['Plugin'][$moduleFamily . '_services_port']['value'];
+            }
+        }
+
+        return "$url:$port";
     }
 
     public function queryModuleServer($uri, $post = false, $hover = false, $moduleFamily = 'Enrichment', &$exception = false)
@@ -190,11 +208,11 @@ class Module extends AppModel
         App::uses('HttpSocket', 'Network/Http');
         if ($hover) {
             $settings = array(
-                'timeout' => Configure::read('Plugin.' . $moduleFamily . '_hover_timeout') ? Configure::read('Plugin.' . $moduleFamily . '_hover_timeout') : 5
+                'timeout' => Configure::read('Plugin.' . $moduleFamily . '_hover_timeout') ?: 5
             );
         } else {
             $settings = array(
-                'timeout' => Configure::read('Plugin.' . $moduleFamily . '_timeout') ? Configure::read('Plugin.' . $moduleFamily . '_timeout') : 10
+                'timeout' => Configure::read('Plugin.' . $moduleFamily . '_timeout') ?: 10
             );
         }
         $sslSettings = array('ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_verify_peer', 'ssl_cafile');

app/Model/ObjectReference.php

@@ -45,8 +45,10 @@ class ObjectReference extends AppModel
         if (empty($this->data['ObjectReference']['uuid'])) {
             $this->data['ObjectReference']['uuid'] = CakeText::uuid();
         }
-        $date = new DateTime();
-        $this->data['ObjectReference']['timestamp'] = $date->getTimestamp();
+        if (empty($this->data['ObjectReference']['uuid'])) {
+            $date = new DateTime();
+            $this->data['ObjectReference']['timestamp'] = $date->getTimestamp();
+        }
         if (!isset($this->data['ObjectReference']['comment'])) {
             $this->data['ObjectReference']['comment'] = '';
         }

app/Model/OrgBlocklist.php

@@ -1,8 +1,8 @@
 <?php
 App::uses('AppModel', 'Model');
-class OrgBlacklist extends AppModel
+class OrgBlocklist extends AppModel
 {
-    public $useTable = 'org_blacklists';
+    public $useTable = 'org_blocklists';
 
     public $recursive = -1;
 
@@ -13,19 +13,19 @@ class OrgBlacklist extends AppModel
                     'change' => 'full'),
             'Containable',
     );
-    public $blacklistFields = array('org_uuid', 'comment', 'org_name');
+    public $blocklistFields = array('org_uuid', 'comment', 'org_name');
 
-    public $blacklistTarget = 'org';
+    public $blocklistTarget = 'org';
 
     public $validate = array(
             'org_uuid' => array(
                     'unique' => array(
                             'rule' => 'isUnique',
-                            'message' => 'Organisation already blacklisted.'
+                            'message' => 'Organisation already blocklisted.'
                     ),
                     'uuid' => array(
-                        'rule' => array('custom', '/^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$/'),
-                        'message' => 'Please provide a valid UUID'
+                        'rule' => 'uuid',
+                        'message' => 'Please provide a valid RFC 4122 UUID'
                     ),
             )
     );
@@ -33,8 +33,8 @@ class OrgBlacklist extends AppModel
     public function beforeValidate($options = array())
     {
         parent::beforeValidate();
-        if (empty($this->data['OrgBlacklist']['id'])) {
-            $this->data['OrgBlacklist']['date_created'] = date('Y-m-d H:i:s');
+        if (empty($this->data['OrgBlocklist']['id'])) {
+            $this->data['OrgBlocklist']['date_created'] = date('Y-m-d H:i:s');
         }
         return true;
     }

app/Model/Organisation.php

@@ -33,9 +33,9 @@ class Organisation extends AppModel
                 'rule' => 'isUnique',
                 'message' => 'An organisation with this UUID already exists.'
             ),
-            'simpleuuid' => array(
-                'rule' => array('custom', '/^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$/'),
-                'message' => 'Please provide a valid UUID',
+            'uuid' => array(
+                'rule' => 'uuid',
+                'message' => 'Please provide a valid RFC 4122 UUID',
                 'allowEmpty' => true
             ),
             'valueNotEmpty' => array(
@@ -68,8 +68,6 @@ class Organisation extends AppModel
         ),
     );
 
-    public $countries = array('Not specified', 'International', 'Afghanistan', 'Albania', 'Algeria', 'Andorra', 'Angola', 'Antigua & Deps', 'Argentina', 'Armenia', 'Australia', 'Austria', 'Azerbaijan', 'Bahamas', 'Bahrain', 'Bangladesh', 'Barbados', 'Belarus', 'Belgium', 'Belize', 'Benin', 'Bhutan', 'Bolivia', 'Bosnia Herzegovina', 'Botswana', 'Brazil', 'Brunei', 'Bulgaria', 'Burkina', 'Burundi', 'Cambodia', 'Cameroon', 'Canada', 'Cape Verde', 'Central African Rep', 'Chad', 'Chile', 'China', 'Colombia', 'Comoros', 'Congo', 'Congo {Democratic Rep}', 'Costa Rica', 'Croatia', 'Cuba', 'Cyprus', 'Czech Republic', 'Denmark', 'Djibouti', 'Dominica', 'Dominican Republic', 'East Timor', 'Ecuador', 'Egypt', 'El Salvador', 'Equatorial Guinea', 'Eritrea', 'Estonia', 'Ethiopia', 'Fiji', 'Finland', 'France', 'Gabon', 'Gambia', 'Georgia', 'Germany', 'Ghana', 'Greece', 'Grenada', 'Guatemala', 'Guinea', 'Guinea-Bissau', 'Guyana', 'Haiti', 'Honduras', 'Hungary', 'Iceland', 'India', 'Indonesia', 'Iran', 'Iraq', 'Ireland {Republic}', 'Israel', 'Italy', 'Ivory Coast', 'Jamaica', 'Japan', 'Jordan', 'Kazakhstan', 'Kenya', 'Kiribati', 'Korea North', 'Korea South', 'Kosovo', 'Kuwait', 'Kyrgyzstan', 'Laos', 'Latvia', 'Lebanon', 'Lesotho', 'Liberia', 'Libya', 'Liechtenstein', 'Lithuania', 'Luxembourg', 'Macedonia', 'Madagascar', 'Malawi', 'Malaysia', 'Maldives', 'Mali', 'Malta', 'Marshall Islands', 'Mauritania', 'Mauritius', 'Mexico', 'Micronesia', 'Moldova', 'Monaco', 'Mongolia', 'Montenegro', 'Morocco', 'Mozambique', 'Myanmar, {Burma}', 'Namibia', 'Nauru', 'Nepal', 'Netherlands', 'New Zealand', 'Nicaragua', 'Niger', 'Nigeria', 'Norway', 'Oman', 'Pakistan', 'Palau', 'Panama', 'Papua New Guinea', 'Paraguay', 'Peru', 'Philippines', 'Poland', 'Portugal', 'Qatar', 'Romania', 'Russian Federation', 'Rwanda', 'St Kitts & Nevis', 'St Lucia', 'Saint Vincent & the Grenadines', 'Samoa', 'San Marino', 'Sao Tome & Principe', 'Saudi Arabia', 'Senegal', 'Serbia', 'Seychelles', 'Sierra Leone', 'Singapore', 'Slovakia', 'Slovenia', 'Solomon Islands', 'Somalia', 'South Africa', 'South Sudan', 'Spain', 'Sri Lanka', 'Sudan', 'Suriname', 'Swaziland', 'Sweden', 'Switzerland', 'Syria', 'Taiwan', 'Tajikistan', 'Tanzania', 'Thailand', 'Togo', 'Tonga', 'Trinidad & Tobago', 'Tunisia', 'Turkey', 'Turkmenistan', 'Tuvalu', 'Uganda', 'Ukraine', 'United Arab Emirates', 'United Kingdom', 'United States of America', 'Uruguay', 'Uzbekistan', 'Vanuatu', 'Vatican City', 'Venezuela', 'Vietnam', 'Yemen', 'Zambia', 'Zimbabwe');
-
     public $organisationAssociations = array(
             'Correlation' => array('table' => 'correlations', 'fields' => array('org_id')),
             'Event' => array('table' => 'events', 'fields' => array('org_id', 'orgc_id')),
@@ -88,7 +86,7 @@ class Organisation extends AppModel
         if (empty($this->data['Organisation']['uuid'])) {
             $this->data['Organisation']['uuid'] = CakeText::uuid();
         } else {
-            $this->data['Organisation']['uuid'] = trim($this->data['Organisation']['uuid']);
+            $this->data['Organisation']['uuid'] = strtolower(trim($this->data['Organisation']['uuid']));
         }
         $date = date('Y-m-d H:i:s');
         if (!empty($this->data['Organisation']['restricted_to_domain'])) {
@@ -106,7 +104,7 @@ class Organisation extends AppModel
         }
         $this->data['Organisation']['date_modified'] = $date;
         if (!isset($this->data['Organisation']['nationality']) || empty($this->data['Organisation']['nationality'])) {
-            $this->data['Organisation']['nationality'] = 'Not specified';
+            $this->data['Organisation']['nationality'] = '';
         }
         return true;
     }
@@ -199,7 +197,7 @@ class Organisation extends AppModel
                 $changed = true;
             }
             if ($force) {
-                $fields = array('type', 'date_created', 'date_modified', 'nationality', 'sector', 'contacts', 'landingpage');
+                $fields = array('type', 'date_created', 'date_modified', 'nationality', 'sector', 'contacts');
                 foreach ($fields as $field) {
                     if (isset($org[$field])) {
                         if ($existingOrg['Organisation'][$field] != $org[$field]) {
@@ -456,4 +454,46 @@ class Organisation extends AppModel
         }
         return $suggestedOrg;
     }
+
+    private function getCountryGalaxyCluster()
+    {
+        static $list;
+        if (!$list) {
+            $file = new File(APP . '/files/misp-galaxy/clusters/country.json');
+            if ($file->exists()) {
+                $list = $this->jsonDecode($file->read())['values'];
+                $file->close();
+            } else {
+                $this->log("MISP Galaxy are not updated, countries will not be available.", LOG_WARNING);
+                $list = [];
+            }
+        }
+        return $list;
+    }
+
+    /**
+     * @param string $countryName
+     * @return string|null
+     */
+    public function getCountryCode($countryName)
+    {
+        foreach ($this->getCountryGalaxyCluster() as $country) {
+            if ($country['description'] === $countryName) {
+                return $country['meta']['ISO'];
+            }
+        }
+        return null;
+    }
+
+    /**
+     * @return string[]
+     */
+    public function getCountries()
+    {
+        $countries = ['International'];
+        foreach ($this->getCountryGalaxyCluster() as $country) {
+            $countries[] = $country['description'];
+        }
+        return $countries;
+    }
 }

app/Model/Post.php

@@ -118,8 +118,8 @@ class Post extends AppModel
         $bodyDetail .= "The following message was added: \n";
         $bodyDetail .= "\n";
         $bodyDetail .= $message . "\n";
-        $tplColorString = !empty(Configure::read('MISP.email_subject_TLP_string')) ? Configure::read('MISP.email_subject_TLP_string') : "tlp:amber";
-        $subject = "[" . Configure::read('MISP.org') . " MISP] New post in discussion " . $post['Post']['thread_id'] . " - ".$tplColorString;
+        $tplColorString = Configure::read('MISP.email_subject_TLP_string') ?: "tlp:amber";
+        $subject = "[" . Configure::read('MISP.org') . " MISP] New post in discussion " . $post['Post']['thread_id'] . " - " . strtoupper($tplColorString);
         foreach ($orgMembers as $recipient) {
             $this->User->sendEmail($recipient, $bodyDetail, $body, $subject);
         }

app/Model/Regexp.php

@@ -63,7 +63,7 @@ class Regexp extends AppModel
         }
         // if we found the original, let's try to find all of the regexp values that match the original in the regexp and replacement fields.
         // We should get a list of all the IDs (and their respective types) of regular expression entries that are duplicates created for various types.
-        // ip-src /127.0.0.1/ -> '' and ip-dst /127.0.0.1/ -> '' (entries that blacklists the ip-source and ip-destination addresses 127.0.0.1) will be returned when editing
+        // ip-src /127.0.0.1/ -> '' and ip-dst /127.0.0.1/ -> '' (entries that blocklists the ip-source and ip-destination addresses 127.0.0.1) will be returned when editing
         // ip-src /127.0.0.1/ -> '', but other /127.0.0.1/ -> 'localhost' will not
         if ($original != null) {
             foreach ($allRegexp as $k => $v) {

app/Model/Server.php

@@ -153,6 +153,7 @@ class Server extends AppModel
             ),
             'console_automation_tasks' => array(
                 'data' => array(
+                    'PullAll' => 'MISP/app/Console/cake Server pullAll [user_id] [full|update]',
                     'Pull' => 'MISP/app/Console/cake Server pull [user_id] [server_id] [full|update]',
                     'Push' => 'MISP/app/Console/cake Server push [user_id] [server_id]',
                     'Cache feeds for quick lookups' => 'MISP/app/Console/cake Server cacheFeed [user_id] [feed_id|all|csv|text|misp]',
@@ -542,6 +543,14 @@ class Server extends AppModel
                             'test' => 'testBool',
                             'type' => 'boolean',
                         ),
+                        'osuser' => array(
+                                'level' => 0,
+                                'description' => __('The Unix user MISP (php) is running as'),
+                                'value' => 'www-data',
+                                'errorMessage' => '',
+                                'test' => 'testForEmpty',
+                                'type' => 'string',
+                        ),
                         'email' => array(
                                 'level' => 0,
                                 'description' => __('The e-mail address that MISP should use for all notifications'),
@@ -767,16 +776,16 @@ class Server extends AppModel
                                 'test' => 'testPasswordResetText',
                                 'type' => 'string'
                         ),
-                        'enableEventBlacklisting' => array(
+                        'enableEventBlocklisting' => array(
                                 'level' => 1,
-                                'description' => __('Since version 2.3.107 you can start blacklisting event UUIDs to prevent them from being pushed to your instance. This functionality will also happen silently whenever an event is deleted, preventing a deleted event from being pushed back from another instance.'),
+                                'description' => __('Since version 2.3.107 you can start blocklisting event UUIDs to prevent them from being pushed to your instance. This functionality will also happen silently whenever an event is deleted, preventing a deleted event from being pushed back from another instance.'),
                                 'value' => true,
                                 'type' => 'boolean',
                                 'test' => 'testBool'
                         ),
-                        'enableOrgBlacklisting' => array(
+                        'enableOrgBlocklisting' => array(
                                 'level' => 1,
-                                'description' => __('Blacklisting organisation UUIDs to prevent the creation of any event created by the blacklisted organisation.'),
+                                'description' => __('Blocklisting organisation UUIDs to prevent the creation of any event created by the blocklisted organisation.'),
                                 'value' => true,
                                 'type' => 'boolean',
                                 'test' => 'testBool'
@@ -2065,6 +2074,15 @@ class Server extends AppModel
                             'test' => 'testBool',
                             'type' => 'boolean',
                         ),
+                        'Sightings_anonymise_as' => array(
+                                'level' => 1,
+                                'description' => __('When pushing sightings to another server, report all sightings from this instance as this orignisation. This effectively hides all sightings from this instance behind a single organisation to the outside world. Sightings pulled from this instance follow the Sightings_policy above.'),
+                                'value' => '0',
+                                'errorMessage' => '',
+                                'test' => 'testLocalOrg',
+                                'type' => 'numeric',
+                                'optionsSource' => 'LocalOrgs',
+                        ),
                         'Sightings_range' => array(
                             'level' => 1,
                             'description' => __('Set the range in which sightings will be taken into account when generating graphs. For example a sighting with a sighted_date of 7 years ago might not be relevant anymore. Setting given in number of days, default is 365 days'),
@@ -2461,10 +2479,9 @@ class Server extends AppModel
 
     private function __checkIfEventIsBlockedBeforePull($event)
     {
-        if (Configure::read('MISP.enableEventBlacklisting') !== false) {
-            $this->EventBlacklist = ClassRegistry::init('EventBlacklist');
-            $r = $this->EventBlacklist->find('first', array('conditions' => array('event_uuid' => $event['Event']['uuid'])));
-            if (!empty($r)) {
+        if (Configure::read('MISP.enableEventBlocklisting') !== false) {
+            $this->EventBlocklist = ClassRegistry::init('EventBlocklist');
+            if ($this->EventBlocklist->isBlocked($event['Event']['uuid'])) {
                 return true;
             }
         }
@@ -2503,7 +2520,44 @@ class Server extends AppModel
                     }
                 }
             }
+            if (isset($event['Event']['Object']) && !empty($event['Event']['Object'])) {
+                foreach ($event['Event']['Object'] as $i => $o) {
+                    switch ($o['distribution']) {
+                        case '1':
+                            $event['Event']['Object'][$i]['distribution'] = '0';
+                            break;
+                        case '2':
+                            $event['Event']['Object'][$i]['distribution'] = '1';
+                            break;
+                    }
+                    if (isset($event['Event']['Object'][$i]['Attribute']) && !empty($event['Event']['Object'][$i]['Attribute'])) {
+                        foreach ($event['Event']['Object'][$i]['Attribute'] as $j => $a) {
+                            switch ($a['distribution']) {
+                                case '1':
+                                    $event['Event']['Object'][$i]['Attribute'][$j]['distribution'] = '0';
+                                    break;
+                                case '2':
+                                    $event['Event']['Object'][$i]['Attribute'][$j]['distribution'] = '1';
+                                    break;
+                            }
+                        }
+                    }
+                }
+            }
+            if (isset($event['Event']['EventReport']) && !empty($event['Event']['EventReport'])) {
+                foreach ($event['Event']['EventReport'] as $key => $r) {
+                    switch ($r['distribution']) {
+                        case '1':
+                            $event['Event']['EventReport'][$key]['distribution'] = '0';
+                            break;
+                        case '2':
+                            $event['Event']['EventReport'][$key]['distribution'] = '1';
+                            break;
+                    }
+                }
+            }
         }
+
         // Distribution, set reporter of the event, being the admin that initiated the pull
         $event['Event']['user_id'] = $user['id'];
         return $event;
@@ -2531,6 +2585,13 @@ class Server extends AppModel
                 }
             }
         }
+        if (!empty($event['Event']['EventReport'])) {
+            foreach ($event['Event']['EventReport'] as $report) {
+                if (empty($report['deleted'])) {
+                    return true;
+                }
+            }
+        }
         return false;
     }
 
@@ -2544,6 +2605,10 @@ class Server extends AppModel
             $result = $eventModel->_add($event, true, $user, $server['Server']['org_id'], $passAlong, true, $jobId);
             if ($result) {
                 $successes[] = $eventId;
+                if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) {
+                    $pubSubTool = $this->getPubSubTool();
+                    $pubSubTool->event_save(array('Event' => $eventId, 'Server' => $server['Server']['id']), 'add_from_connected_server');
+                }
             } else {
                 $fails[$eventId] = __('Failed (partially?) because of validation errors: ') . json_encode($eventModel->validationErrors, true);
             }
@@ -2554,6 +2619,10 @@ class Server extends AppModel
                 $result = $eventModel->_edit($event, $user, $existingEvent['Event']['id'], $jobId, $passAlong, $force);
                 if ($result === true) {
                     $successes[] = $eventId;
+                    if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) {
+                        $pubSubTool = $this->getPubSubTool();
+                        $pubSubTool->event_save(array('Event' => $eventId, 'Server' => $server['Server']['id']), 'edit_from_connected_server');
+                    }
                 } elseif (isset($result['error'])) {
                     $fails[$eventId] = $result['error'];
                 } else {
@@ -2590,62 +2659,6 @@ class Server extends AppModel
         return true;
     }
 
-    private function __handlePulledProposals($proposals, $events, $job, $jobId, $eventModel, $user)
-    {
-        $pulledProposals = array();
-        if (!empty($proposals)) {
-            $shadowAttribute = ClassRegistry::init('ShadowAttribute');
-            $shadowAttribute->recursive = -1;
-            $uuidEvents = array_flip($events);
-            foreach ($proposals as $k => &$proposal) {
-                $proposal = $proposal['ShadowAttribute'];
-                $oldsa = $shadowAttribute->findOldProposal($proposal);
-                $proposal['event_id'] = $uuidEvents[$proposal['event_uuid']];
-                if (!$oldsa || $oldsa['timestamp'] < $proposal['timestamp']) {
-                    if ($oldsa) {
-                        $shadowAttribute->delete($oldsa['id']);
-                    }
-                    if (!isset($pulledProposals[$proposal['event_id']])) {
-                        $pulledProposals[$proposal['event_id']] = 0;
-                    }
-                    $pulledProposals[$proposal['event_id']]++;
-                    if (isset($proposal['old_id'])) {
-                        $oldAttribute = $eventModel->Attribute->find('first', array('recursive' => -1, 'conditions' => array('uuid' => $proposal['uuid'])));
-                        if ($oldAttribute) {
-                            $proposal['old_id'] = $oldAttribute['Attribute']['id'];
-                        } else {
-                            $proposal['old_id'] = 0;
-                        }
-                    }
-                    // check if this is a proposal from an old MISP instance
-                    if (!isset($proposal['Org']) && isset($proposal['org']) && !empty($proposal['org'])) {
-                        $proposal['Org'] = $proposal['org'];
-                        $proposal['EventOrg'] = $proposal['event_org'];
-                    } elseif (!isset($proposal['Org']) && !isset($proposal['EventOrg'])) {
-                        continue;
-                    }
-                    $proposal['org_id'] = $this->Organisation->captureOrg($proposal['Org'], $user);
-                    $proposal['event_org_id'] = $this->Organisation->captureOrg($proposal['EventOrg'], $user);
-                    unset($proposal['Org']);
-                    unset($proposal['EventOrg']);
-                    $shadowAttribute->create();
-                    if (!isset($proposal['deleted']) || !$proposal['deleted']) {
-                        if ($shadowAttribute->save($proposal)) {
-                            $shadowAttribute->sendProposalAlertEmail($proposal['event_id']);
-                        }
-                    }
-                }
-                if ($jobId) {
-                    if ($k % 50 == 0) {
-                        $job->id =  $jobId;
-                        $job->saveField('progress', 50 * (($k + 1) / count($proposals)) + 50);
-                    }
-                }
-            }
-        }
-        return $pulledProposals;
-    }
-
     public function pull($user, $id = null, $technique=false, $server, $jobId = false, $force = false)
     {
         $this->Job = ClassRegistry::init('Job');
@@ -2855,29 +2868,29 @@ class Server extends AppModel
                 $eventUuids = array_column($eventArray, 'uuid');
             }
         } else {
-            if (Configure::read('MISP.enableEventBlacklisting') !== false) {
-                $this->EventBlacklist = ClassRegistry::init('EventBlacklist');
-                $blacklistHits = $this->EventBlacklist->find('list', array(
+            if (Configure::read('MISP.enableEventBlocklisting') !== false) {
+                $this->EventBlocklist = ClassRegistry::init('EventBlocklist');
+                $blocklistHits = $this->EventBlocklist->find('list', array(
                     'recursive' => -1,
-                    'conditions' => array('EventBlacklist.event_uuid' => array_column($eventArray, 'uuid')),
-                    'fields' => array('EventBlacklist.event_uuid', 'EventBlacklist.event_uuid'),
+                    'conditions' => array('EventBlocklist.event_uuid' => array_column($eventArray, 'uuid')),
+                    'fields' => array('EventBlocklist.event_uuid', 'EventBlocklist.event_uuid'),
                 ));
                 foreach ($eventArray as $k => $event) {
-                    if (isset($blacklistHits[$event['uuid']])) {
+                    if (isset($blocklistHits[$event['uuid']])) {
                         unset($eventArray[$k]);
                     }
                 }
             }
 
-            if (Configure::read('MISP.enableOrgBlacklisting') !== false) {
-                $this->OrgBlacklist = ClassRegistry::init('OrgBlacklist');
-                $blacklistHits = $this->OrgBlacklist->find('list', array(
+            if (Configure::read('MISP.enableOrgBlocklisting') !== false) {
+                $this->OrgBlocklist = ClassRegistry::init('OrgBlocklist');
+                $blocklistHits = $this->OrgBlocklist->find('list', array(
                     'recursive' => -1,
-                    'conditions' => array('OrgBlacklist.org_uuid' => array_unique(array_column($eventArray, 'orgc_uuid'))),
-                    'fields' => array('OrgBlacklist.org_uuid', 'OrgBlacklist.org_uuid'),
+                    'conditions' => array('OrgBlocklist.org_uuid' => array_unique(array_column($eventArray, 'orgc_uuid'))),
+                    'fields' => array('OrgBlocklist.org_uuid', 'OrgBlocklist.org_uuid'),
                 ));
                 foreach ($eventArray as $k => $event) {
-                    if (isset($blacklistHits[$event['orgc_uuid']])) {
+                    if (isset($blocklistHits[$event['orgc_uuid']])) {
                         unset($eventArray[$k]);
                     }
                 }
@@ -2897,21 +2910,32 @@ class Server extends AppModel
         return $eventUuids;
     }
 
-    public function push($id = null, $technique=false, $jobId = false, $HttpSocket, $user)
+    /**
+     * @param int $id Server ID
+     * @param string|int $technique Can be 'full', 'incremental' or event ID
+     * @param int|false $jobId
+     * @param HttpSocket $HttpSocket
+     * @param array $user
+     * @return array|bool
+     * @throws Exception
+     */
+    public function push($id, $technique, $jobId = false, $HttpSocket, array $user)
     {
         if ($jobId) {
             $job = ClassRegistry::init('Job');
-            $job->read(null, $jobId);
+        }
+        $server = $this->find('first', ['conditions' => ['Server.id' => $id]]);
+        if (!$server) {
+            throw new NotFoundException('Server not found');
         }
         $this->Event = ClassRegistry::init('Event');
-        $this->read(null, $id);
-        $url = $this->data['Server']['url'];
+        $url = $server['Server']['url'];
         $push = $this->checkVersionCompatibility($id, $user);
         if (is_array($push) && !$push['canPush'] && !$push['canSight']) {
             $push = 'Remote instance is outdated or no permission to push.';
         }
         if (!is_array($push)) {
-            $message = sprintf('Push to server %s failed. Reason: %s', $id, $push);
+            $message = __('Push to server %s failed. Reason: %s', $id, $push);
             $this->Log = ClassRegistry::init('Log');
             $this->Log->create();
             $this->Log->save(array(
@@ -2930,14 +2954,14 @@ class Server extends AppModel
             return $push;
         }
 
-        // sync events if user is capable
-        if ($push['canPush']) {
+        // sync events if user is capable and server is configured for push
+        if ($push['canPush'] && $server['Server']['push']) {
             if ("full" == $technique) {
                 $eventid_conditions_key = 'Event.id >';
                 $eventid_conditions_value = 0;
             } elseif ("incremental" == $technique) {
                 $eventid_conditions_key = 'Event.id >';
-                $eventid_conditions_value = $this->data['Server']['lastpushedid'];
+                $eventid_conditions_value = $server['Server']['lastpushedid'];
             } elseif (intval($technique) !== 0) {
                 $eventid_conditions_key = 'Event.id';
                 $eventid_conditions_value = intval($technique);
@@ -2949,19 +2973,24 @@ class Server extends AppModel
                 'contain' => array('Organisation', 'SharingGroupOrg' => array('Organisation'), 'SharingGroupServer')
             ));
             $sgIds = array();
-            foreach ($sgs as $k => $sg) {
-                if ($this->Event->SharingGroup->checkIfServerInSG($sg, $this->data)) {
+            foreach ($sgs as $sg) {
+                if ($this->Event->SharingGroup->checkIfServerInSG($sg, $server)) {
                     $sgIds[] = $sg['SharingGroup']['id'];
                 }
             }
             if (empty($sgIds)) {
                 $sgIds = array(-1);
             }
+            $tableName = $this->Event->EventReport->table;
+            $eventReportQuery = sprintf('EXISTS (SELECT id, deleted FROM %s WHERE %s.event_id = Event.id and %s.deleted = 0)', $tableName, $tableName, $tableName);
             $findParams = array(
                     'conditions' => array(
                             $eventid_conditions_key => $eventid_conditions_value,
                             'Event.published' => 1,
-                            'Event.attribute_count >' => 0,
+                            'OR' => array(
+                                array('Event.attribute_count >' => 0),
+                                array($eventReportQuery),
+                            ),
                             'OR' => array(
                                 array(
                                     'AND' => array(
@@ -2983,62 +3012,61 @@ class Server extends AppModel
             );
             $eventIds = $this->Event->find('all', $findParams);
             $eventUUIDsFiltered = $this->getEventIdsForPush($id, $HttpSocket, $eventIds, $user);
-            if ($eventUUIDsFiltered === false || empty($eventUUIDsFiltered)) {
-                $pushFailed = true;
-            }
             if (!empty($eventUUIDsFiltered)) {
                 $eventCount = count($eventUUIDsFiltered);
                 // now process the $eventIds to push each of the events sequentially
-                if (!empty($eventUUIDsFiltered)) {
-                    $successes = array();
-                    $fails = array();
-                    $lowestfailedid = null;
-                    foreach ($eventUUIDsFiltered as $k => $eventUuid) {
-                        $params = array();
-                        if (!empty($this->data['Server']['push_rules'])) {
-                            $push_rules = json_decode($this->data['Server']['push_rules'], true);
-                            if (!empty($push_rules['tags']['NOT'])) {
-                                $params['blockedAttributeTags'] = $push_rules['tags']['NOT'];
-                            }
+                $successes = array();
+                $fails = array();
+                foreach ($eventUUIDsFiltered as $k => $eventUuid) {
+                    $params = array();
+                    if (!empty($server['Server']['push_rules'])) {
+                        $push_rules = json_decode($server['Server']['push_rules'], true);
+                        if (!empty($push_rules['tags']['NOT'])) {
+                            $params['blockedAttributeTags'] = $push_rules['tags']['NOT'];
                         }
+                    }
+                    $params = array_merge($params, array(
+                        'event_uuid' => $eventUuid,
+                        'includeAttachments' => true,
+                        'includeAllTags' => true,
+                        'deleted' => array(0,1),
+                        'excludeGalaxy' => 1
+                    ));
+                    if (empty($server['Server']['push_sightings'])) {
                         $params = array_merge($params, array(
-                            'event_uuid' => $eventUuid,
-                            'includeAttachments' => true,
-                            'includeAllTags' => true,
-                            'deleted' => array(0,1),
-                            'excludeGalaxy' => 1
+                            'noSightings' => 1
                         ));
-                        $event = $this->Event->fetchEvent($user, $params);
-                        $event = $event[0];
-                        $event['Event']['locked'] = 1;
-                        $result = $this->Event->uploadEventToServer($event, $this->data, $HttpSocket);
-                        if ('Success' === $result) {
-                            $successes[] = $event['Event']['id'];
-                        } else {
-                            $fails[$event['Event']['id']] = $result;
-                        }
-                        if ($jobId && $k%10 == 0) {
-                            $job->saveField('progress', 100 * $k / $eventCount);
-                        }
                     }
-                    if (count($fails) > 0) {
-                        // there are fails, take the lowest fail
-                        $lastpushedid = min(array_keys($fails));
+                    $event = $this->Event->fetchEvent($user, $params);
+                    $event = $event[0];
+                    $event['Event']['locked'] = 1;
+                    $result = $this->Event->uploadEventToServer($event, $server, $HttpSocket);
+                    if ('Success' === $result) {
+                        $successes[] = $event['Event']['id'];
                     } else {
-                        // no fails, take the highest success
-                        $lastpushedid = max($successes);
+                        $fails[$event['Event']['id']] = $result;
+                    }
+                    if ($jobId && $k % 10 == 0) {
+                        $job->saveProgress($jobId, null, 100 * $k / $eventCount);
                     }
-                    // increment lastid based on the highest ID seen
-                    // Save the entire Server data instead of just a single field, so that the logger can be fed with the extra fields.
-                    $this->data['Server']['lastpushedid'] = $lastpushedid;
-                    $this->save($this->data);
                 }
+                if (count($fails) > 0) {
+                    // there are fails, take the lowest fail
+                    $lastpushedid = min(array_keys($fails));
+                } else {
+                    // no fails, take the highest success
+                    $lastpushedid = max($successes);
+                }
+                // increment lastid based on the highest ID seen
+                // Save the entire Server data instead of just a single field, so that the logger can be fed with the extra fields.
+                $server['Server']['lastpushedid'] = $lastpushedid;
+                $this->save($server);
             }
-            $this->syncProposals($HttpSocket, $this->data, null, null, $this->Event);
+            $this->syncProposals($HttpSocket, $server, null, null, $this->Event);
         }
 
         if ($push['canPush'] || $push['canSight']) {
-            $sightingSuccesses = $this->syncSightings($HttpSocket, $this->data, $user, $this->Event);
+            $sightingSuccesses = $this->syncSightings($HttpSocket, $server, $user, $this->Event);
         } else {
             $sightingSuccesses = array();
         }
@@ -3116,7 +3144,7 @@ class Server extends AppModel
             $event = $eventModel->fetchEvent($user, $options = array('event_uuid' => $eventId, 'metadata' => true));
             if (!empty($event)) {
                 $event = $event[0];
-                $event['Sighting'] = $this->Sighting->attachToEvent($event, $user);
+                $event['Sighting'] = $this->Sighting->attachToEvent($event, $user, null, false, true);
                 $result = $eventModel->uploadEventToServer($event, $server, $HttpSocket, 'sightings');
                 if ($result === 'Success') {
                     $successes[] = 'Sightings for event ' .  $event['Event']['id'];
@@ -3126,7 +3154,7 @@ class Server extends AppModel
         return $successes;
     }
 
-    public function syncProposals($HttpSocket, $server, $sa_id = null, $event_id = null, $eventModel)
+    public function syncProposals($HttpSocket, array $server, $sa_id = null, $event_id = null, $eventModel)
     {
         $saModel = ClassRegistry::init('ShadowAttribute');
         $HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
@@ -3585,7 +3613,7 @@ class Server extends AppModel
         if ($this->testForEmpty($value) !== true) {
             return $this->testForEmpty($value);
         }
-        $regex = "%^(?<proto>https?)://(?<host>(?:(?:\w|-)+\.)+[a-z]{2,5})(?::(?<port>[0-9]+))?(?<base>/[a-z0-9_\-\.]+)?$%i";
+        $regex = "%^(?<proto>https?)://(?<host>(?:(?:\w|-)+\.)+[a-z]{2,})(?::(?<port>[0-9]+))?(?<base>/[a-z0-9_\-\.]+)?$%i";
 	if ( !preg_match($regex, $value, $matches)
                 || strtolower($matches['proto']) != strtolower($this->getProto())
                 || strtolower($matches['host']) != strtolower($this->getHost()) ) {
@@ -4760,9 +4788,9 @@ class Server extends AppModel
         $dbActualIndexes = array();
         $dataSource = $this->getDataSource()->config['datasource'];
         if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
-            $sqlGetTable = sprintf('SELECT TABLE_NAME FROM information_schema.tables WHERE table_schema = %s;', "'" . $this->getDataSource()->config['database'] . "'");
+            $sqlGetTable = sprintf('SELECT TABLE_NAME FROM information_schema.tables WHERE table_schema = %s ORDER BY TABLE_NAME;', "'" . $this->getDataSource()->config['database'] . "'");
             $sqlResult = $this->query($sqlGetTable);
-            $tables = HASH::extract($sqlResult, '{n}.tables.TABLE_NAME');
+            $tables = Hash::extract($sqlResult, '{n}.tables.TABLE_NAME');
             foreach ($tables as $table) {
                 $sqlSchema = sprintf(
                     "SELECT %s
@@ -4785,7 +4813,7 @@ class Server extends AppModel
     public function compareDBSchema($dbActualSchema, $dbExpectedSchema)
     {
         // Column that should be ignored while performing the comparison
-        $whiteListFields = array(
+        $allowedlistFields = array(
             'users' => array('external_auth_required', 'external_auth_key'),
         );
         $nonCriticalColumnElements = array('is_nullable', 'collation_name');
@@ -4817,8 +4845,8 @@ class Server extends AppModel
 
                 $additionalKeysInActualSchema = array_diff($existingColumnKeys, $expectedColumnKeys);
                 foreach($additionalKeysInActualSchema as $additionalKeys) {
-                    if (isset($whiteListFields[$tableName]) && in_array($additionalKeys, $whiteListFields[$tableName])) {
-                        continue; // column is whitelisted
+                    if (isset($allowedlistFields[$tableName]) && in_array($additionalKeys, $allowedlistFields[$tableName])) {
+                        continue; // column is allowedlisted
                     }
                     $dbDiff[$tableName][] = array(
                         'description' => sprintf(__('Column `%s` exists but should not'), $additionalKeys),
@@ -4828,8 +4856,8 @@ class Server extends AppModel
                     );
                 }
                 foreach ($keyedExpectedColumn as $columnName => $column) {
-                    if (isset($whiteListFields[$tableName]) && in_array($columnName, $whiteListFields[$tableName])) {
-                        continue; // column is whitelisted
+                    if (isset($allowedlistFields[$tableName]) && in_array($columnName, $allowedlistFields[$tableName])) {
+                        continue; // column is allowedlisted
                     }
                     if (isset($keyedActualColumn[$columnName])) {
                         $colDiff = array_diff_assoc($column, $keyedActualColumn[$columnName]);
@@ -4886,53 +4914,120 @@ class Server extends AppModel
         return $dbDiff;
     }
 
-    public function compareDBIndexes($actualIndex, $expectedIndex, $dbExpectedSchema)
+    /**
+     * Returns `true` if given column for given table contains just unique values.
+     *
+     * @param string $tableName
+     * @param string $columnName
+     * @return bool
+     */
+    private function checkIfColumnContainsJustUniqueValues($tableName, $columnName)
     {
-        $defaultIndexKeylength = 255;
-        $whitelistTables = array();
+        $db = $this->getDataSource();
+        $duplicates = $this->query(
+            sprintf('SELECT %s, COUNT(*) c FROM %s GROUP BY %s HAVING c > 1;',
+                $db->name($columnName), $db->name($tableName), $db->name($columnName))
+        );
+        return empty($duplicates);
+    }
+
+    private function generateSqlDropIndexQuery($tableName, $columnName)
+    {
+        return sprintf('DROP INDEX `%s` ON %s;',
+            $columnName,
+            $tableName
+        );
+    }
+
+    private function generateSqlIndexQuery(array $dbExpectedSchema, $tableName, $columnName, $shouldBeUnique = false, $defaultIndexKeylength = 255)
+    {
+        $columnData = Hash::extract($dbExpectedSchema['schema'][$tableName], "{n}[column_name=$columnName]");
+        if (empty($columnData)) {
+            throw new Exception("Index in db_schema.json is defined for `$tableName.$columnName`, but this column is not defined.");
+        }
+
+        $columnData = $columnData[0];
+        if ($columnData['data_type'] === 'varchar') {
+            $keyLength = sprintf('(%s)', $columnData['character_maximum_length'] < $defaultIndexKeylength ? $columnData['character_maximum_length'] : $defaultIndexKeylength);
+        } elseif ($columnData['data_type'] === 'text') {
+            $keyLength = sprintf('(%s)', $defaultIndexKeylength);
+        } else {
+            $keyLength = '';
+        }
+        return sprintf('CREATE%s INDEX `%s` ON `%s` (`%s`%s);',
+            $shouldBeUnique ? ' UNIQUE' : '',
+            $columnName,
+            $tableName,
+            $columnName,
+            $keyLength
+        );
+    }
+
+    public function compareDBIndexes(array $actualIndex, array $expectedIndex, array $dbExpectedSchema)
+    {
+        $allowedlistTables = array();
         $indexDiff = array();
-        foreach($expectedIndex as $tableName => $indexes) {
+        foreach ($expectedIndex as $tableName => $indexes) {
             if (!array_key_exists($tableName, $actualIndex)) {
                 continue; // If table does not exists, it is covered by the schema diagnostic
-            } elseif(in_array($tableName, $whitelistTables)) {
-                continue; // Ignore whitelisted tables
+            } elseif(in_array($tableName, $allowedlistTables)) {
+                continue; // Ignore allowedlisted tables
             } else {
-                $tableIndexDiff = array_diff($indexes, $actualIndex[$tableName]); // check for missing indexes
-                if (count($tableIndexDiff) > 0) {
-                    foreach($tableIndexDiff as $columnDiff) {
-                        $columnData  = Hash::extract($dbExpectedSchema['schema'][$tableName], sprintf('{n}[column_name=%s]', $columnDiff))[0];
-                        $message = sprintf(__('Column `%s` should be indexed'), $columnDiff);
-                        if ($columnData['data_type'] == 'varchar') {
-                            $keyLength = sprintf('(%s)', $columnData['character_maximum_length'] < $defaultIndexKeylength ? $columnData['character_maximum_length'] : $defaultIndexKeylength);
-                        } elseif ($columnData['data_type'] == 'text') {
-                            $keyLength = sprintf('(%s)', $defaultIndexKeylength);
-                        } else {
-                            $keyLength = '';
-                        }
-                        $sql = sprintf('CREATE INDEX `%s` ON `%s` (`%s`%s);',
-                            $columnDiff,
-                            $tableName,
-                            $columnDiff,
-                            $keyLength
-                        );
+                $tableIndexDiff = array_diff(array_keys($indexes), array_keys($actualIndex[$tableName])); // check for missing indexes
+                foreach ($tableIndexDiff as $columnDiff) {
+                    $shouldBeUnique = $indexes[$columnDiff];
+                    if ($shouldBeUnique && !$this->checkIfColumnContainsJustUniqueValues($tableName, $columnDiff)) {
                         $indexDiff[$tableName][$columnDiff] = array(
-                            'message' => $message,
-                            'sql' => $sql
+                            'message' => __('Column `%s` should be unique indexed, but contains duplicate values', $columnDiff),
+                            'sql' => '',
                         );
+                        continue;
                     }
+
+                    $message = __('Column `%s` should be indexed', $columnDiff);
+                    $indexDiff[$tableName][$columnDiff] = array(
+                        'message' => $message,
+                        'sql' => $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $columnDiff, $shouldBeUnique),
+                    );
                 }
-                $tableIndexDiff = array_diff($actualIndex[$tableName], $indexes); // check for additional indexes
-                if (count($tableIndexDiff) > 0) {
-                    foreach($tableIndexDiff as $columnDiff) {
-                        $message = sprintf(__('Column `%s` is indexed but should not'), $columnDiff);
-                        $sql = sprintf('DROP INDEX `%s` ON %s;',
-                            $columnDiff,
-                            $tableName
-                        );
-                        $indexDiff[$tableName][$columnDiff] = array(
-                            'message' => $message,
-                            'sql' => $sql
-                        );
+                $tableIndexDiff = array_diff(array_keys($actualIndex[$tableName]), array_keys($indexes)); // check for additional indexes
+                foreach ($tableIndexDiff as $columnDiff) {
+                    $message = __('Column `%s` is indexed but should not', $columnDiff);
+                    $indexDiff[$tableName][$columnDiff] = array(
+                        'message' => $message,
+                        'sql' => $this->generateSqlDropIndexQuery($tableName, $columnDiff),
+                    );
+                }
+                foreach ($indexes as $column => $unique) {
+                    if (isset($actualIndex[$tableName][$column]) && $actualIndex[$tableName][$column] != $unique) {
+                        if ($actualIndex[$tableName][$column]) {
+                            $sql = $this->generateSqlDropIndexQuery($tableName, $column);
+                            $sql .= '<br>' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, false);
+
+                            $message = __('Column `%s` has unique index, but should be non unique', $column);
+                            $indexDiff[$tableName][$column] = array(
+                                'message' => $message,
+                                'sql' => $sql,
+                            );
+                        } else {
+                            if (!$this->checkIfColumnContainsJustUniqueValues($tableName, $column)) {
+                                $message = __('Column `%s` should be unique index, but contains duplicate values', $column);
+                                $indexDiff[$tableName][$column] = array(
+                                    'message' => $message,
+                                    'sql' => '',
+                                );
+                                continue;
+                            }
+
+                            $sql = $this->generateSqlDropIndexQuery($tableName, $column);
+                            $sql .= '<br>' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, true);
+
+                            $message = __('Column `%s` should be unique index', $column);
+                            $indexDiff[$tableName][$column] = array(
+                                'message' => $message,
+                                'sql' => $sql,
+                            );
+                        }
                     }
                 }
             }
@@ -4940,16 +5035,27 @@ class Server extends AppModel
         return $indexDiff;
     }
 
+    /**
+     * Returns indexes for given schema and table in array, where key is column name and value is `true` if
+     * index is index is unique, `false` otherwise.
+     *
+     * @param string $database
+     * @param string $table
+     * @return array
+     */
     public function getDatabaseIndexes($database, $table)
     {
         $sqlTableIndex = sprintf(
-            "SELECT DISTINCT TABLE_NAME, COLUMN_NAME FROM information_schema.statistics WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s';",
+            "SELECT DISTINCT TABLE_NAME, COLUMN_NAME, NON_UNIQUE FROM information_schema.statistics WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s';",
             $database,
             $table
         );
         $sqlTableIndexResult = $this->query($sqlTableIndex);
-        $tableIndex = Hash::extract($sqlTableIndexResult, '{n}.statistics.COLUMN_NAME');
-        return $tableIndex;
+        $output = [];
+        foreach ($sqlTableIndexResult as $index) {
+            $output[$index['statistics']['COLUMN_NAME']] = $index['statistics']['NON_UNIQUE'] == 0;
+        }
+        return $output;
     }
 
     public function writeableDirsDiagnostics(&$diagnostic_errors)
@@ -5080,9 +5186,8 @@ class Server extends AppModel
         $gpgStatus = 0;
         if (Configure::read('GnuPG.email') && Configure::read('GnuPG.homedir')) {
             $continue = true;
-            $gpgTool = new GpgTool();
             try {
-                $gpg = $gpgTool->initializeGpg();
+                $gpg = GpgTool::initializeGpg();
             } catch (Exception $e) {
                 $this->logException("Error during initializing GPG.", $e, LOG_NOTICE);
                 $gpgStatus = 2;
@@ -5200,18 +5305,19 @@ class Server extends AppModel
 
     public function workerDiagnostics(&$workerIssueCount)
     {
+        $worker_array = array(
+            'cache' => array('ok' => false),
+            'default' => array('ok' => false),
+            'email' => array('ok' => false),
+            'prio' => array('ok' => false),
+            'update' => array('ok' => false),
+            'scheduler' => array('ok' => false)
+        );
         try {
             $this->ResqueStatus = new ResqueStatus\ResqueStatus(Resque::redis());
         } catch (Exception $e) {
             // redis connection failed
-            return array(
-                    'cache' => array('ok' => false),
-                    'default' => array('ok' => false),
-                    'email' => array('ok' => false),
-                    'prio' => array('ok' => false),
-                    'update' => array('ok' => false),
-                    'scheduler' => array('ok' => false)
-            );
+            return $worker_array;
         }
         $workers = $this->ResqueStatus->getWorkers();
         if (function_exists('posix_getpwuid')) {
@@ -5220,14 +5326,6 @@ class Server extends AppModel
         } else {
             $currentUser = trim(shell_exec('whoami'));
         }
-        $worker_array = array(
-                'cache' => array('ok' => true),
-                'default' => array('ok' => true),
-                'email' => array('ok' => true),
-                'prio' => array('ok' => true),
-                'update' => array('ok' => true),
-                'scheduler' => array('ok' => true)
-        );
         $procAccessible = file_exists('/proc');
         foreach ($workers as $pid => $worker) {
             $entry = ($worker['type'] == 'regular') ? $worker['queue'] : $worker['type'];
@@ -5245,7 +5343,13 @@ class Server extends AppModel
                 $ok = false;
                 $workerIssueCount++;
             }
-            $worker_array[$entry]['workers'][] = array('pid' => $pid, 'user' => $worker['user'], 'alive' => $alive, 'correct_user' => $correct_user, 'ok' => $ok);
+            $worker_array[$entry]['workers'][] = array(
+                'pid' => $pid,
+                'user' => $worker['user'],
+                'alive' => $alive,
+                'correct_user' => $correct_user,
+                'ok' => $ok
+            );
         }
         foreach ($worker_array as $k => $queue) {
             if (isset($worker_array[$k]['workers'])) {
@@ -5309,6 +5413,27 @@ class Server extends AppModel
         }
     }
 
+    public function killAllWorkers($user = false, $force = false)
+    {
+        $this->ResqueStatus = new ResqueStatus\ResqueStatus(Resque::redis());
+        $workers = $this->ResqueStatus->getWorkers();
+        $killed = array();
+        foreach ($workers as $pid => $worker) {
+            if (!is_numeric($pid)) {
+                continue;
+            }
+            if (substr_count(trim(shell_exec('ps -p ' . $pid)), PHP_EOL) > 0) {
+                shell_exec('kill ' . ($force ? ' -9 ' : '') . $pid . ' > /dev/null 2>&1 &');
+                $this->ResqueStatus->removeWorker($pid);
+                $this->__logRemoveWorker($user, $pid, $worker['queue'], false);
+            } else {
+                $this->ResqueStatus->removeWorker($pid);
+                $this->__logRemoveWorker($user, $pid, $worker['queue'], true);
+            }
+        }
+        return $killed;
+    }
+
     public function workerRemoveDead($user = false)
     {
         $this->ResqueStatus = new ResqueStatus\ResqueStatus(Resque::redis());

app/Model/ShadowAttribute.php

@@ -1,12 +1,13 @@
 <?php
-
 App::uses('AppModel', 'Model');
 App::uses('Folder', 'Utility');
 App::uses('File', 'Utility');
 App::uses('AttachmentTool', 'Tools');
+App::uses('ComplexTypeTool', 'Tools');
 
 /**
  * @property Event $Event
+ * @property Attribute $Attribute
  */
 class ShadowAttribute extends AppModel
 {
@@ -58,7 +59,6 @@ class ShadowAttribute extends AppModel
     // explanations of certain fields to be used in various views
     public $fieldDescriptions = array(
             'signature' => array('desc' => 'Is this attribute eligible to automatically create an IDS signature (network IDS or host IDS) out of it ?'),
-            //'private' => array('desc' => 'Prevents upload of this single Attribute to other CyDefSIG servers', 'formdesc' => 'Prevents upload of <em>this single Attribute</em> to other CyDefSIG servers.<br/>Used only when the Event is NOT set as Private')
     );
 
     // if these then a category my have upload to be zipped
@@ -122,20 +122,20 @@ class ShadowAttribute extends AppModel
         ),
         'to_ids' => array(
             'boolean' => array(
-                'rule' => array('boolean'),
+                'rule' => 'boolean',
                 'required' => false,
             ),
         ),
         'uuid' => array(
             'uuid' => array(
-                'rule' => array('custom', '/^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$/'),
-                'message' => 'Please provide a valid UUID'
+                'rule' => 'uuid',
+                'message' => 'Please provide a valid RFC 4122 UUID'
             ),
         ),
         'proposal_to_delete' => array(
-                'boolean' => array(
-                        'rule' => array('boolean'),
-                ),
+            'boolean' => array(
+                'rule' => 'boolean',
+            ),
         ),
         'first_seen' => array(
             'rule' => array('datetimeOrNull'),
@@ -152,15 +152,8 @@ class ShadowAttribute extends AppModel
     public function __construct($id = false, $table = null, $ds = null)
     {
         parent::__construct($id, $table, $ds);
-        $this->virtualFields = Set::merge($this->virtualFields, array(
-            //'distribution' => 'IF (Attribute.private=true, "Your organization only", IF (Attribute.cluster=true, "This Community-only", "All communities"))',
-            //'distribution' => 'IF (ShadowAttribute.private=true AND ShadowAttribute.cluster=false, "Your organization only", IF (ShadowAttribute.private=true AND ShadowAttribute.cluster=true, "This server-only", IF (ShadowAttribute.private=false AND ShadowAttribute.cluster=true, "This Community-only", IF (ShadowAttribute.communitie=true, "Connected communities" , "All communities"))))',
-        ));
-        $this->fieldDescriptions = Set::merge($this->fieldDescriptions, array(
-            //'distribution' => array('desc' => 'This fields indicates the intended distribution of the attribute (same as when adding an event, see Add Event)'),
-        ));
-        $this->categoryDefinitions = $this->Event->Attribute->categoryDefinitions;
-        $this->typeDefinitions = $this->Event->Attribute->typeDefinitions;
+        $this->categoryDefinitions = $this->Attribute->categoryDefinitions;
+        $this->typeDefinitions = $this->Attribute->typeDefinitions;
     }
 
     // The Associations below have been created with all possible keys, those that are not needed can be removed
@@ -211,7 +204,7 @@ class ShadowAttribute extends AppModel
         if (isset($sa['ShadowAttribute'])) {
             $sa = $sa['ShadowAttribute'];
         }
-        if (in_array($sa['type'], $this->Event->Attribute->nonCorrelatingTypes)) {
+        if (in_array($sa['type'], $this->Attribute->nonCorrelatingTypes)) {
             return;
         }
         $this->ShadowAttributeCorrelation = ClassRegistry::init('ShadowAttributeCorrelation');
@@ -222,14 +215,14 @@ class ShadowAttribute extends AppModel
             $correlatingValues[] = $sa['value2'];
         }
         foreach ($correlatingValues as $k => $cV) {
-            $correlatingAttributes[$k] = $this->Event->Attribute->find('all', array(
+            $correlatingAttributes[$k] = $this->Attribute->find('all', array(
                     'conditions' => array(
                             'AND' => array(
                                     'OR' => array(
                                             'Attribute.value1' => $cV,
                                             'Attribute.value2' => $cV
                                     ),
-                                    'Attribute.type !=' => $this->Event->Attribute->nonCorrelatingTypes,
+                                    'Attribute.type !=' => $this->Attribute->nonCorrelatingTypes,
                                     'Attribute.deleted' => 0,
                                     'Attribute.event_id !=' => $sa['event_id']
                             ),
@@ -304,9 +297,6 @@ class ShadowAttribute extends AppModel
         parent::beforeValidate();
         // remove leading and trailing blanks
         //$this->trimStringFields(); // TODO
-        if (isset($this->data['ShadowAttribute']['value'])) {
-            $this->data['ShadowAttribute']['value'] = trim($this->data['ShadowAttribute']['value']);
-        }
 
         if (!isset($this->data['ShadowAttribute']['comment'])) {
             $this->data['ShadowAttribute']['comment'] = '';
@@ -316,6 +306,18 @@ class ShadowAttribute extends AppModel
             return false;
         }
 
+        // make some changes to the inserted value
+        if (isset($this->data['ShadowAttribute']['value'])) {
+            $value = trim($this->data['ShadowAttribute']['value']);
+            $value = ComplexTypeTool::refangValue($value, $this->data['ShadowAttribute']['type']);
+            $value = $this->Attribute->modifyBeforeValidation($this->data['ShadowAttribute']['type'], $value);
+            $this->data['ShadowAttribute']['value'] = $value;
+        }
+
+        if (!isset($this->data['ShadowAttribute']['org'])) {
+            $this->data['ShadowAttribute']['org'] = '';
+        }
+
         if (empty($this->data['ShadowAttribute']['timestamp'])) {
             $date = new DateTime();
             $this->data['ShadowAttribute']['timestamp'] = $date->getTimestamp();
@@ -325,16 +327,15 @@ class ShadowAttribute extends AppModel
             $this->data['ShadowAttribute']['proposal_to_delete'] = 0;
         }
 
-        // make some last changes to the inserted value
-        $this->data['ShadowAttribute']['value'] = $this->Event->Attribute->modifyBeforeValidation($this->data['ShadowAttribute']['type'], $this->data['ShadowAttribute']['value']);
-
         // generate UUID if it doesn't exist
         if (empty($this->data['ShadowAttribute']['uuid'])) {
             $this->data['ShadowAttribute']['uuid'] = CakeText::uuid();
+        } else {
+            $this->data['ShadowAttribute']['uuid'] = strtolower($this->data['ShadowAttribute']['uuid']);
         }
 
         if (!empty($this->data['ShadowAttribute']['type']) && empty($this->data['ShadowAttribute']['category'])) {
-            $this->data['ShadowAttribute']['category'] = $this->Event->Attribute->typeDefinitions[$this->data['ShadowAttribute']['type']]['default_category'];
+            $this->data['ShadowAttribute']['category'] = $this->Attribute->typeDefinitions[$this->data['ShadowAttribute']['type']]['default_category'];
         }
 
         // always return true, otherwise the object cannot be saved
@@ -360,44 +361,43 @@ class ShadowAttribute extends AppModel
 
     public function validCategory($fields)
     {
-        return $this->Event->Attribute->validCategory($fields);
+        return $this->Attribute->validCategory($fields);
     }
 
     public function validateAttributeValue($fields)
     {
         $value = $fields['value'];
-        return $this->Event->Attribute->runValidation($value, $this->data['ShadowAttribute']['type']);
+        return $this->Attribute->runValidation($value, $this->data['ShadowAttribute']['type']);
     }
 
     public function getCompositeTypes()
     {
-        // build the list of composite Attribute.type dynamically by checking if type contains a |
-        // default composite types
-        $compositeTypes = array('malware-sample');  // TODO hardcoded composite
-        // dynamically generated list
-        foreach (array_keys($this->typeDefinitions) as $type) {
-            $pieces = explode('|', $type);
-            if (2 == count($pieces)) {
-                $compositeTypes[] = $type;
-            }
-        }
-        return $compositeTypes;
+        return $this->Attribute->getCompositeTypes();
     }
 
     public function typeIsMalware($type)
     {
-        return in_array($type, $this->zippedDefinitions);
+        return $this->Attribute->typeIsAttachment($type);
     }
 
     public function typeIsAttachment($type)
     {
-        return in_array($type, $this->zippedDefinitions) || in_array($type, $this->uploadDefinitions);
+        return $this->Attribute->typeIsAttachment($type);
     }
 
-    public function base64EncodeAttachment($attribute)
+    public function base64EncodeAttachment(array $attribute)
     {
-        $content = $this->loadAttachmentTool()->getShadowContent($attribute['event_id'], $attribute['id']);
-        return base64_encode($content);
+        try {
+            return base64_encode($this->getAttachment($attribute));
+        } catch (NotFoundException $e) {
+            $this->log($e->getMessage(), LOG_NOTICE);
+            return '';
+        }
+    }
+
+    public function getAttachment($attribute, $path_suffix='')
+    {
+        return $this->loadAttachmentTool()->getShadowContent($attribute['event_id'], $attribute['id'], $path_suffix);
     }
 
     public function saveBase64EncodedAttachment($attribute)
@@ -434,15 +434,7 @@ class ShadowAttribute extends AppModel
     // check whether the variable is null or datetime
     public function datetimeOrNull($fields)
     {
-        $k = array_keys($fields)[0];
-        $seen = $fields[$k];
-        try {
-            new DateTime($seen);
-            $returnValue = true;
-        } catch (Exception $e) {
-            $returnValue = false;
-        }
-        return $returnValue || is_null($seen);
+        return $this->Attribute->datetimeOrNull($fields);
     }
 
     public function setDeleted($id)
@@ -479,22 +471,29 @@ class ShadowAttribute extends AppModel
         }
     }
 
-    public function getEventContributors($id)
+    /**
+     * @param int $eventId
+     * @return array Key is organisation ID, value is an organisation name
+     */
+    public function getEventContributors($eventId)
     {
-        $orgs = $this->find('all', array('fields' => array(
-            'DISTINCT(ShadowAttribute.org_id)'),
-            'conditions' => array('event_id' => $id),
+        $orgs = $this->find('all', array(
+            'fields' => array('DISTINCT(ShadowAttribute.org_id)'),
+            'conditions' => array('event_id' => $eventId),
             'recursive' => -1,
             'order' => false
         ));
-        $org_ids = array();
-        $this->Organisation = ClassRegistry::init('Organisation');
-        foreach ($orgs as $org) {
-            $org_ids[] = $this->Organisation->find('first', array('recursive' => -1, 'fields' => array('id', 'name'), 'conditions' => array('Organisation.id' => $org['ShadowAttribute']['org_id'])));
+        if (empty($orgs)) {
+            return [];
         }
-        return $org_ids;
-    }
 
+        $this->Organisation = ClassRegistry::init('Organisation');
+        return $this->Organisation->find('list', array(
+            'recursive' => -1,
+            'fields' => array('id', 'name'),
+            'conditions' => array('Organisation.id' => Hash::extract($orgs, "{n}.ShadowAttribute.org_id")))
+        );
+    }
 
     /**
      * Sends an email to members of the organization that owns the event
@@ -699,7 +698,7 @@ class ShadowAttribute extends AppModel
                     'OR' => array(
                         'Event.org_id' => $user['org_id'],
                         ['AND' => [
-                            'Event.distribution' => array(1,2,3,5),
+                            'Event.distribution' => array(1,2,3),
                             $unpublishedPrivate ? ['Event.published' => 1] : [],
                         ]],
                         ['AND' => [

app/Model/SharingGroup.php

@@ -24,8 +24,8 @@ class SharingGroup extends AppModel
         ),
         'uuid' => array(
             'uuid' => array(
-                'rule' => array('custom', '/^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$/'),
-                'message' => 'Please provide a valid UUID'
+                'rule' => 'uuid',
+                'message' => 'Please provide a valid RFC 4122 UUID'
             ),
         )
     );
@@ -65,6 +65,8 @@ class SharingGroup extends AppModel
         parent::beforeValidate();
         if (empty($this->data['SharingGroup']['uuid'])) {
             $this->data['SharingGroup']['uuid'] = CakeText::uuid();
+        } else {
+            $this->data['SharingGroup']['uuid'] = strtolower($this->data['SharingGroup']['uuid']);
         }
         $date = date('Y-m-d H:i:s');
         if (empty($this->data['SharingGroup']['created'])) {

app/Model/Sighting.php

@@ -5,6 +5,7 @@ App::uses('TmpFileTool', 'Tools');
 
 /**
  * @property Attribute $Attribute
+ * @property Event $Event
  */
 class Sighting extends AppModel
 {
@@ -21,6 +22,7 @@ class Sighting extends AppModel
         'attribute_id' => 'numeric',
         'org_id' => 'numeric',
         'date_sighting' => 'numeric',
+        'uuid' => 'uuid',
         'type' => array(
             'rule' => array('inList', array(0, 1, 2)),
             'message' => 'Invalid type. Valid options are: 0 (Sighting), 1 (False-positive), 2 (Expiration).'
@@ -51,12 +53,13 @@ class Sighting extends AppModel
     public function beforeValidate($options = array())
     {
         parent::beforeValidate();
-        $date = date('Y-m-d H:i:s');
         if (empty($this->data['Sighting']['id']) && empty($this->data['Sighting']['date_sighting'])) {
-            $this->data['Sighting']['date_sighting'] = $date;
+            $this->data['Sighting']['date_sighting'] = date('Y-m-d H:i:s');
         }
         if (empty($this->data['Sighting']['uuid'])) {
             $this->data['Sighting']['uuid'] = CakeText::uuid();
+        } else {
+            $this->data['Sighting']['uuid'] = strtolower($this->data['Sighting']['uuid']);
         }
         return true;
     }
@@ -207,20 +210,18 @@ class Sighting extends AppModel
     /**
      * @param array $event
      * @param array $user
-     * @param array|int|null $attribute Attribute array or attribute ID
-     * @param bool $extraConditions
+     * @param array|int|null $attribute Attribute model or attribute ID
+     * @param array|bool $extraConditions
+     * @param bool $forSync
      * @return array|int
      */
-    public function attachToEvent(array $event, array $user, $attribute = null, $extraConditions = false)
+    public function attachToEvent(array $event, array $user, $attribute = null, $extraConditions = false, $forSync = false)
     {
-        $ownEvent = false;
-        if ($user['Role']['perm_site_admin'] || $event['Event']['org_id'] == $user['org_id']) {
-            $ownEvent = true;
-        }
+        $ownEvent = $user['Role']['perm_site_admin'] || $event['Event']['org_id'] == $user['org_id'];
 
         $contain = [];
         $conditions = array('Sighting.event_id' => $event['Event']['id']);
-        if (is_array($attribute)) {
+        if (isset($attribute['Attribute']['id'])) {
             $conditions['Sighting.attribute_id'] = $attribute['Attribute']['id'];
         } elseif (is_numeric($attribute)) {
             $conditions['Sighting.attribute_id'] = $attribute;
@@ -247,112 +248,68 @@ class Sighting extends AppModel
         // If the event has any sightings for the user's org, then the user is a sighting reporter for the event too.
         // This means that he /she has access to the sightings data contained within
         if (!$ownEvent && Configure::read('Plugin.Sightings_policy') == 1) {
-            $temp = $this->find('first', array('recursive' => -1, 'conditions' => array('Sighting.event_id' => $event['Event']['id'], 'Sighting.org_id' => $user['org_id'])));
+            $temp = $this->find('first', array(
+                'recursive' => -1,
+                'conditions' => array(
+                    'Sighting.event_id' => $event['Event']['id'],
+                    'Sighting.org_id' => $user['org_id'],
+                )
+            ));
             if (empty($temp)) {
                 return array();
             }
         }
 
         $sightings = $this->find('all', array(
-                'conditions' => $conditions,
-                'recursive' => -1,
-                'contain' => $contain,
+            'conditions' => $conditions,
+            'recursive' => -1,
+            'contain' => $contain,
         ));
         if (empty($sightings)) {
             return array();
         }
         $anonymise = Configure::read('Plugin.Sightings_anonymise');
+        $anonymiseAs = Configure::read('Plugin.Sightings_anonymise_as');
+        $anonOrg = null;
+        if ($forSync && !empty($anonymiseAs)) {
+            $this->Organisation = ClassRegistry::init('Organisation');
+            $anonOrg = $this->Organisation->find('first', [
+                'recursive' => -1,
+                'conditions' => ['Organisation.id' => $anonymiseAs],
+                'fields' => ['Organisation.id', 'Organisation.uuid', 'Organisation.name']
+            ]);
+        }
         foreach ($sightings as $k => $sighting) {
             if (
                 ($sighting['Sighting']['org_id'] == 0 && !empty($sighting['Organisation'])) ||
-                $anonymise
+                $anonymise || !empty($anonOrg)
             ) {
                 if ($sighting['Sighting']['org_id'] != $user['org_id']) {
-                    unset($sightings[$k]['Sighting']['org_id']);
-                    unset($sightings[$k]['Organisation']);
+                    if (empty($anonOrg)) {
+                        unset($sighting['Sighting']['org_id']);
+                        unset($sighting['Organisation']);
+                    } else {
+                        $sighting['Sighting']['org_id'] = $anonOrg['Organisation']['id'];
+                        $sighting['Organisation'] = $anonOrg['Organisation'];
+                    }
                 }
             }
             // rearrange it to match the event format of fetchevent
-            if (isset($sightings[$k]['Organisation'])) {
-                $sightings[$k]['Sighting']['Organisation'] = $sightings[$k]['Organisation'];
+            if (isset($sighting['Organisation'])) {
+                $sighting['Sighting']['Organisation'] = $sighting['Organisation'];
             }
             // zeroq: add attribute UUID to sighting to make synchronization easier
             if (isset($sighting['Attribute']['uuid'])) {
-                $sightings[$k]['Sighting']['attribute_uuid'] = $sighting['Attribute']['uuid'];
+                $sighting['Sighting']['attribute_uuid'] = $sighting['Attribute']['uuid'];
             } else {
-                $sightings[$k]['Sighting']['attribute_uuid'] = $attribute['Attribute']['uuid'];
+                $sighting['Sighting']['attribute_uuid'] = $attribute['Attribute']['uuid'];
             }
 
-            $sightings[$k] = $sightings[$k]['Sighting'] ;
+            $sightings[$k] = $sighting['Sighting'] ;
         }
         return $sightings;
     }
 
-    /*
-     * Loop through all attributes of an event, including those in objects
-     * and pass each value to SightingDB. If there's a hit, append the data
-     * directly to the attributes
-     */
-    public function attachSightingDB($event, $user)
-    {
-        if (!empty(Configure::read('Plugin.Sightings_sighting_db_enable'))) {
-            $host = empty(Configure::read('Plugin.Sightings_sighting_db_host')) ? 'localhost' : Configure::read('Plugin.Sightings_sighting_db_host');
-            $port = empty(Configure::read('Plugin.Sightings_sighting_db_port')) ? 9999 : Configure::read('Plugin.Sightings_sighting_db_port');
-            App::uses('SyncTool', 'Tools');
-            $syncTool = new SyncTool();
-            $params = array(
-                'ssl_verify_peer' => false,
-                'ssl_verify_peer_name' => false,
-                'ssl_verify_host' => false
-            );
-            $HttpSocket = $syncTool->createHttpSocket($params);
-            if (!empty($event['Attribute'])) {
-                $event['Attribute'] = $this->__attachSightingDBToAttribute($event['Attribute'], $user, $host, $port, $HttpSocket);
-            }
-            if (!empty($event['Object'])) {
-                foreach ($event['Object'] as &$object) {
-                    if (!empty($object['Attribute'])) {
-                        $object['Attribute'] = $this->__attachSightingDBToAttribute($object['Attribute'], $user, $host, $port, $HttpSocket);
-                    }
-                }
-            }
-        }
-        return $event;
-    }
-
-    private function __attachSightingDBToAttribute($attributes, $user, $host, $port, $HttpSocket = false)
-    {
-        if (empty($HttpSocket)) {
-            App::uses('SyncTool', 'Tools');
-            $syncTool = new SyncTool();
-            $params = array(
-                'ssl_allow_self_signed' => true,
-                'ssl_verify_peer' => false
-            );
-            $HttpSocket = $syncTool->createHttpSocket($params);
-        }
-        foreach($attributes as &$attribute) {
-            $response = $HttpSocket->get(
-                sprintf(
-                    '%s:%s/r/all/%s?val=%s',
-                    $host,
-                    $port,
-                    $attribute['type'],
-                    rtrim(str_replace('/', '_', str_replace('+', '-', base64_encode($attribute['value']))), '=')
-                )
-            );
-            if ($response->code == 200) {
-                $responseData = json_decode($response->body, true);
-                if ($responseData !== false) {
-                    if (!isset($responseData['error'])) {
-                        $attribute['SightingDB'] = $responseData;
-                    }
-                }
-            }
-        }
-        return $attributes;
-    }
-
     public function saveSightings($id, $values, $timestamp, $user, $type = false, $source = false, $sighting_uuid = false, $publish = false, $saveOnBehalfOf = false)
     {
         if (!in_array($type, array(0, 1, 2))) {
@@ -402,16 +359,20 @@ class Sighting extends AppModel
         foreach ($attributes as $attribute) {
             if ($type === '2') {
                 // remove existing expiration by the same org if it exists
-                $this->deleteAll(array('Sighting.org_id' => $user['org_id'], 'Sighting.type' => $type, 'Sighting.attribute_id' => $attribute['Attribute']['id']));
+                $this->deleteAll(array(
+                    'Sighting.org_id' => $user['org_id'],
+                    'Sighting.type' => $type,
+                    'Sighting.attribute_id' => $attribute['Attribute']['id'],
+                ));
             }
             $this->create();
             $sighting = array(
-                    'attribute_id' => $attribute['Attribute']['id'],
-                    'event_id' => $attribute['Attribute']['event_id'],
-                    'org_id' => ($saveOnBehalfOf === false) ? $user['org_id'] : $saveOnBehalfOf,
-                    'date_sighting' => $timestamp,
-                    'type' => $type,
-                    'source' => $source
+                'attribute_id' => $attribute['Attribute']['id'],
+                'event_id' => $attribute['Attribute']['event_id'],
+                'org_id' => ($saveOnBehalfOf === false) ? $user['org_id'] : $saveOnBehalfOf,
+                'date_sighting' => $timestamp,
+                'type' => $type,
+                'source' => $source,
             );
             // zeroq: allow setting a specific uuid
             if ($sighting_uuid) {
@@ -456,11 +417,6 @@ class Sighting extends AppModel
         return $result;
     }
 
-    public function generateRandomFileName()
-    {
-        return (new RandomTool())->random_str(false, 12);
-    }
-
     public function addUuids()
     {
         $sightings = $this->find('all', array(
@@ -523,11 +479,18 @@ class Sighting extends AppModel
         return $sightingsRearranged;
     }
 
-    public function getSightingsForObjectIds($user, $tagList, $context = 'event', $type = '0')
+    /**
+     * @param $user - not used
+     * @param array $tagIds
+     * @param string $context 'event' or 'attribute'
+     * @param string|false $type
+     * @return array
+     */
+    public function getSightingsForObjectIds($user, array $tagIds, $context = 'event', $type = '0')
     {
         $conditions = array(
             'Sighting.date_sighting >' => $this->getMaximumRange(),
-            ucfirst($context) . 'Tag.tag_id' => $tagList
+            ucfirst($context) . 'Tag.tag_id' => $tagIds
         );
         if ($type !== false) {
             $conditions['Sighting.type'] = $type;
@@ -784,10 +747,7 @@ class Sighting extends AppModel
      */
     public function bulkSaveSightings($eventId, $sightings, $user, $passAlong = null)
     {
-        $event = $this->Event->fetchEvent($user, array_merge(array(
-             'metadata' => 1,
-             'flatten' => true,
-        ), is_string($eventId) ? array('event_uuid' => $eventId) : array('eventid' => $eventId)));
+        $event = $this->Event->fetchSimpleEvent($user, $eventId);
         if (empty($event)) {
             return 'Event not found or not accessible by this user.';
         }
@@ -809,7 +769,7 @@ class Sighting extends AppModel
             }
         }
         if ($saved > 0) {
-            $this->Event->publishRouter($eventId, $passAlong, $user, 'sightings');
+            $this->Event->publishRouter($event['Event']['id'], $passAlong, $user, 'sightings');
         }
         return $saved;
     }

app/Model/Tag.php

@@ -1,10 +1,6 @@
 <?php
 App::uses('AppModel', 'Model');
 
-/**
- * @property EventTag $EventTag
- * @property AttributeTag $AttributeTag
- */
 class Tag extends AppModel
 {
     public $useTable = 'tags';

app/Model/TagCollection.php

@@ -31,7 +31,7 @@ class TagCollection extends AppModel
         )
     );
 
-    public $whitelistedItems = false;
+    public $allowedlistedItems = false;
 
     public $validate = array(
         'name' => array(
@@ -42,7 +42,13 @@ class TagCollection extends AppModel
                     'rule' => 'isUnique',
                     'message' => 'A similar name already exists.',
             ),
-        )
+        ),
+        'uuid' => array(
+            'uuid' => array(
+                'rule' => 'uuid',
+                'message' => 'Please provide a valid RFC 4122 UUID'
+            ),
+        ),
     );
 
     public function beforeValidate($options = array())
@@ -51,6 +57,8 @@ class TagCollection extends AppModel
         // generate UUID if it doesn't exist
         if (empty($this->data['TagCollection']['uuid'])) {
             $this->data['TagCollection']['uuid'] = CakeText::uuid();
+        } else {
+            $this->data['TagCollection']['uuid'] = strtolower($this->data['TagCollection']['uuid']);
         }
         return true;
     }

app/Model/User.php

@@ -223,7 +223,7 @@ class User extends AppModel
         'Containable'
     );
 
-    /** @var Crypt_GPG|null|false */
+    /** @var CryptGpgExtended|null|false */
     private $gpg;
 
     public function beforeValidate($options = array())
@@ -303,9 +303,11 @@ class User extends AppModel
         return true;
     }
 
-    // Checks if the GnuPG key is a valid key, but also import it in the keychain.
-    // this will NOT fail on keys that can only be used for signing but not encryption!
-    // the method in verifyUsers will fail in that case.
+    /**
+     * Checks if the GnuPG key is a valid key.
+     * @param array $check
+     * @return bool
+     */
     public function validateGpgkey($check)
     {
         // LATER first remove the old gpgkey from the keychain
@@ -320,12 +322,11 @@ class User extends AppModel
             return true;
         }
         try {
-            $keyImportOutput = $gpg->importKey($check['gpgkey']);
-            if (!empty($keyImportOutput['fingerprint'])) {
-                return true;
-            }
+            $gpgTool = new GpgTool($gpg);
+            $gpgTool->validateGpgKey($check['gpgkey']);
+            return true;
         } catch (Exception $e) {
-            $this->logException("Exception during importing GPG key", $e);
+            $this->logException("Exception during validating GPG key", $e, LOG_NOTICE);
             return false;
         }
     }
@@ -453,25 +454,40 @@ class User extends AppModel
                 )));
     }
 
-    public function verifySingleGPG($user, $gpg = null)
+    /**
+     * 0 - true if key is valid
+     * 1 - User e-mail
+     * 2 - Error message
+     * 3 - Not used
+     * 4 - Key fingerprint
+     * 5 - Key fingerprint
+     * @param array $user
+     * @return array
+     */
+    public function verifySingleGPG(array $user)
     {
-        if ($gpg === null) {
-            $gpg = $this->initializeGpg();
-            if (!$gpg) {
-                $result[2] = 'GnuPG is not configured on this system.';
-                $result[0] = true;
-                return $result;
-            }
+        $result = [0 => false, 1 => $user['User']['email']];
+
+        $gpg = $this->initializeGpg();
+        if (!$gpg) {
+            $result[2] = 'GnuPG is not configured on this system.';
+            return $result;
         }
-        $result = array();
+
         try {
             $currentTimestamp = time();
-            $temp = $gpg->importKey($user['User']['gpgkey']);
-            $key = $gpg->getKeys($temp['fingerprint']);
-            $result[5] = $temp['fingerprint'];
-            $subKeys = $key[0]->getSubKeys();
-            $sortedKeys = array('valid' => 0, 'expired' => 0, 'noEncrypt' => 0);
-            foreach ($subKeys as $subKey) {
+            $keys = $gpg->keyInfo($user['User']['gpgkey']);
+            if (count($keys) !== 1) {
+                $result[2] = 'Multiple or no key found';
+                return $result;
+            }
+
+            $key = $keys[0];
+            $result[4] = $key->getPrimaryKey()->getFingerprint();
+            $result[5] = $result[4];
+
+            $sortedKeys = ['valid' => 0, 'expired' => 0, 'noEncrypt' => 0];
+            foreach ($key->getSubKeys() as $subKey) {
                 $expiration = $subKey->getExpirationDate();
                 if ($expiration != 0 && $currentTimestamp > $expiration) {
                     $sortedKeys['expired']++;
@@ -491,14 +507,12 @@ class User extends AppModel
                 if ($sortedKeys['noEncrypt']) {
                     $result[2] .= ' Found ' . $sortedKeys['noEncrypt'] . ' subkey(s) that are sign only.';
                 }
+            } else {
                 $result[0] = true;
             }
         } catch (Exception $e) {
             $result[2] = $e->getMessage();
-            $result[0] = true;
         }
-        $result[1] = $user['User']['email'];
-        $result[4] = $temp['fingerprint'];
         return $result;
     }
 
@@ -522,7 +536,7 @@ class User extends AppModel
         }
         $results = [];
         foreach ($users as $k => $user) {
-            $results[$user['User']['id']] = $this->verifySingleGPG($user, $gpg);
+            $results[$user['User']['id']] = $this->verifySingleGPG($user);
         }
         return $results;
     }
@@ -585,36 +599,38 @@ class User extends AppModel
         return $user;
     }
 
-    // get the current user and rearrange it to be in the same format as in the auth component
-    public function getAuthUser($id)
+    /**
+     * @param int $id
+     * @return array|null
+     */
+    public function getUserById($id)
     {
         if (empty($id)) {
             throw new NotFoundException('Invalid user ID.');
         }
-        $conditions = array('User.id' => $id);
-        $user = $this->find(
+        return $this->find(
             'first',
             array(
-                'conditions' => $conditions,
+                'conditions' => array('User.id' => $id),
                 'recursive' => -1,
                 'contain' => array(
                     'Organisation',
                     'Role',
                     'Server',
-                    'UserSetting'
+                    'UserSetting',
                 )
             )
         );
+    }
+
+    // get the current user and rearrange it to be in the same format as in the auth component
+    public function getAuthUser($id)
+    {
+        $user = $this->getUserById($id);
         if (empty($user)) {
             return $user;
         }
-        // Rearrange it a bit to match the Auth object created during the login
-        $user['User']['Role'] = $user['Role'];
-        $user['User']['Organisation'] = $user['Organisation'];
-        $user['User']['Server'] = $user['Server'];
-        $user['User']['UserSetting'] = $user['UserSetting'];
-        unset($user['Organisation'], $user['Role'], $user['Server']);
-        return $user['User'];
+        return $this->rearrangeToAuthForm($user);
     }
 
     // get the current user and rearrange it to be in the same format as in the auth component
@@ -625,11 +641,7 @@ class User extends AppModel
         if (empty($user)) {
             return $user;
         }
-        // Rearrange it a bit to match the Auth object created during the login
-        $user['User']['Role'] = $user['Role'];
-        $user['User']['Organisation'] = $user['Organisation'];
-        $user['User']['Server'] = $user['Server'];
-        return $user['User'];
+        return $this->rearrangeToAuthForm($user);
     }
 
     public function getAuthUserByExternalAuth($auth_key)
@@ -650,11 +662,28 @@ class User extends AppModel
         if (empty($user)) {
             return $user;
         }
-        // Rearrange it a bit to match the Auth object created during the login
+        return $this->rearrangeToAuthForm($user);
+    }
+
+    /**
+     * User model is a mess. Sometimes it is necessary to convert User model to form that is created during the login
+     * process. This method do that work for you.
+     *
+     * @param array $user
+     * @return array
+     */
+    public function rearrangeToAuthForm(array $user)
+    {
+        if (!isset($user['User'])) {
+            throw new InvalidArgumentException('Invalid user model provided.');
+        }
+
         $user['User']['Role'] = $user['Role'];
         $user['User']['Organisation'] = $user['Organisation'];
         $user['User']['Server'] = $user['Server'];
-        unset($user['Organisation'], $user['Role'], $user['Server']);
+        if (isset($user['UserSetting'])) {
+            $user['User']['UserSetting'] = $user['UserSetting'];
+        }
         return $user['User'];
     }
 
@@ -703,7 +732,7 @@ class User extends AppModel
             'conditions' => $conditions,
             'recursive' => -1,
             'fields' => array('id', 'email', 'gpgkey', 'certif_public', 'org_id', 'disabled'),
-            'contain' => ['Role' => ['fields' => ['perm_site_admin']], 'Organisation' => ['fields' => ['id']]],
+            'contain' => ['Role' => ['fields' => ['perm_site_admin', 'perm_audit']], 'Organisation' => ['fields' => ['id']]],
         ));
         foreach ($users as $k => $user) {
             $user = $user['User'];
@@ -726,10 +755,21 @@ class User extends AppModel
         $sendEmail->sendExternal($params);
     }
 
-    // all e-mail sending is now handled by this method
-    // Just pass the user ID in an array that is the target of the e-mail along with the message body and the alternate message body if the message cannot be encrypted
-    // the remaining two parameters are the e-mail subject and a secondary user object which will be used as the replyto address if set. If it is set and an encryption key for the replyTo user exists, then his/her public key will also be attached
-    public function sendEmail($user, $body, $bodyNoEnc = false, $subject, $replyToUser = false)
+    /**
+     * All e-mail sending is now handled by this method
+     * Just pass the user array that is the target of the e-mail along with the message body and the alternate message body if the message cannot be encrypted
+     * the remaining two parameters are the e-mail subject and a secondary user object which will be used as the replyto address if set. If it is set and an encryption key for the replyTo user exists, then his/her public key will also be attached
+     *
+     * @param array $user
+     * @param string $body
+     * @param string|false $bodyNoEnc
+     * @param string $subject
+     * @param array|false $replyToUser
+     * @return bool
+     * @throws Crypt_GPG_BadPassphraseException
+     * @throws Crypt_GPG_Exception
+     */
+    public function sendEmail(array $user, $body, $bodyNoEnc = false, $subject, $replyToUser = false)
     {
         if ($user['User']['disabled']) {
             return true;
@@ -792,7 +832,7 @@ class User extends AppModel
      */
     public function searchGpgKey($email)
     {
-        $gpgTool = new GpgTool();
+        $gpgTool = new GpgTool(null);
         return $gpgTool->searchGpgKey($email);
     }
 
@@ -803,7 +843,7 @@ class User extends AppModel
      */
     public function fetchGpgKey($fingerprint)
     {
-        $gpgTool = new GpgTool();
+        $gpgTool = new GpgTool($this->initializeGpg());
         return $gpgTool->fetchGpgKey($fingerprint);
     }
 
@@ -1288,7 +1328,7 @@ class User extends AppModel
     /**
      * Initialize GPG. Returns `null` if initialization failed.
      *
-     * @return null|Crypt_GPG
+     * @return null|CryptGpgExtended
      */
     private function initializeGpg()
     {
@@ -1301,8 +1341,7 @@ class User extends AppModel
         }
 
         try {
-            $gpgTool = new GpgTool();
-            $this->gpg = $gpgTool->initializeGpg();
+            $this->gpg = GpgTool::initializeGpg();
             return $this->gpg;
         } catch (Exception $e) {
             $this->logException("GPG couldn't be initialized, GPG encryption and signing will be not available.", $e, LOG_NOTICE);

app/Model/Warninglist.php

@@ -1,5 +1,10 @@
 <?php
 App::uses('AppModel', 'Model');
+
+/**
+ * @property WarninglistType $WarninglistType
+ * @property WarninglistEntry $WarninglistEntry
+ */
 class Warninglist extends AppModel
 {
     public $useTable = 'warninglists';
@@ -23,27 +28,138 @@ class Warninglist extends AppModel
     );
 
     public $hasMany = array(
-            'WarninglistEntry' => array(
-                'dependent' => true
-            ),
-            'WarninglistType' => array(
-                'dependent' => true
-            )
+        'WarninglistEntry' => array(
+            'dependent' => true
+        ),
+        'WarninglistType' => array(
+            'dependent' => true
+        )
     );
 
     private $__tlds = array(
         'TLDs as known by IANA'
     );
 
-    public function beforeValidate($options = array())
+    /** @var array */
+    private $entriesCache = [];
+
+    /** @var array|null */
+    private $enabledCache = null;
+
+    private $showForAll;
+
+    public function __construct($id = false, $table = null, $ds = null)
     {
-        parent::beforeValidate();
-        return true;
+        parent::__construct($id, $table, $ds);
+        $this->showForAll = Configure::read('MISP.warning_for_all');
     }
 
-    public function getTldLists()
+    /**
+     * Attach warninglist matches to attributes or proposals with IDS mark.
+     *
+     * @param array $attributes
+     * @return array Warninglist ID => name
+     */
+    public function attachWarninglistToAttributes(array &$attributes)
     {
-        return $this->__tlds;
+        if (empty($attributes)) {
+            return [];
+        }
+
+        $enabledWarninglists = $this->getEnabled();
+        if (empty($enabledWarninglists)) {
+            return []; // no warninglist is enabled
+        }
+
+        try {
+            $redis = $this->setupRedisWithException();
+        } catch (Exception $e) {
+            // fallback to default implementation when redis is not available
+            $eventWarnings = [];
+            foreach ($attributes as $pos => $attribute) {
+                $attributes[$pos] = $this->checkForWarning($attribute, $enabledWarninglists);
+                if (isset($attributes[$pos]['warnings'])) {
+                    foreach ($attribute['warnings'] as $match) {
+                        $eventWarnings[$match['warninglist_id']] = $match['warninglist_name'];
+                    }
+                }
+            }
+            return $eventWarnings;
+        }
+
+        $warninglistIdToName = [];
+        $enabledTypes = [];
+        foreach ($enabledWarninglists as $warninglist) {
+            $warninglistIdToName[$warninglist['Warninglist']['id']] = $warninglist['Warninglist']['name'];
+            foreach ($warninglist['types'] as $type) {
+                $enabledTypes[$type] = true;
+            }
+        }
+
+        $redisResultToAttributePos = [];
+        $keysToGet = [];
+        foreach ($attributes as $pos => $attribute) {
+            if (($attribute['to_ids'] || $this->showForAll) && (isset($enabledTypes[$attribute['type']]) || isset($enabledTypes['ALL']))) {
+                $redisResultToAttributePos[] = $pos;
+                // Use hash as binary string to save memory and CPU time
+                // Hash contains just attribute type and value, so can be reused in another event attributes
+                $keysToGet[] = 'misp:wlc:' . md5($attribute['type'] . ':' . $attribute['value'], true);
+            }
+        }
+
+        if (empty($keysToGet)) {
+            return []; // no attribute suitable for warninglist check
+        }
+
+        $eventWarnings = [];
+        $saveToCache = [];
+        foreach ($redis->mget($keysToGet) as $pos => $result) {
+            if ($result === false) { // not in cache
+                $attribute = $attributes[$redisResultToAttributePos[$pos]];
+                $attribute = $this->checkForWarning($attribute, $enabledWarninglists);
+
+                $store = [];
+                if (isset($attribute['warnings'])) {
+                    foreach ($attribute['warnings'] as $match) {
+                        $warninglistId = $match['warninglist_id'];
+                        $attributes[$redisResultToAttributePos[$pos]]['warnings'][] = [
+                            'value' => $match['value'],
+                            'match' => $match['match'],
+                            'warninglist_id' => $warninglistId,
+                            'warninglist_name' => $warninglistIdToName[$warninglistId],
+                        ];
+                        $eventWarnings[$warninglistId] = $warninglistIdToName[$warninglistId];
+
+                        $store[$warninglistId] = [$match['value'], $match['match']];
+                    }
+                }
+
+                $attributeKey = $keysToGet[$pos];
+                $saveToCache[$attributeKey] = empty($store) ? '' : json_encode($store);
+
+            } elseif (!empty($result)) { // skip empty string that means no warning list match
+                $matchedWarningList = json_decode($result, true);
+                foreach ($matchedWarningList as $warninglistId => $matched) {
+                    $attributes[$redisResultToAttributePos[$pos]]['warnings'][] = [
+                        'value' => $matched[0],
+                        'match' => $matched[1],
+                        'warninglist_id' => $warninglistId,
+                        'warninglist_name' => $warninglistIdToName[$warninglistId],
+                    ];
+                    $eventWarnings[$warninglistId] = $warninglistIdToName[$warninglistId];
+                }
+            }
+        }
+
+        if (!empty($saveToCache)) {
+            $pipe = $redis->multi(Redis::PIPELINE);
+            foreach ($saveToCache as $attributeKey => $json) {
+                $redis->setex($attributeKey, 3600, $json); // cache for one hour
+            }
+            $pipe->exec();
+        }
+
+        return $eventWarnings;
     }
 
     public function update()
@@ -52,8 +168,9 @@ class Warninglist extends AppModel
         $updated = array('success' => [], 'fails' => []);
         foreach ($directories as $dir) {
             $file = new File($dir . DS . 'list.json');
-            $list = json_decode($file->read(), true);
+            $list = $this->jsonDecode($file->read());
             $file->close();
+
             if (!isset($list['version'])) {
                 $list['version'] = 1;
             }
@@ -63,9 +180,9 @@ class Warninglist extends AppModel
                 $list['type'] = $list['type'][0];
             }
             $current = $this->find('first', array(
-                    'conditions' => array('name' => $list['name']),
-                    'recursive' => -1,
-                    'fields' => array('*')
+                'conditions' => array('name' => $list['name']),
+                'recursive' => -1,
+                'fields' => array('*')
             ));
             if (empty($current) || $list['version'] > $current['Warninglist']['version']) {
                 $result = $this->__updateList($list, $current);
@@ -99,7 +216,7 @@ class Warninglist extends AppModel
         return $result;
     }
 
-    private function __updateList($list, $current)
+    private function __updateList(array $list, array $current)
     {
         $list['enabled'] = 0;
         $warninglist = array();
@@ -149,31 +266,53 @@ class Warninglist extends AppModel
         }
     }
 
-    // regenerate the warninglist caches, but if an ID is passed along, only regen the entries for the given ID.
-    // This allows us to enable/disable a single warninglist without regenerating all caches
-    public function regenerateWarninglistCaches($id = false)
+    /**
+     * Regenerate the warninglist caches, but if an ID is passed along, only regen the entries for the given ID.
+     * This allows us to enable/disable a single warninglist without regenerating all caches.
+     * @param int|null $id
+     * @return bool
+     */
+    public function regenerateWarninglistCaches($id = null)
     {
         $redis = $this->setupRedis();
         if ($redis === false) {
             return false;
         }
-        $warninglists = $this->find('all', array('contain' => array('WarninglistType'), 'conditions' => array('enabled' => 1)));
+
+        if (method_exists($redis, 'unlink')) {
+            // Delete attributes cache non blocking way if available
+            $redis->unlink($redis->keys('misp:wlc:*'));
+        } else {
+            $redis->del($redis->keys('misp:wlc:*'));
+        }
+
+        if ($id === null) {
+            // delete all cached entries when regenerating whole cache
+            $redis->del($redis->keys('misp:warninglist_entries_cache:*'));
+        }
+
+        $warninglists = $this->find('all', array(
+            'contain' => array('WarninglistType'),
+            'conditions' => array('enabled' => 1),
+            'fields' => ['id', 'name', 'type'],
+        ));
         $this->cacheWarninglists($warninglists);
+
         foreach ($warninglists as $warninglist) {
             if ($id && $warninglist['Warninglist']['id'] != $id) {
                 continue;
             }
             $entries = $this->WarninglistEntry->find('list', array(
-                    'recursive' => -1,
-                    'conditions' => array('warninglist_id' => $warninglist['Warninglist']['id']),
-                    'fields' => array('value')
+                'recursive' => -1,
+                'conditions' => array('warninglist_id' => $warninglist['Warninglist']['id']),
+                'fields' => array('value')
             ));
             $this->cacheWarninglistEntries($entries, $warninglist['Warninglist']['id']);
         }
         return true;
     }
 
-    public function cacheWarninglists($warninglists)
+    private function cacheWarninglists(array $warninglists)
     {
         $redis = $this->setupRedis();
         if ($redis !== false) {
@@ -186,7 +325,7 @@ class Warninglist extends AppModel
         return false;
     }
 
-    public function cacheWarninglistEntries($warninglistEntries, $id)
+    private function cacheWarninglistEntries(array $warninglistEntries, $id)
     {
         $redis = $this->setupRedis();
         if ($redis !== false) {
@@ -196,7 +335,7 @@ class Warninglist extends AppModel
                 $redis->sAddArray($key, $warninglistEntries);
             } else {
                 foreach ($warninglistEntries as $entry) {
-                    $redis->sAdd('misp:warninglist_entries_cache:' . $id, $entry);
+                    $redis->sAdd($key, $entry);
                 }
             }
             return true;
@@ -204,62 +343,59 @@ class Warninglist extends AppModel
         return false;
     }
 
-    public function getWarninglists($conditions)
+    /**
+     * @return array
+     */
+    public function getEnabled()
     {
-        $redis = $this->setupRedis();
-        if ($redis !== false) {
-            if ($redis->sCard('misp:warninglist_cache') === 0) {
-                if (!empty($conditions)) {
-                    $warninglists = $this->find('all', array('contain' => array('WarninglistType'), 'conditions' => $conditions));
-                } else {
-                    $warninglists = $this->find('all', array('contain' => array('WarninglistType'), 'conditions' => array('enabled' => 1)));
-                }
-                if (empty($conditions)) {
-                    $this->cacheWarninglists($warninglists);
-                }
-                return $warninglists;
-            } else {
-                $warninglists = $redis->sMembers('misp:warninglist_cache');
-                foreach ($warninglists as $k => $v) {
-                    $warninglists[$k] = json_decode($v, true);
-                }
-                if (!empty($conditions)) {
-                    foreach ($warninglists as $k => $v) {
-                        foreach ($conditions as $k2 => $v2) {
-                            if ($v['Warninglist'][$k2] != $v2) {
-                                unset($warninglists[$k]);
-                                continue 2;
-                            }
-                        }
-                    }
-                }
-                return $warninglists;
-            }
+        if (isset($this->enabledCache)) {
+            return $this->enabledCache;
         }
-    }
 
-    public function getWarninglistEntries($id)
-    {
         $redis = $this->setupRedis();
-        if ($redis !== false) {
-            if ($redis->sCard('misp:warninglist_entries_cache:' . $id) === 0) {
-                $entries = $this->WarninglistEntry->find('list', array(
-                        'recursive' => -1,
-                        'conditions' => array('warninglist_id' => $id),
-                        'fields' => array('value')
-                ));
-                $this->cacheWarninglistEntries($entries, $id);
-            } else {
-                $entries = $redis->sMembers('misp:warninglist_entries_cache:' . $id);
+        if ($redis !== false && $redis->exists('misp:warninglist_cache')) {
+            $warninglists = $redis->sMembers('misp:warninglist_cache');
+            foreach ($warninglists as $k => $v) {
+                $warninglists[$k] = json_decode($v, true);
             }
         } else {
-            $entries = $this->WarninglistEntry->find('list', array(
-                    'recursive' => -1,
-                    'conditions' => array('warninglist_id' => $id),
-                    'fields' => array('value')
+            $warninglists = $this->find('all', array(
+                'contain' => array('WarninglistType'),
+                'conditions' => array('enabled' => 1),
+                'fields' => ['id', 'name', 'type'],
             ));
+            $this->cacheWarninglists($warninglists);
+        }
+
+        foreach ($warninglists as &$warninglist) {
+            $warninglist['types'] = [];
+            foreach ($warninglist['WarninglistType'] as $wt) {
+                $warninglist['types'][] = $wt['type'];
+            }
+            unset($warninglist['WarninglistType']);
+        }
+        $this->enabledCache = $warninglists;
+        return $warninglists;
+    }
+
+    /**
+     * @param int $id
+     * @return array
+     */
+    private function getWarninglistEntries($id)
+    {
+        $redis = $this->setupRedis();
+        if ($redis !== false && $redis->exists('misp:warninglist_entries_cache:' . $id)) {
+            return $redis->sMembers('misp:warninglist_entries_cache:' . $id);
+        } else {
+            $entries = array_values($this->WarninglistEntry->find('list', array(
+                'recursive' => -1,
+                'conditions' => array('warninglist_id' => $id),
+                'fields' => array('value')
+            )));
+            $this->cacheWarninglistEntries($entries, $id);
+            return $entries;
         }
-        return $entries;
     }
 
     /**
@@ -271,6 +407,7 @@ class Warninglist extends AppModel
     {
         $outputValues = [];
         foreach ($inputValues as $v) {
+            $v = strtolower($v);
             $parts = explode('/', $v, 2);
             if (filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) {
                 $maximumNetmask = 32;
@@ -289,58 +426,69 @@ class Warninglist extends AppModel
                 continue;
             }
 
-            $outputValues[$v] = $v;
+            $outputValues[$v] = true;
         }
         return $outputValues;
     }
 
-    public function fetchForEventView()
+    /**
+     * For 'hostname', 'string' and 'cidr' warninglist type, values are just in keys to save memory.
+     *
+     * @param array $warninglist
+     * @return array
+     */
+    public function getFilteredEntries(array $warninglist)
     {
-        $warninglists = $this->getWarninglists(array('enabled' => 1));
-        if (empty($warninglists)) {
-            return array();
+        $id = $warninglist['Warninglist']['id'];
+        if (isset($this->entriesCache[$id])) {
+            return $this->entriesCache[$id];
         }
-        foreach ($warninglists as $k => &$t) {
-            $t['values'] = $this->getWarninglistEntries($t['Warninglist']['id']);
-            $t['values'] = array_values($t['values']);
 
-            if ($t['Warninglist']['type'] === 'hostname') {
-                $values = [];
-                foreach ($t['values'] as $v) {
-                    $v = rtrim($v, '.');
-                    $values[$v] = $v;
-                }
-                $t['values'] = $values;
-            } else if ($t['Warninglist']['type'] === 'string') {
-                $t['values'] = array_combine($t['values'], $t['values']);
-            } else if ($t['Warninglist']['type'] === 'cidr') {
-                $t['values'] = $this->filterCidrList($t['values']);
+        $values = $this->getWarninglistEntries($id);
+        if ($warninglist['Warninglist']['type'] === 'hostname') {
+            $output = [];
+            foreach ($values as $v) {
+                $v = strtolower(trim($v, '.'));
+                $output[$v] = true;
             }
-
-            foreach ($t['WarninglistType'] as $wt) {
-                $t['types'][] = $wt['type'];
+            $values = $output;
+        } else if ($warninglist['Warninglist']['type'] === 'string') {
+            $output = [];
+            foreach ($values as $v) {
+                $output[$v] = true;
             }
-            unset($warninglists[$k]['WarninglistType']);
+            $values = $output;
+        } else if ($warninglist['Warninglist']['type'] === 'cidr') {
+            $values = $this->filterCidrList($values);
         }
-        return $warninglists;
+
+        $this->entriesCache[$id] = $values;
+
+        return $values;
     }
 
-    public function simpleCheckForWarning($object, $warninglists, $returnVerboseValue = false)
+    /**
+     * @param array $object
+     * @param array|null $warninglists If null, all enabled warninglists will be used
+     * @return array
+     */
+    public function checkForWarning(array $object, $warninglists = null)
     {
-        if ($object['to_ids']) {
+        if ($warninglists === null) {
+            $warninglists = $this->getEnabled();
+        }
+
+        if ($object['to_ids'] || $this->showForAll) {
             foreach ($warninglists as $list) {
                 if (in_array('ALL', $list['types']) || in_array($object['type'], $list['types'])) {
-                    $result = $this->__checkValue($list['values'], $object['value'], $object['type'], $list['Warninglist']['type']);
-                    if (!empty($result)) {
-                        if ($returnVerboseValue) {
-                            $object['warnings'][] = array(
-                                'value' => $result,
-                                'warninglist_name' => $list['Warninglist']['name'],
-                                'warninglist_id' => $list['Warninglist']['id']
-                            );
-                        } else {
-                            $object['warnings'][$result][] = $list['Warninglist']['name'];
-                        }
+                    $result = $this->__checkValue($this->getFilteredEntries($list), $object['value'], $object['type'], $list['Warninglist']['type']);
+                    if ($result !== false) {
+                        $object['warnings'][] = array(
+                            'match' => $result[0],
+                            'value' => $result[1],
+                            'warninglist_name' => $list['Warninglist']['name'],
+                            'warninglist_id' => $list['Warninglist']['id'],
+                        );
                     }
                 }
             }
@@ -348,85 +496,36 @@ class Warninglist extends AppModel
         return $object;
     }
 
-    public function checkForWarning($object, &$eventWarnings, $warningLists, $returnVerboseValue = false)
-    {
-        if ($object['to_ids']) {
-            foreach ($warningLists as $list) {
-                if (in_array('ALL', $list['types']) || in_array($object['type'], $list['types'])) {
-                    $result = $this->__checkValue($list['values'], $object['value'], $object['type'], $list['Warninglist']['type'], $returnVerboseValue);
-                    if (!empty($result)) {
-                        if ($returnVerboseValue) {
-                            $object['warnings'][] = array(
-                                'value' => $result,
-                                'warninglist_name' => $list['Warninglist']['name'],
-                                'warninglist_id' => $list['Warninglist']['id']
-                            );
-                        } else {
-                            $object['warnings'][$result][] = $list['Warninglist']['name'];
-                        }
-                        if (!isset($eventWarnings[$list['Warninglist']['id']])) {
-                            $eventWarnings[$list['Warninglist']['id']] = $list['Warninglist']['name'];
-                        }
-                    }
-                }
-            }
-        }
-        return $object;
-    }
-
-    public function setWarnings(&$event, &$warninglists)
-    {
-        if (empty($event['objects'])) {
-            return $event;
-        }
-        $eventWarnings = array();
-        foreach ($event['objects'] as &$object) {
-            if ($object['to_ids']) {
-                foreach ($warninglists as $list) {
-                    if (in_array('ALL', $list['types']) || in_array($object['type'], $list['types'])) {
-                        $result = $this->__checkValue($list['values'], $object['value'], $object['type'], $list['Warninglist']['type']);
-                        if (!empty($result)) {
-                            $object['warnings'][$result][] = $list['Warninglist']['name'];
-                            if (!in_array($list['Warninglist']['name'], $eventWarnings)) {
-                                $eventWarnings[$list['Warninglist']['id']] = $list['Warninglist']['name'];
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        $event['Event']['warnings'] = $eventWarnings;
-        return $event;
-    }
-
-    private function __checkValue($listValues, $value, $type, $listType, $returnVerboseValue = false)
+    /**
+     * @param array $listValues
+     * @param string $value
+     * @param string $type
+     * @param string $listType
+     * @return array|false [Matched value, attribute value that matched]
+     */
+    private function __checkValue($listValues, $value, $type, $listType)
     {
         if ($type === 'malware-sample' || strpos($type, '|') !== false) {
-            $value = explode('|', $value);
+            $value = explode('|', $value, 2);
         } else {
             $value = array($value);
         }
-        $components = array(0, 1);
-        foreach ($components as $component) {
-            if (!isset($value[$component])) {
-                continue;
-            }
+        foreach ($value as $v) {
             if ($listType === 'cidr') {
-                $result = $this->__evalCIDRList($listValues, $value[$component]);
+                $result = $this->__evalCidrList($listValues, $v);
             } elseif ($listType === 'string') {
-                $result = $this->__evalString($listValues, $value[$component]);
+                $result = $this->__evalString($listValues, $v);
             } elseif ($listType === 'substring') {
-                $result = $this->__evalSubString($listValues, $value[$component]);
+                $result = $this->__evalSubString($listValues, $v);
             } elseif ($listType === 'hostname') {
-                $result = $this->__evalHostname($listValues, $value[$component]);
+                $result = $this->__evalHostname($listValues, $v);
             } elseif ($listType === 'regex') {
-                $result = $this->__evalRegex($listValues, $value[$component]);
+                $result = $this->__evalRegex($listValues, $v);
+            } else {
+                $result = false;
             }
-            if (!empty($result)) {
-                if ($returnVerboseValue) {
-                    return $value[$component];
-                }
-                return ($component + 1);
+            if ($result !== false) {
+                return [$result, $v];
             }
         }
         return false;
@@ -434,47 +533,59 @@ class Warninglist extends AppModel
 
     public function quickCheckValue($listValues, $value, $type)
     {
-        $typeMapping = array(
-            'cidr' => '__evalCIDRList',
-            'string' => '__evalString',
-            'substring' => '__evalSubString',
-            'hostname' => '__evalHostname',
-            'regex' => '__evalRegex'
-        );
-        $result = $this->{$typeMapping[$type]}($listValues, $value);
-        return (!empty($result) ? 1 : false);
+        return $this->__checkValue($listValues, $value, '', $type) !== false;
     }
 
-    // This requires an IP type attribute in a non CIDR notation format
-    // For the future we can expand this to look for CIDR overlaps?
-    private function __evalCIDRList($listValues, $value)
+    private function __evalCidrList($listValues, $value)
     {
+        $valueMask = null;
+        if (strpos($value, '/') !== false) {
+            list($value, $valueMask) = explode('/', $value);
+        }
+
+        $match = false;
         if (filter_var($value, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) {
             // This code converts IP address to all possible CIDRs that can contains given IP address
             // and then check if given hash table contains that CIDR.
             $ip = ip2long($value);
-            for ($bits = 0; $bits <= 32; $bits++) {
+            // Start from 1, because doesn't make sense to check 0.0.0.0/0 match
+            for ($bits = 1; $bits <= 32; $bits++) {
                 $mask = -1 << (32 - $bits);
                 $needle = long2ip($ip & $mask) . "/$bits";
                 if (isset($listValues[$needle])) {
-                    return true;
+                    $match = $needle;
+                    break;
                 }
             }
 
         } elseif (filter_var($value, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) {
-            foreach ($listValues as $lv) {
-                if (strpos($lv, ':') !== false) { // IPv6 CIDR must contain dot
+            foreach ($listValues as $lv => $foo) {
+                if (strpos($lv, ':') !== false) { // Filter out IPv4 CIDR, IPv6 CIDR must contain colon
                     if ($this->__ipv6InCidr($value, $lv)) {
-                        return true;
+                        $match = $lv;
+                        break;
                     }
                 }
             }
         }
 
-        return false;
+        if ($match && $valueMask) {
+            $matchMask = explode('/', $match)[1];
+            if ($valueMask < $matchMask) {
+                return false;
+            }
+        }
+
+        return $match;
     }
 
-    // Using solution from https://github.com/symfony/symfony/blob/master/src/Symfony/Component/HttpFoundation/IpUtils.php
+    /**
+     * Using solution from https://github.com/symfony/symfony/blob/master/src/Symfony/Component/HttpFoundation/IpUtils.php
+     *
+     * @param string $ip
+     * @param string $cidr
+     * @return bool
+     */
     private function __ipv6InCidr($ip, $cidr)
     {
         list($address, $netmask) = explode('/', $cidr);
@@ -494,10 +605,17 @@ class Warninglist extends AppModel
         return true;
     }
 
+    /**
+     * Check for exact match.
+     *
+     * @param array $listValues
+     * @param string $value
+     * @return false
+     */
     private function __evalString($listValues, $value)
     {
         if (isset($listValues[$value])) {
-            return true;
+            return $value;
         }
         return false;
     }
@@ -506,7 +624,7 @@ class Warninglist extends AppModel
     {
         foreach ($listValues as $listValue) {
             if (strpos($value, $listValue) !== false) {
-                return true;
+                return $listValue;
             }
         }
         return false;
@@ -515,7 +633,7 @@ class Warninglist extends AppModel
     private function __evalHostname($listValues, $value)
     {
         // php's parse_url is dumb, so let's use some hacky workarounds
-        if (strpos($value, '//') == false) {
+        if (strpos($value, '//') === false) {
             $value = explode('/', $value);
             $hostname = $value[0];
         } else {
@@ -529,14 +647,14 @@ class Warninglist extends AppModel
         $hostname = rtrim($hostname, '.');
         $hostname = explode('.', $hostname);
         $rebuilt = '';
-        foreach (array_reverse($hostname) as $k => $piece) {
+        foreach (array_reverse($hostname) as $piece) {
             if (empty($rebuilt)) {
                 $rebuilt = $piece;
             } else {
                 $rebuilt = $piece . '.' . $rebuilt;
             }
             if (isset($listValues[$rebuilt])) {
-                return true;
+                return $rebuilt;
             }
         }
         return false;
@@ -546,23 +664,30 @@ class Warninglist extends AppModel
     {
         foreach ($listValues as $listValue) {
             if (preg_match($listValue, $value)) {
-                return true;
+                return $listValue;
             }
         }
         return false;
     }
 
+    /**
+     * @return array
+     */
     public function fetchTLDLists()
     {
-        $tldLists = $this->find('list', array('conditions' => array('Warninglist.name' => $this->__tlds), 'recursive' => -1, 'fields' => array('Warninglist.id', 'Warninglist.name')));
+        $tldLists = $this->find('list', array(
+            'conditions' => array('Warninglist.name' => $this->__tlds),
+            'recursive' => -1,
+            'fields' => array('Warninglist.id', 'Warninglist.id')
+        ));
         $tlds = array();
         if (!empty($tldLists)) {
-            $tldLists = array_keys($tldLists);
-            $tlds = $this->WarninglistEntry->find('list', array('conditions' => array('WarninglistEntry.warninglist_id' => $tldLists), 'fields' => array('WarninglistEntry.value')));
-            if (!empty($tlds)) {
-                foreach ($tlds as $key => $value) {
-                    $tlds[$key] = strtolower($value);
-                }
+            $tlds = $this->WarninglistEntry->find('list', array(
+                'conditions' => array('WarninglistEntry.warninglist_id' => $tldLists),
+                'fields' => array('WarninglistEntry.value')
+            ));
+            foreach ($tlds as $key => $value) {
+                $tlds[$key] = strtolower($value);
             }
         }
         if (!in_array('onion', $tlds)) {
@@ -571,11 +696,20 @@ class Warninglist extends AppModel
         return $tlds;
     }
 
-    public function filterWarninglistAttributes($warninglists, $attribute)
+    /**
+     * @param array $attribute
+     * @param array|null $warninglists If null, all enabled warninglists will be used
+     * @return bool
+     */
+    public function filterWarninglistAttribute(array $attribute, $warninglists = null)
     {
+        if ($warninglists === null) {
+            $warninglists = $this->getEnabled();
+        }
+
         foreach ($warninglists as $warninglist) {
             if (in_array('ALL', $warninglist['types']) || in_array($attribute['type'], $warninglist['types'])) {
-                $result = $this->__checkValue($warninglist['values'], $attribute['value'], $attribute['type'], $warninglist['Warninglist']['type']);
+                $result = $this->__checkValue($this->getFilteredEntries($warninglist), $attribute['value'], $attribute['type'], $warninglist['Warninglist']['type']);
                 if ($result !== false) {
                     return false;
                 }
@@ -583,4 +717,20 @@ class Warninglist extends AppModel
         }
         return true;
     }
+
+    public function missingTldLists()
+    {
+        $missingTldLists = array();
+        foreach ($this->__tlds as $tldList) {
+            $temp = $this->find('first', array(
+                'recursive' => -1,
+                'conditions' => array('Warninglist.name' => $tldList),
+                'fields' => array('Warninglist.id')
+            ));
+            if (empty($temp)) {
+                $missingTldLists[] = $tldList;
+            }
+        }
+        return $missingTldLists;
+    }
 }

app/Test/ComplexTypeToolTest.php

@@ -420,6 +420,15 @@ EOT;
         $this->assertEquals('vulnerability', $results[0]['default_type']);
     }
 
+    public function testCheckFreeTextCveLowercase(): void
+    {
+        $complexTypeTool = new ComplexTypeTool();
+        $results = $complexTypeTool->checkFreeText('cve-2019-16202');
+        $this->assertCount(1, $results);
+        $this->assertEquals('CVE-2019-16202', $results[0]['value']);
+        $this->assertEquals('vulnerability', $results[0]['default_type']);
+    }
+
     public function testCheckFreeTextAs(): void
     {
         $complexTypeTool = new ComplexTypeTool();
@@ -438,6 +447,15 @@ EOT;
         $this->assertEquals('md5', $results[0]['default_type']);
     }
 
+    public function testCheckFreeTextMd5Uppercase(): void
+    {
+        $complexTypeTool = new ComplexTypeTool();
+        $results = $complexTypeTool->checkFreeText('9E107D9D372BB6826BD81D3542A419D6');
+        $this->assertCount(1, $results);
+        $this->assertEquals('9E107D9D372BB6826BD81D3542A419D6', $results[0]['value']);
+        $this->assertEquals('md5', $results[0]['default_type']);
+    }
+
     public function testCheckFreeTextSha1(): void
     {
         $complexTypeTool = new ComplexTypeTool();

app/View/Allowedlists/admin_add.ctp

@@ -1,7 +1,7 @@
-<div class="whitelist form">
-<?php echo $this->Form->create('Whitelist');?>
+<div class="allowedlist form">
+<?php echo $this->Form->create('Allowedlist');?>
     <fieldset>
-        <legend><?php echo __('Add Signature Whitelist');?></legend>
+        <legend><?php echo __('Add Signature Allowedlist');?></legend>
     <?php
         echo $this->Form->input('name', array(
             'class' => 'input-xxlarge'
@@ -15,5 +15,5 @@ echo $this->Form->end();
 ?>
 </div>
 <?php
-    echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'whitelist', 'menuItem' => 'add'));
+    echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'allowedlist', 'menuItem' => 'add'));
 ?>

app/View/Allowedlists/admin_edit.ctp

@@ -1,7 +1,7 @@
-<div class="whitelist form">
-<?php echo $this->Form->create('Whitelist');?>
+<div class="allowedlist form">
+<?php echo $this->Form->create('Allowedlist');?>
     <fieldset>
-        <legend><?php echo __('Edit Signature Whitelist');?></legend>
+        <legend><?php echo __('Edit Signature Allowedlist');?></legend>
     <?php
         echo $this->Form->input('id');
         echo $this->Form->input('name', array(
@@ -15,5 +15,5 @@
 ?>
 </div>
 <?php
-    echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'whitelist', 'menuItem' => 'edit', 'id' => $this->Form->value('Whitelist.id')));
+    echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'allowedlist', 'menuItem' => 'edit', 'id' => $this->Form->value('Allowedlist.id')));
 ?>

app/View/Allowedlists/admin_index.ctp

@@ -1,5 +1,5 @@
-<div class="whitelist index">
-    <h2><?php echo __('Signature Whitelist');?></h2>
+<div class="allowedlist index">
+    <h2><?php echo __('Signature Allowedlist');?></h2>
     <p><?php echo __('Regex entries (in the standard php regex /{regex}/{modifier} format) entered below will restrict matching attributes from being included in the IDS flag sensitive exports (such as NIDS exports).');?></p>
     <div class="pagination">
         <ul>
@@ -25,11 +25,11 @@
     </tr><?php
 foreach ($list as $item):?>
     <tr>
-        <td class="short"><?php echo h($item['Whitelist']['id']);?>&nbsp;</td>
-        <td><?php echo h($item['Whitelist']['name']);?>&nbsp;</td>
+        <td class="short"><?php echo h($item['Allowedlist']['id']);?>&nbsp;</td>
+        <td><?php echo h($item['Allowedlist']['name']);?>&nbsp;</td>
         <td class="short action-links">
-            <?php echo $this->Html->link('', array('admin' => true, 'action' => 'edit', $item['Whitelist']['id']), array('class' => 'fa fa-edit', 'title' => __('Edit'), 'aria-label' => __('Edit')));?>
-            <?php echo $this->Form->postLink('', array('admin' => true, 'action' => 'delete', $item['Whitelist']['id']), array('class' => 'fa fa-trash', 'title' => __('Delete'), 'aria-label' => __('Delete')), __('Are you sure you want to delete "%s"?', $item['Whitelist']['name']));?>
+            <?php echo $this->Html->link('', array('admin' => true, 'action' => 'edit', $item['Allowedlist']['id']), array('class' => 'fa fa-edit', 'title' => __('Edit'), 'aria-label' => __('Edit')));?>
+            <?php echo $this->Form->postLink('', array('admin' => true, 'action' => 'delete', $item['Allowedlist']['id']), array('class' => 'fa fa-trash', 'title' => __('Delete'), 'aria-label' => __('Delete')), __('Are you sure you want to delete "%s"?', $item['Allowedlist']['name']));?>
         </td>
     </tr><?php
 endforeach;?>
@@ -52,4 +52,4 @@ endforeach;?>
     </div>
 </div>
 <?php
-    echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'whitelist', 'menuItem' => 'index'));
+    echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'allowedlist', 'menuItem' => 'index'));

app/View/Allowedlists/index.ctp

@@ -1,5 +1,5 @@
-<div class="whitelist index">
-    <h2><?php echo __('Signature Whitelist');?></h2>
+<div class="allowedlist index">
+    <h2><?php echo __('Signature Allowedlist');?></h2>
     <p><?php echo __('Regex entries (in the standard php regex /{regex}/{modifier} format) entered below will restrict matching attributes from being included in the IDS flag sensitive exports (such as NIDS exports).');?></p>
     <div class="pagination">
         <ul>
@@ -24,8 +24,8 @@
     </tr><?php
 foreach ($list as $item):?>
     <tr>
-        <td class="short"><?php echo h($item['Whitelist']['id']);?>&nbsp;</td>
-        <td><?php echo h($item['Whitelist']['name']);?>&nbsp;</td>
+        <td class="short"><?php echo h($item['Allowedlist']['id']);?>&nbsp;</td>
+        <td><?php echo h($item['Allowedlist']['name']);?>&nbsp;</td>
     </tr><?php
 endforeach;?>
     </table>
@@ -47,4 +47,4 @@ endforeach;?>
     </div>
 </div>
 <?php
-    echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'whitelist', 'menuItem' => 'index'));
+    echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'allowedlist', 'menuItem' => 'index'));

app/View/Attributes/ajax/attributeEditCategoryForm.ctp

@@ -2,8 +2,8 @@
     echo $this->Form->create('Attribute', array('class' => 'inline-form inline-field-form', 'id' => 'Attribute_' . $object['id'] . '_category_form', 'url' => $baseurl . '/attributes/editField/' . $object['id']));
 ?>
 <div class='inline-input inline-input-container'>
-<div class="inline-input-accept inline-input-button inline-input-passive"><span class = "icon-ok" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
-<div class="inline-input-decline inline-input-button inline-input-passive"><span class = "icon-remove" role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
+<div class="inline-input-accept inline-input-button inline-input-passive"><span class="fas fa-check" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
+<div class="inline-input-decline inline-input-button inline-input-passive"><span class="fas fa-times" role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
 <?php
     echo $this->Form->input('category', array(
         'options' => array(array_combine($typeCategory[$object['type']], $typeCategory[$object['type']])),

app/View/Attributes/ajax/attributeEditCommentForm.ctp

@@ -2,8 +2,8 @@
     echo $this->Form->create('Attribute', array('class' => 'inline-form inline-field-form', 'id' => 'Attribute_' . $object['id'] . '_comment_form', 'url' => $baseurl . '/attributes/editField/' . $object['id']));
 ?>
     <div class='inline-input inline-input-container'>
-    <div class="inline-input-accept inline-input-button inline-input-passive"><span class = "icon-ok" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
-    <div class="inline-input-decline inline-input-button inline-input-passive"><span class = "icon-remove" role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
+    <div class="inline-input-accept inline-input-button inline-input-passive"><span class="fas fa-check" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
+    <div class="inline-input-decline inline-input-button inline-input-passive"><span class="fas fa-times" role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
         <?php
             echo $this->Form->input('comment', array(
                     'type' => 'textarea',

app/View/Attributes/ajax/attributeEditDistributionForm.ctp

@@ -2,8 +2,8 @@
     echo $this->Form->create('Attribute', array('class' => 'inline-form inline-field-form', 'id' => 'Attribute_' . $object['id'] . '_distribution_form', 'url' => $baseurl . '/attributes/editField/' . $object['id']));
 ?>
 <div class='inline-input inline-input-container'>
-    <div class="inline-input-accept inline-input-button inline-input-passive"><span class = "icon-ok" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
-    <div class="inline-input-decline inline-input-button inline-input-passive"><span class = "icon-remove" role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
+    <div class="inline-input-accept inline-input-button inline-input-passive"><span class="fas fa-check" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
+    <div class="inline-input-decline inline-input-button inline-input-passive"><span class="fas fa-times"role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
     <?php
         echo $this->Form->input('distribution', array(
                 'options' => array($distributionLevels),

app/View/Attributes/ajax/attributeEditTypeForm.ctp

@@ -2,8 +2,8 @@
     echo $this->Form->create('Attribute', array('class' => 'inline-form inline-field-form', 'id' => 'Attribute_' . $object['id'] . '_type_form', 'url' => $baseurl . '/attributes/editField/' . $object['id']));
 ?>
 <div class='inline-input inline-input-container'>
-    <div class="inline-input-accept inline-input-button inline-input-passive"><span class = "icon-ok" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
-    <div class="inline-input-decline inline-input-button inline-input-passive"><span class = "icon-remove" role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
+    <div class="inline-input-accept inline-input-button inline-input-passive"><span class="fas fa-check" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
+    <div class="inline-input-decline inline-input-button inline-input-passive"><span class="fas fa-times" role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
 <?php
     echo $this->Form->input('type', array(
         'options' => array(array_combine($categoryDefinitions[$object['category']]['types'], $categoryDefinitions[$object['category']]['types'])),

app/View/Attributes/ajax/attributeEditValueForm.ctp

@@ -3,8 +3,8 @@
     echo $this->Form->create('Attribute', array('class' => 'inline-form inline-field-form', 'url' => $baseurl . '/attributes/editField/' . $object['id'], 'id' => 'Attribute_' . $object['id'] . '_value_form', 'default' => false));
 ?>
     <div class='inline-input inline-input-container'>
-    <div class="inline-input-accept inline-input-button inline-input-passive"><span class = "icon-ok" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
-    <div class="inline-input-decline inline-input-button inline-input-passive"><span class = "icon-remove" role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
+    <div class="inline-input-accept inline-input-button inline-input-passive"><span class="fas fa-check" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
+    <div class="inline-input-decline inline-input-button inline-input-passive"><span class="fas fa-times" role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
 <?php
     echo $this->Form->input('value', array(
             'type' => 'textarea',

app/View/Attributes/ajax/attributeViewFieldForm.ctp

@@ -1,8 +1,12 @@
 <?php
-if ($value === 'No') {
-    echo '<input type="checkbox" disabled>';
-} else if ($value === 'Yes') {
-    echo '<input type="checkbox" checked disabled>';
+if ($field === 'value') {
+    echo $this->element('Events/View/value_field', ['object' => $object['Attribute']]);
 } else {
-    echo nl2br(h($value)) . '&nbsp;';
+    if ($value === 'No') {
+        echo '<input type="checkbox" disabled>';
+    } else if ($value === 'Yes') {
+        echo '<input type="checkbox" checked disabled>';
+    } else {
+        echo nl2br(h($value)) . '&nbsp;';
+    }
 }

app/View/Attributes/ajax/hover_enrichment.ctp

@@ -1,49 +1,59 @@
-<div style="overflow-y:auto;max-height:75vh">
-    <?php
-        foreach ($results as $enrichment_type => $enrichment_values):
-            echo sprintf('<h5><span class="hover_enrichment_title blue">%s</span>:</h5>', Inflector::humanize(h($enrichment_type)));
-            if (empty($enrichment_values)) {
-                echo '<div style="padding: 2px;"><span class="empty_results_text red">Empty results</span></div>';
-                continue;
+<div style="overflow-y:auto;<?= $persistent ? 'max-height:98%;padding: .3em 1em' : 'max-height:75vh' ?>">
+<?php
+$formatValue = function (array $attribute) {
+    switch ($attribute['type']) {
+        case 'link':
+            return '<a href="' . h($attribute['value']) . '" rel="noreferrer noopener" target="_blank">' . h($attribute['value']) . '</a>';
+        default:
+            return h($attribute['value']);
+    }
+};
+foreach ($results as $enrichment_type => $enrichment_values) {
+    echo sprintf('<h5 class="hover_enrichment_title blue">%s:</h5>', Inflector::humanize(h($enrichment_type)));
+    if (isset($enrichment_values['error'])) {
+        echo '<div style="padding: 2px;" class="red">' . __('Error: %s', h($enrichment_values['error']))  . '</div>';
+        continue;
+    }
+    if (empty($enrichment_values)) {
+        echo '<div style="padding: 2px;" class="red">' . __('Empty results') . '</div>';
+        continue;
+    }
+    if (!empty($enrichment_values['Object'])) {
+        foreach ($enrichment_values['Object'] as $object) {
+            echo '<h6 class="bold blue">' . __('Object: %s', h($object['name'])) . '</h6>';
+            echo '<table class="table table-striped table-condensed">';
+            foreach ($object['Attribute'] as $object_attribute) {
+                echo '<tr><th style="width: 15em">' . h($object_attribute['object_relation']) . '</th><td>' . $formatValue($object_attribute) . '</td></tr>';
             }
-            if (!empty($enrichment_values['Object'])) {
-                echo '<h6><span class="bold blue">Objects</span></h6>';
-                foreach ($enrichment_values['Object'] as $object) {
-                    echo '<span class="object_name bold blue">' . h($object['name']) . '</span><br />';
-                    foreach ($object['Attribute'] as $object_attribute) {
-                        echo '<div style="padding: 2px;"><pre class="object_attribute">';
-                        echo '<span class="attribute_object_relation bold blue">' . h($object_attribute['object_relation']) . '</span>';
-                        echo ': <span class="attribute_value red">' . h($object_attribute['value']) . '</span></pre></div>';
+            echo '</table>';
+        }
+        unset($enrichment_values['Object']);
+    }
+    if (!empty($enrichment_values['Attribute'])) {
+        echo '<h6 class="bold blue">' . __('Attributes') . '</h6>';
+        echo '<table class="table table-striped table-condensed">';
+        foreach ($enrichment_values['Attribute'] as $attribute) {
+            echo '<tr><th style="width: 15em">' . h($attribute['type']). '</th><td>' . $formatValue($attribute) . '</td></tr>';
+        }
+        echo '</table>';
+        unset($enrichment_values['Attribute']);
+    }
+    foreach ($enrichment_values as $attributes) {
+        foreach ($attributes as $attribute) {
+            echo '<div style="padding: 2px;">';
+            if (is_array($attribute)) {
+                foreach ($attribute as $attribute_name => $attribute_value) {
+                    if (!is_numeric($attribute_name)) {
+                        echo '<strong>' . h($attribute_name) . ':</strong> ';
                     }
+                    echo ' ' . h($attribute_value);
                 }
-                unset($enrichment_values['Object']);
+            } else {
+                echo h($attribute);
             }
-            if (!empty($enrichment_values['Attribute'])) {
-                echo '<h6><span class="bold blue">Attributes</span><br />';
-                foreach ($enrichment_values['Attribute'] as $attribute) {
-                    echo '<div style="padding: 2px;"><pre class="attribute">';
-                    echo '<span class="attribute_type bold blue">' . h($attribute['type']) . '</span>';
-                    echo ': <span class="attribute_value red">' . h($attribute['value']) . '</span></pre></div>';
-                }
-                unset($enrichment_values['Attribute']);
-            }
-            foreach ($enrichment_values as $attributes):
-                foreach ($attributes as $attribute):
-                    echo '<div style="padding: 2px;">';
-                    if (is_array($attribute)) {
-                        foreach ($attribute as $attribute_name => $attribute_value) {
-                            if (!is_numeric($attribute_name)) {
-                                echo '<span class="hover_enrichment_text blue">' . h($attribute_name) . ':</span>';
-                            }
-                            echo '<span><pre class="hover_enrichment_text red">' . h($attribute_value) . '</pre></span>';
-                        }
-                    } else {
-                        echo '<span><pre class="hover_enrichment_text red ">' . h($attribute) . '</pre></span>';
-                    }
-                    echo '</div>';
-                endforeach;
-            endforeach;
-            echo "<br/>";
-        endforeach;
-    ?>
+            echo '</div>';
+        }
+    }
+}
+?>
 </div>

app/View/Attributes/index.ctp

@@ -63,11 +63,12 @@
             __('Actions')
         );
         foreach ($headers as $k => &$header) {
-            $header = sprintf('<th>%s</th>', $header);
+            $header = "<th>$header</th>";
         }
-        $rows = array(
-            sprintf('<tr>%s</tr>', implode('', $headers))
-        );
+        ?>
+    <table class="table table-striped table-hover table-condensed">
+        <tr><?= implode('', $headers) ?></tr>
+        <?php
         $currentCount = 0;
         if ($isSearch == 1) {
             // sanitize data
@@ -94,11 +95,10 @@
             $mayModify = ($isSiteAdmin || ($isAclModify && $event['Event']['user_id'] == $me['id'] && $attribute['Event']['orgc_id'] == $me['org_id']) || ($isAclModifyOrg && $attribute['Event']['orgc_id'] == $me['org_id']));
             $mayPublish = ($isAclPublish && $attribute['Event']['orgc_id'] == $me['org_id']);
             $mayChangeCorrelation = !Configure::read('MISP.completely_disable_correlation') && ($isSiteAdmin || ($mayModify && Configure::read('MISP.allow_disabling_correlation')));
-            $mayModify = $attribute['Event']['orgc_id'] === $me['org_id'] ? true : false;
             if (!empty($attribute['Attribute']['RelatedAttribute'])) {
                 $event['RelatedAttribute'] = array($attribute['Attribute']['id'] => $attribute['Attribute']['RelatedAttribute']);
             }
-            $rows[] =  $this->element('/Events/View/row_attribute', array(
+            echo $this->element('/Events/View/row_attribute', array(
                 'object' => $attribute['Attribute'],
                 'k' => $k,
                 'mayModify' => $mayModify,
@@ -113,8 +113,8 @@
                 'context' => 'list'
             ));
         }
-        echo sprintf('<table class="table table-striped table-hover table-condensed">%s</table>', implode('', $rows));
     ?>
+    </table>
     <p>
     <?php
     echo $this->Paginator->counter(array(
@@ -144,7 +144,7 @@ if ($isSearch == 1){
 ?>
 <script type="text/javascript">
 // tooltips
-$(document).ready(function () {
+$(function () {
     $("td, div").tooltip({
         'placement': 'top',
         'container' : 'body',

app/View/Communities/index.ctp

@@ -100,10 +100,5 @@
         $('#quickFilterButton').click(function() {
             runIndexQuickFilter('/context:' + passedArgsArray['context']);
         });
-        $('#quickFilterField').on('keypress', function (e) {
-            if(e.which === 13) {
-                runIndexQuickFilter('/context:' + passedArgsArray['context']);
-            }
-        });
     });
 </script>

app/View/Dashboards/list_templates.ctp

@@ -98,10 +98,5 @@
         $('#quickFilterButton').click(function() {
             runIndexQuickFilter();
         });
-        $('#quickFilterField').on('keypress', function (e) {
-            if(e.which === 13) {
-                runIndexQuickFilter();
-            }
-        });
     });
 </script>

app/View/Elements/EventReports/reportEditor.ctp

@@ -0,0 +1,19 @@
+<script>
+    'use strict';
+    var proxyMISPElements = null
+    var eventid = '<?= !isset($eventid) ? '' : h($eventid) ?>'
+    var reportid = '<?= h($reportid) ?>'
+    var invalidMessage = '<?= __('invalid scope or id') ?>'
+</script>
+
+<?php
+    echo $this->element('genericElements/assetLoader', [
+        'js' => [
+            'markdownEditor/event-report',
+        ],
+        'css' => [
+            'markdownEditor/event-report',
+            ]
+    ]);
+    
+?>

app/View/Elements/EventReports/reportHelpModal.ctp

@@ -0,0 +1,74 @@
+<?php
+    $formatDifferences = [
+        __('No html support, typographer & autolinker'),
+        __('An additional syntax to reference MISP Elements'),
+    ];
+    $allowedScopes = ['attribute', 'object', 'galaxymatrix' ,'tag'];
+    $allowedScopesHtml = '<code>' . implode('</code> <code>', $allowedScopes) . '</code>';
+?>
+
+<h2><?= __('Markdown format') ?></h2>
+<p><?= __('The suported markdown format is similar to %s with some differences:', sprintf('<a href="%s" target="_blank">GFM</a>', 'https://github.github.com/gfm/')) ?></p>
+<ul>
+    <?php foreach($formatDifferences as $formatDifference): ?>
+        <li><?= $formatDifference ?></li>
+    <?php endforeach; ?>
+</ul>
+
+<h2><?= __('Markdown extended format') ?></h2>
+<p><?= __('In order to have a visually pleasant report but more importantly, avoid hardcoding element\'s value or ID, MISP elements such as attributes and objects can be referenced with the following special syntax') ?></p>
+<h4 style="text-align: center;">
+    <code style="font-size: 14px;">@[scope](UUID)</code>
+</h4>
+<span><?= __('Where:') ?></span>
+<ul>
+    <li><b>scope</b>: <?= __('Is the scope to which the UUID is related to.') ?></li>
+    <ul>
+        <li><?= __('Can be one of the following: %s', $allowedScopesHtml) ?></li>
+    </ul>
+    <li><b>UUID</b>: <?= __('Is the UUID of the MISP element with only one exception for the tag') ?></li>
+</ul>
+<span><?= __('Examples:') ?></span>
+<ul>
+    <li><code>@[attribute](5f1accda-cde4-47fc-baf1-6ab8f331dc3b)</code></li>
+    <li><code>@[object](5f1accda-cde4-47fc-baf1-6ab8f331dc3b)</code></li>
+    <li><code>@[galaxymatrix](5f1accda-cde4-47fc-baf1-6ab8f331dc3b)</code></li>
+</ul>
+
+<h4><?= __('Pictures from attachment-type attributes') ?></h4>
+<p><?= __('Syntax for pictures is like the syntax for referencing MISP elements but with two differences:') ?></p>
+<ul>
+    <li><?= __('The addition of the %s character to indicate that the picture should be displayed and not the atttribute', '<code>!</code>') ?></li>
+    <li><?= __('The scope is fixed to %s as only attributes can contain a file', '<code>attribute</code>') ?></li>
+</ul>
+
+<h4 style="text-align: center;">
+    <code style="font-size: 14px;">@![attribute](UUID)</code>
+</h4>
+<span><?= __('Examples:') ?></span>
+<ul>
+    <li><code>@![attribute](5f1accda-cde4-47fc-baf1-6ab8f331dc3b)</code></li>
+</ul>
+
+<h4><?= __('Tags') ?></h4>
+<p><?=  __('Syntax for representing tags is similar the syntax for referencing MISP elements but with two differences:') ?></p>
+<ul>
+    <li><?= __('The scope is fixed to %s', '<code>tag</code>') ?></li>
+    <li><?= __('The UUID is replaced by the tag name sa tags don\'t have UUID') ?></li>
+</ul>
+<span><?= __('Examples:') ?></span>
+<ul>
+    <li><code>@[tag](tlp:green)</code></li>
+    <li><code>@[tag](misp-galaxy:threat-actor="APT 29")</code></li>
+</ul>
+
+<h4><?= __('Event\'s Galaxy matrixes') ?></h4>
+<p><?=  __('Syntax for embedding the ATT&CK matrix or any other galaxy matrixes is similar to the syntax for referencing MISP elements:') ?></p>
+<ul>
+    <li><?= __('The scope is fixed to %s', '<code>galaxymatrix</code>') ?></li>
+    <li><?= __('The matrix will be generated for the whole event for which the report is linked to') ?></li>
+</ul>
+<span><?= __('Examples:') ?></span>
+<ul>
+    <li><code>@[galaxymatrix](5f1accda-cde4-47fc-baf1-6ab8f331dc3b)</code></li>
+</ul>

app/View/Elements/Events/View/eventSightingValue.ctp

@@ -1,21 +1,29 @@
 <?php
-    echo sprintf(
-        '%s (%s) %s %s',
-        sprintf(
-            '<span id="eventSightingCount" class="bold sightingsCounter" data-toggle="popover" data-trigger="hover" data-content="%s">%s</span>',
-            $sightingPopover,
-            count($event['Sighting'])
-        ),
-        sprintf(
-            '<span id="eventOwnSightingCount" class="green bold sightingsCounter" data-toggle="popover" data-trigger="hover" data-content="%s">%s</span>',
-            $sightingPopover,
-            isset($ownSightings) ? count($ownSightings) : 0
-        ),
-        (Configure::read('Plugin.Sightings_policy')) ? '' : __('- restricted to own organisation only.'),
-        sprintf(
-            '<span class="icon-wrench useCursorPointer sightings_advanced_add" title="%s" role="button" tabindex="0" aria-label="%s" data-object-id="%s" data-object-context="event">&nbsp;</span>',
-             __('Advanced Sightings'),
-            __('Advanced Sightings'),
-            h($event['Event']['id'])
-        )
-    );
+$ownOrgSightingsCount = 0;
+if (isset($event['Sighting'])) {
+    $meOrgId = $this->get('me')['org_id'];
+    foreach ($event['Sighting'] as $sighting) {
+        if (isset($sighting['org_id']) && $sighting['org_id'] == $meOrgId) {
+            ++$ownOrgSightingsCount;
+        }
+    }
+}
+
+echo sprintf(
+    '%s (%s) %s %s',
+    sprintf(
+        '<span id="eventSightingCount" class="bold sightingsCounter">%s</span>',
+        count($event['Sighting'])
+    ),
+    sprintf(
+        '<span id="eventOwnSightingCount" class="green bold sightingsCounter">%s</span>',
+        $ownOrgSightingsCount
+    ),
+    (Configure::read('Plugin.Sightings_policy')) ? '' : __('- restricted to own organisation only.'),
+    sprintf(
+        '<span class="fas fa-wrench useCursorPointer sightings_advanced_add" title="%s" role="button" tabindex="0" aria-label="%s" data-object-id="%s" data-object-context="event">&nbsp;</span>',
+        __('Advanced Sightings'),
+        __('Advanced Sightings'),
+        h($event['Event']['id'])
+    )
+);

app/View/Elements/Events/View/row_attribute.ctp

@@ -1,46 +1,62 @@
 <?php
-  $tr_class = '';
-  if (empty($context)) {
-      $context = 'event';
-  }
-  $linkClass = 'blue';
-  if ($event['Event']['id'] != $object['event_id']) {
+$tr_class = '';
+if (empty($context)) {
+    $context = 'event';
+}
+$linkClass = 'blue';
+if ($event['Event']['id'] != $object['event_id']) {
     if (!$isSiteAdmin && $event['extensionEvents'][$object['event_id']]['Orgc']['id'] != $me['org_id']) {
-      $mayModify = false;
+        $mayModify = false;
     }
-  }
-  $editScope = ($isSiteAdmin || $mayModify) ? 'Attribute' : 'ShadowAttribute';
-  if (!empty($child)) {
+}
+$editScope = ($isSiteAdmin || $mayModify) ? 'Attribute' : 'ShadowAttribute';
+if (!empty($child)) {
     if ($child === 'last' && empty($object['ShadowAttribute'])) {
-      $tr_class .= ' tableHighlightBorderBottom borderBlue';
+        $tr_class .= ' tableHighlightBorderBottom borderBlue';
     } else {
-      $tr_class .= ' tableHighlightBorderCenter borderBlue';
+        $tr_class .= ' tableHighlightBorderCenter borderBlue';
     }
     if (!empty($object['ShadowAttribute'])) {
-      $tr_class .= ' tableInsetOrangeFirst';
+        $tr_class .= ' tableInsetOrangeFirst';
     }
-  } else {
+} else {
     $child = false;
     if (!empty($object['ShadowAttribute'])) {
-      $tr_class .= ' tableHighlightBorderTop borderOrange';
+        $tr_class .= ' tableHighlightBorderTop borderOrange';
     }
-  }
-  if (!empty($object['deleted'])) {
+}
+if (!empty($object['deleted'])) {
     $tr_class .= ' deleted-attribute';
-  }
-  if (!empty($k)) {
+}
+if (!empty($k)) {
     $tr_class .= ' row_' . h($k);
-  }
+}
+
+$objectId = h($object['id']);
+
+$quickEdit = function($fieldName) use ($editScope, $object, $event) {
+    if ($object['deleted']) {
+        return ''; // deleted attributes are not editable
+    }
+    if ($editScope === 'ShadowAttribute') {
+        return ''; // currently it is not supported to create proposals trough quick edit
+    }
+    if ($fieldName === 'value' && ($object['type'] === 'attachment' || $object['type'] === 'malware-sample')) {
+        return '';
+    }
+    return " onmouseenter=\"quickEditHover(this, '$editScope', '{$object['id']}', '$fieldName', {$event['Event']['id']});\"";
+}
+
 ?>
-<tr id = "Attribute_<?php echo h($object['id']); ?>_tr" class="<?php echo $tr_class; ?>" tabindex="0">
+<tr id="Attribute_<?= $objectId ?>_tr" class="<?php echo $tr_class; ?>" tabindex="0">
   <?php
     if (($mayModify || !empty($extended)) && empty($disable_multi_select)):
   ?>
-      <td style="width:10px;" data-position="<?php echo 'attribute_' . h($object['id']); ?>">
+      <td style="width:10px;" data-position="<?php echo 'attribute_' . $objectId ?>">
       <?php
         if ($mayModify):
       ?>
-          <input id = "select_<?php echo $object['id']; ?>" class="select_attribute row_checkbox" type="checkbox" data-id="<?php echo $object['id'];?>" aria-label="<?php echo __('Select attribute');?>" />
+          <input id="select_<?= $objectId ?>" class="select_attribute row_checkbox" type="checkbox" data-id="<?= $objectId ?>" aria-label="<?php echo __('Select attribute');?>" />
       <?php
         endif;
       ?>
@@ -49,11 +65,9 @@
     endif;
   ?>
     <td class="short context hidden">
-      <?php echo h($object['id']); ?>
-    </td>
-    <td class="short context hidden">
-      <?php echo h($object['uuid']); ?>
+      <?= $objectId ?>
     </td>
+    <td class="short context hidden uuid quickSelect"><?php echo h($object['uuid']); ?></td>
     <td class="short context hidden">
         <?php echo $this->element('/Events/View/seen_field', array('object' => $object)); ?>
     </td>
@@ -65,13 +79,10 @@
     ?>
       <td class="short">
         <?php
-            $event_info = '';
-            if (!empty($extended)) {
-                $event_info = sprintf('title="%s%s"',
-                    __('Event info') . ':&#10;     ',
-                    $object['event_id'] != $event['Event']['id'] ? h($event['extensionEvents'][$object['event_id']]['info']) : h($event['Event']['info'])
-                );
-            }
+            $event_info = sprintf('title="%s%s"',
+                __('Event info') . ':&#10;     ',
+                $object['event_id'] != $event['Event']['id'] ? h($event['extensionEvents'][$object['event_id']]['info']) : h($event['Event']['info'])
+            );
         ?>
         <?php echo '<a href="' . $baseurl . '/events/view/' . h($object['event_id']) . '" ' . $event_info . '>' . h($object['event_id']) . '</a>'; ?>
       </td>
@@ -91,13 +102,13 @@
       ?>
       &nbsp;
     </td>
-    <td class="short" onmouseenter="quickEditHover(this, '<?php echo $editScope; ?>', '<?php echo $object['id']; ?>', 'category', <?php echo $event['Event']['id'];?>);">
-      <div id = "Attribute_<?php echo $object['id']; ?>_category_placeholder" class = "inline-field-placeholder"></div>
-      <div id = "Attribute_<?php echo $object['id']; ?>_category_solid" class="inline-field-solid">
+    <td class="short"<?= $quickEdit('category') ?>>
+      <div id="Attribute_<?= $objectId ?>_category_placeholder" class="inline-field-placeholder"></div>
+      <div id="Attribute_<?= $objectId ?>_category_solid" class="inline-field-solid">
         <?php echo h($object['category']); ?>
       </div>
     </td>
-    <td class="short" onmouseenter="quickEditHover(this, '<?php echo $editScope; ?>', '<?php echo $object['id']; ?>', 'type', <?php echo $event['Event']['id'];?>);">
+    <td class="short"<?= $quickEdit('type') ?>>
       <?php
         if (!empty($object['object_relation'])):
       ?>
@@ -105,34 +116,26 @@
       <?php
         endif;
       ?>
-      <div></div>
-      <div id = "Attribute_<?php echo $object['id']; ?>_type_placeholder" class = "inline-field-placeholder"></div>
-      <div id = "Attribute_<?php echo $object['id']; ?>_type_solid" class="inline-field-solid">
+      <div id="Attribute_<?= $objectId ?>_type_placeholder" class="inline-field-placeholder"></div>
+      <div id="Attribute_<?= $objectId ?>_type_solid" class="inline-field-solid">
         <?php echo h($object['type']); ?>
       </div>
     </td>
-    <?php
-        if ('attachment' !== $object['type'] && 'malware-sample' !== $object['type']):
-            $editable = ' onmouseenter="quickEditHover(this, \'' . $editScope . '\', \'' . $object['id'] . '\', \'value\', \'' . $event['Event']['id'] . '\' );"';
-        else:
-            $editable = '';
-        endif;
-    ?>
-    <td id="Attribute_<?php echo h($object['id']); ?>_container" class="showspaces limitedWidth shortish" <?php echo $editable; ?>>
-    <div id="Attribute_<?php echo $object['id']; ?>_value_placeholder" class="inline-field-placeholder"></div>
-      <div id = "Attribute_<?php echo $object['id']; ?>_value_solid" class="inline-field-solid">
+    <td id="Attribute_<?= $objectId ?>_container" class="showspaces limitedWidth shortish"<?= $quickEdit('value') ?>>
+      <div id="Attribute_<?= $objectId ?>_value_placeholder" class="inline-field-placeholder"></div>
+      <div id="Attribute_<?= $objectId ?>_value_solid" class="inline-field-solid">
         <span>
         <?php
             $spanExtra = '';
             $popupButton = '';
             if (Configure::read('Plugin.Enrichment_hover_enable') && isset($modules) && isset($modules['hover_type'][$object['type']])) {
                 $commonDataFields = sprintf(
-                    'data-object-type="%s" data-object-id="%s"',
-                    "Attribute",
-                    h($object['id'])
+                    'data-object-type="Attribute" data-object-id="%s"',
+                    $objectId
                 );
+
                 $spanExtra = sprintf(' class="eventViewAttributeHover" %s', $commonDataFields);
-                $popupButton = sprintf('<i class="fa fa-search-plus useCursorPointer eventViewAttributePopup" %s></i>', $commonDataFields);
+                $popupButton = sprintf('<i class="fa fa-search-plus useCursorPointer eventViewAttributePopup noPrint" title="%s" %s></i>', __('Show hover enrichment'), $commonDataFields);
             }
             echo sprintf(
                 '<span%s style="white-space: pre-wrap;">%s</span> %s',
@@ -144,21 +147,17 @@
         </span>
         <?php
           if (isset($object['warnings'])) {
-            $temp = '';
-            $components = array(1 => 0, 2 => 1);
-            $valueParts = explode('|', $object['value']);
-            foreach ($components as $component => $valuePart) {
-              if (isset($object['warnings'][$component]) && isset($valueParts[$valuePart])) {
-                foreach ($object['warnings'][$component] as $warning) $temp .= '<span class=\'bold\'>' . h($valueParts[$valuePart]) . '</span>: <span class=\'red\'>' . h($warning) . '</span><br />';
+              $temp = '';
+              foreach ($object['warnings'] as $warning) {
+                  $temp .= '<span class="bold">' . h($warning['match']) . ':</span> <span class="red">' . h($warning['warninglist_name']) . '</span><br>';
               }
-            }
             echo ' <span aria-label="' . __('warning') . '" role="img" tabindex="0" class="fa fa-exclamation-triangle" data-placement="right" data-toggle="popover" data-content="' . h($temp) . '" data-trigger="hover" data-placement="right">&nbsp;</span>';
           }
         ?>
       </div>
     </td>
     <td class="short">
-      <div class="attributeTagContainer" id="#Attribute_<?php echo h($object['id']);?>_tr .attributeTagContainer">
+      <div class="attributeTagContainer">
         <?php echo $this->element('ajaxTags', array('attributeId' => $object['id'], 'tags' => $object['AttributeTag'], 'tagAccess' => ($isSiteAdmin || $mayModify || $me['org_id'] == $event['Event']['org_id']), 'context' => $context, 'scope' => 'attribute', 'tagConflicts' => isset($object['tagConflicts']) ? $object['tagConflicts'] : array())); ?>
       </div>
     </td>
@@ -170,12 +169,12 @@
             }
             echo sprintf(
                 '<td class="shortish"><div %s>%s</div></td>',
-                'class="attributeRelatedTagContainer" id="#Attribute_' . h($object['id']) . 'Related_tr .attributeTagContainer"',
+                'class="attributeRelatedTagContainer" id="#Attribute_' . $objectId . 'Related_tr .attributeTagContainer"',
                 $element
             );
         }
     ?>
-    <td class="short" id="attribute_<?php echo h($object['id']); ?>_galaxy">
+    <td class="short" id="attribute_<?= $objectId ?>_galaxy">
       <?php
         echo $this->element('galaxyQuickViewMini', array(
           'mayModify' => $mayModify,
@@ -186,20 +185,20 @@
         ));
       ?>
     </td>
-    <td class="showspaces bitwider" onmouseenter="quickEditHover(this, '<?php echo $editScope; ?>', '<?php echo $object['id']; ?>', 'comment', <?php echo $event['Event']['id'];?>);">
-      <div id = "Attribute_<?php echo $object['id']; ?>_comment_placeholder" class = "inline-field-placeholder"></div>
-      <div id = "Attribute_<?php echo $object['id']; ?>_comment_solid" class="inline-field-solid">
+    <td class="showspaces bitwider"<?= $quickEdit('comment') ?>>
+      <div id="Attribute_<?= $objectId ?>_comment_placeholder" class="inline-field-placeholder"></div>
+      <div id="Attribute_<?= $objectId ?>_comment_solid" class="inline-field-solid">
         <?php echo nl2br(h($object['comment'])); ?>&nbsp;
       </div>
     </td>
     <td class="short" style="padding-top:3px;">
       <input
-        id="correlation_toggle_<?php echo h($object['id']); ?>"
+        id="correlation_toggle_<?= $objectId ?>"
         class="correlation-toggle"
         aria-label="<?php echo __('Toggle correlation');?>"
         title="<?php echo __('Toggle correlation');?>"
         type="checkbox"
-        data-attribute-id="<?php echo h($object['id']); ?>"
+        data-attribute-id="<?= $objectId ?>"
         <?php
           echo $object['disable_correlation'] ? '' : ' checked';
           echo ($mayChangeCorrelation && empty($event['Event']['disable_correlation'])) ? '' : ' disabled';
@@ -207,7 +206,7 @@
       >
     </td>
     <td class="shortish">
-      <ul class="inline" style="margin:0px;">
+      <ul class="inline" style="margin:0">
         <?php
           if (!empty($event['RelatedAttribute'][$object['id']])) {
             echo $this->element('Events/View/attribute_correlations', array(
@@ -220,7 +219,7 @@
       </ul>
     </td>
     <td class="shortish">
-      <ul class="inline" style="margin:0px;">
+      <ul class="inline" style="margin:0">
         <?php
             if (!empty($object['Feed'])) {
                 foreach ($object['Feed'] as $feed) {
@@ -238,10 +237,10 @@
                         $popover .= '<span class=\'bold black\'>' . Inflector::humanize(h($k)) . '</span>: <span class="blue">' . $v . '</span><br />';
                     }
                     $liContents = '';
-                    if ($isSiteAdmin) {
+                    if ($isSiteAdmin || $hostOrgUser) {
                         if ($feed['source_format'] == 'misp') {
                             $liContents .= sprintf(
-                                '<form action="%s/feeds/previewIndex/%s" method="post" style="margin:0px;line-height:auto;">%s%s</form>',
+                                '<form action="%s/feeds/previewIndex/%s" method="post" style="margin:0;line-height:auto;">%s%s</form>',
                                 $baseurl,
                                 h($feed['id']),
                                 sprintf(
@@ -273,7 +272,7 @@
                         );
                     }
                     echo sprintf(
-                        '<li style="padding-right: 0px; padding-left:0px;">%s</li>',
+                        '<li style="padding-right: 0; padding-left:0;">%s</li>',
                         $liContents
                     );
                 }
@@ -311,7 +310,7 @@
                             );
                         }
                         echo sprintf(
-                            '<li style="padding-right: 0px; padding-left:0px;">%s</li>',
+                            '<li style="padding-right:0; padding-left:0;">%s</li>',
                             $liContents
                         );
                     }
@@ -321,20 +320,20 @@
       </ul>
     </td>
     <td class="short">
-        <div id = "Attribute_<?php echo $object['id']; ?>_to_ids_placeholder" class = "inline-field-placeholder"></div>
-        <div id = "Attribute_<?php echo $object['id']; ?>_to_ids_solid" class="inline-field-solid">
-            <input type="checkbox" class="toids-toggle" id="toids_toggle_<?php echo h($object['id']); ?>" data-attribute-id="<?php echo h($object['id']); ?>" aria-label="<?php echo __('Toggle IDS flag');?>" title="<?php echo __('Toggle IDS flag');?>" <?php echo $object['to_ids'] ? 'checked' : ''; ?> >
+        <div id="Attribute_<?= $objectId ?>_to_ids_placeholder" class="inline-field-placeholder"></div>
+        <div id="Attribute_<?= $objectId ?>_to_ids_solid" class="inline-field-solid">
+            <input type="checkbox" class="toids-toggle" id="toids_toggle_<?= $objectId ?>" data-attribute-id="<?= $objectId ?>" aria-label="<?= __('Toggle IDS flag') ?>" title="<?= __('Toggle IDS flag') ?>"<?= $object['to_ids'] ? ' checked' : ''; ?><?= $mayModify ? '' : ' disabled' ?>>
         </div>
     </td>
-    <td class="short" onmouseenter="quickEditHover(this, '<?php echo $editScope; ?>', '<?php echo $object['id']; ?>', 'distribution', <?php echo $event['Event']['id'];?>);">
+    <td class="short"<?= $quickEdit('distribution') ?>>
         <?php
             $turnRed = '';
             if ($object['distribution'] == 0) {
                 $turnRed = 'style="color:red"';
             }
         ?>
-        <div id = "Attribute_<?php echo $object['id']; ?>_distribution_placeholder" class = "inline-field-placeholder"></div>
-        <div id = "Attribute_<?php echo $object['id']; ?>_distribution_solid" <?php echo $turnRed; ?> class="inline-field-solid">
+        <div id="Attribute_<?= $objectId ?>_distribution_placeholder" class="inline-field-placeholder"></div>
+        <div id="Attribute_<?= $objectId ?>_distribution_solid" <?php echo $turnRed; ?> class="inline-field-solid">
             <?php
                 if ($object['distribution'] == 4):
             ?>
@@ -349,19 +348,15 @@
     <?php
         echo $this->element('/Events/View/sighting_field', array(
           'object' => $object,
-          'tr_class' => $tr_class,
-          'page' => $page
         ));
         if (!empty($includeSightingdb)) {
             echo $this->element('/Events/View/sightingdb_field', array(
               'object' => $object,
-              'tr_class' => $tr_class,
-              'page' => $page
             ));
         }
         if (!empty($includeDecayScore)): ?>
             <td class="decayingScoreField">
-                  <div id = "Attribute_<?php echo h($object['id']); ?>_score_solid" class="inline-field-solid">
+                  <div id="Attribute_<?= $objectId ?>_score_solid" class="inline-field-solid">
                     <?php echo $this->element('DecayingModels/View/attribute_decay_score', array('scope' => 'object', 'object' => $object, 'uselink' => true)); ?>
                   </div>
             </td>
@@ -373,25 +368,25 @@
         if ($object['deleted']):
           if ($isSiteAdmin || $mayModify):
       ?>
-          <span class="icon-repeat useCursorPointer" title="<?php echo __('Restore attribute');?>" role="button" tabindex="0" aria-label="<?php echo __('Restore attribute');?>" onClick="deleteObject('attributes', 'restore', '<?php echo h($object['id']); ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
-          <span class="fa fa-trash useCursorPointer" title="<?php echo __('Permanently delete attribute');?>" role="button" tabindex="0" aria-label="i<?php echo __('Permanently delete attribute');?>" onClick="deleteObject('attributes', 'delete', '<?php echo h($object['id']) . '/true'; ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
+          <span class="fas fa-redo useCursorPointer" title="<?php echo __('Restore attribute');?>" role="button" tabindex="0" aria-label="<?php echo __('Restore attribute');?>" onClick="deleteObject('attributes', 'restore', '<?= $objectId ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
+          <span class="fa fa-trash useCursorPointer" title="<?php echo __('Permanently delete attribute');?>" role="button" tabindex="0" aria-label="i<?php echo __('Permanently delete attribute');?>" onClick="deleteObject('attributes', 'delete', '<?= $objectId . '/true'; ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
       <?php
           endif;
         else:
           if ($isAclAdd && ($isSiteAdmin || !$mayModify)):
             if (isset($modules) && isset($modules['types'][$object['type']])):
       ?>
-        <span class="icon-asterisk useCursorPointer" title="<?php echo __('Query enrichment');?>" role="button" tabindex="0" aria-label="<?php echo __('Query enrichment');?>" onClick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?php echo h($object['id']);?>/ShadowAttribute');" title="<?php echo __('Propose enrichment');?>">&nbsp;</span>
+        <span class="fas fa-asterisk useCursorPointer" role="button" tabindex="0" aria-label="<?php echo __('Query enrichment');?>" onClick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/ShadowAttribute');" title="<?php echo __('Propose enrichment');?>">&nbsp;</span>
       <?php
             endif;
             if (isset($cortex_modules) && isset($cortex_modules['types'][$object['type']])):
       ?>
-        <span class="icon-eye-open useCursorPointer" title="<?php echo __('Query Cortex');?>" role="button" tabindex="0" aria-label="<?php echo __('Query Cortex');?>" onClick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?php echo h($object['id']);?>/ShadowAttribute/Cortex');" title="<?php echo __('Propose enrichment through Cortex');?>"></span>
+        <span class="icon-eye-open useCursorPointer" title="<?php echo __('Query Cortex');?>" role="button" tabindex="0" aria-label="<?php echo __('Query Cortex');?>" onClick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/ShadowAttribute/Cortex');" title="<?php echo __('Propose enrichment through Cortex');?>"></span>
       <?php
             endif;
       ?>
-            <a href="<?php echo $baseurl;?>/shadow_attributes/edit/<?php echo $object['id']; ?>" title="<?php echo __('Propose Edit');?>" aria-label="<?php echo __('Propose Edit');?>" class="fa fa-comment useCursorPointer"></a>
-            <span class="fa fa-trash useCursorPointer" title="<?php echo __('Propose Deletion');?>" role="button" tabindex="0" aria-label="Propose deletion" onClick="deleteObject('shadow_attributes', 'delete', '<?php echo h($object['id']); ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
+            <a href="<?php echo $baseurl;?>/shadow_attributes/edit/<?= $objectId ?>" title="<?php echo __('Propose Edit');?>" aria-label="<?php echo __('Propose Edit');?>" class="fa fa-comment useCursorPointer"></a>
+            <span class="fa fa-trash useCursorPointer" title="<?php echo __('Propose Deletion');?>" role="button" tabindex="0" aria-label="Propose deletion" onClick="deleteObject('shadow_attributes', 'delete', '<?= $objectId ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
       <?php
             if ($isSiteAdmin):
       ?>
@@ -401,24 +396,24 @@
           if ($isSiteAdmin || $mayModify):
             if (isset($modules) && isset($modules['types'][$object['type']])):
       ?>
-        <span class="icon-asterisk useCursorPointer" onClick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?php echo h($object['id']);?>/Attribute');" title="<?php echo __('Add enrichment');?>" role="button" tabindex="0" aria-label="<?php echo __('Add enrichment');?>">&nbsp;</span>
+        <span class="fas fa-asterisk useCursorPointer" onClick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/Attribute');" title="<?php echo __('Add enrichment');?>" role="button" tabindex="0" aria-label="<?php echo __('Add enrichment');?>">&nbsp;</span>
       <?php
             endif;
             if (isset($cortex_modules) && isset($cortex_modules['types'][$object['type']])):
       ?>
-        <span class="icon-eye-open useCursorPointer" onClick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?php echo h($object['id']);?>/Attribute/Cortex');" title="<?php echo __('Add enrichment');?>" role="button" tabindex="0" aria-label="<?php echo __('Add enrichment via Cortex');?>"></span>
+        <span class="icon-eye-open useCursorPointer" onClick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/Attribute/Cortex');" title="<?php echo __('Add enrichment');?>" role="button" tabindex="0" aria-label="<?php echo __('Add enrichment via Cortex');?>"></span>
       <?php
             endif;
       ?>
-            <a href="<?php echo $baseurl;?>/attributes/edit/<?php echo $object['id']; ?>" title="<?php echo __('Edit');?>" aria-label="<?php echo __('Edit');?>" class="fa fa-edit useCursorPointer"></a>
+            <a href="<?php echo $baseurl;?>/attributes/edit/<?= $objectId ?>" title="<?php echo __('Edit');?>" aria-label="<?php echo __('Edit');?>" class="fa fa-edit useCursorPointer"></a>
           <?php
             if (empty($event['Event']['publish_timestamp'])):
           ?>
-            <span class="fa fa-trash useCursorPointer" title="<?php echo __('Permanently delete attribute');?>" role="button" tabindex="0" aria-label="i<?php echo __('Permanently delete attribute');?>" onClick="deleteObject('attributes', 'delete', '<?php echo h($object['id']) . '/true'; ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
+            <span class="fa fa-trash useCursorPointer" title="<?php echo __('Permanently delete attribute');?>" role="button" tabindex="0" aria-label="i<?php echo __('Permanently delete attribute');?>" onClick="deleteObject('attributes', 'delete', '<?= $objectId . '/true'; ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
           <?php
             else:
           ?>
-            <span class="fa fa-trash useCursorPointer" title="<?php echo __('Soft-delete attribute');?>" role="button" tabindex="0" aria-label="<?php echo __('Soft-delete attribute');?>" onClick="deleteObject('attributes', 'delete', '<?php echo h($object['id']); ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
+            <span class="fa fa-trash useCursorPointer" title="<?php echo __('Soft-delete attribute');?>" role="button" tabindex="0" aria-label="<?php echo __('Soft-delete attribute');?>" onClick="deleteObject('attributes', 'delete', '<?= $objectId ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
           <?php
             endif;
           endif;
@@ -427,19 +422,18 @@
   </td>
 </tr>
 <?php
-  if (!empty($object['ShadowAttribute'])) {
+if (!empty($object['ShadowAttribute'])) {
     end($object['ShadowAttribute']);
     $lastElement = key($object['ShadowAttribute']);
     foreach ($object['ShadowAttribute'] as $propKey => $proposal) {
-      echo $this->element('/Events/View/row_' . $proposal['objectType'], array(
-        'object' => $proposal,
-        'mayModify' => $mayModify,
-        'mayChangeCorrelation' => $mayChangeCorrelation,
-        'page' => $page,
-        'fieldCount' => $fieldCount,
-        'child' => $propKey == $lastElement ? 'last' : true,
-        'objectContainer' => $child
-      ));
+        echo $this->element('/Events/View/row_' . $proposal['objectType'], array(
+            'object' => $proposal,
+            'mayModify' => $mayModify,
+            'mayChangeCorrelation' => $mayChangeCorrelation,
+            'fieldCount' => $fieldCount,
+            'child' => $propKey == $lastElement ? 'last' : true,
+            'objectContainer' => $child
+        ));
     }
-  }
-?>
+}
+

app/View/Elements/Events/View/row_object.ctp

@@ -1,5 +1,4 @@
 <?php
-  $tr_class = '';
   $linkClass = 'white';
   $currentType = 'Object';
   $tr_class = 'tableHighlightBorderTop borderBlue';
@@ -18,24 +17,14 @@
   <?php
     if ($mayModify || $extended):
   ?>
-    <td style="width:10px;" data-position="<?php echo h($object['objectType']) . '_' . h($object['id']); ?>">
-      <?php
-        if ($mayModify):
-      ?>
-        <input id="select_object_<?php echo $object['id']; ?>" class="select_object row_checkbox" type="checkbox" data-id="<?php echo $object['id'];?>" />
-      <?php
-        endif;
-      ?>
-    </td>
+    <td style="width:10px;"></td>
   <?php
     endif;
   ?>
   <td class="short context hidden">
     <?php echo h($object['id']); ?>
   </td>
-  <td class="short context hidden">
-    <?php echo h($object['uuid']); ?>
-  </td>
+  <td class="short context hidden uuid quickSelect"><?php echo h($object['uuid']); ?></td>
   <td class="short context hidden">
       <?php echo $this->element('/Events/View/seen_field', array('object' => $object)); ?>
   </td>
@@ -99,12 +88,11 @@
   <td class="shortish" onmouseenter="quickEditHover(this, 'Object', '<?php echo $object['id']; ?>', 'distribution', <?php echo $event['Event']['id'];?>);">
     <?php
       $turnRed = '';
-      if ($object['objectType'] == 0 && $object['distribution'] == 0) $turnRed = 'style="color:red"';
+      if ($object['distribution'] == 0) $turnRed = 'style="color:red"';
     ?>
     <div id="<?php echo $currentType . '_' . $object['id'] . '_distribution_placeholder'; ?>" class="inline-field-placeholder"></div>
     <div id="<?php echo $currentType . '_' . $object['id'] . '_distribution_solid'; ?>" <?php echo $turnRed; ?> class="inline-field-solid">
       <?php
-        if ($object['objectType'] == 0) {
           if ($object['distribution'] == 4):
       ?>
         <a href="<?php echo $baseurl; ?>/sharing_groups/view/<?php echo h($object['sharing_group_id']); ?>"><?php echo h($object['SharingGroup']['name']);?></a>
@@ -112,7 +100,6 @@
           else:
             echo h($shortDist[$object['distribution']]);
           endif;
-        }
       ?>&nbsp;
     </div>
   </td>
@@ -130,9 +117,11 @@
     <?php
       if ($mayModify && empty($object['deleted'])) {
         echo sprintf(
-          '<a href="%s/objects/edit/%s" title="Edit" aria-label="Edit" class="fa fa-edit white useCursorPointer"></a>',
+          '<a href="%s/objects/edit/%s" title="%s" aria-label="%s" class="fa fa-edit white useCursorPointer"></a> ',
           $baseurl,
-          h($object['id'])
+          h($object['id']),
+          __('Edit'),
+          __('Edit')
         );
         echo sprintf(
           '<span class="fa fa-trash white useCursorPointer" title="%1$s" role="button" tabindex="0" aria-label="%1$s" onClick="%2$s"></span>',
@@ -156,7 +145,6 @@
         'object' => $attribute,
         'mayModify' => $mayModify,
         'mayChangeCorrelation' => $mayChangeCorrelation,
-        'page' => $page,
         'fieldCount' => $fieldCount,
         'child' => $attrKey == $lastElement ? 'last' : true
       ));

app/View/Elements/Events/View/row_object_reference.ctp

@@ -32,7 +32,7 @@
 <div id="Object_<?php echo $object['id']; ?>_references_collapsible" class="collapse">
 <?php
   foreach ($object['ObjectReference'] as $reference):
-    if (!empty($reference['Object'])) {
+    if (isset($reference['Object'])) {
       $uuid = $reference['Object']['uuid'];
       $output = ' (' . $reference['Object']['name'] . ': ' . $reference['Object']['name'] . ')';
       $objectType = 'Object';
@@ -43,13 +43,12 @@
     }
     $uuid = empty($reference['Object']) ? $reference['Attribute']['uuid'] : $reference['Object']['uuid'];
     $idref = $reference['deleted'] ? $reference['id'] . '/1' : $reference['id'];
+    $linkText = h($reference['relationship_type']) . ' ' . $objectType . ' ' . $reference['referenced_id'] . h($output);
 ?>
     &nbsp;&nbsp;
-    <a class="bold white useCursorPointer <?php echo $reference['deleted'] ? 'strikethrough' : ''; ?>" onClick="pivotObjectReferences('<?php echo h($currentUri); ?>', '<?php echo $uuid; ?>')">
-      <?php echo h($reference['relationship_type']) . ' ' . $objectType . ' ' . $reference['referenced_id'] . h($output);?>
-    </a>
+    <a class="bold white useCursorPointer<?= $reference['deleted'] ? ' strikethrough' : ''; ?>" onclick="pivotObjectReferences('<?= h($currentUri) ?>', '<?= $uuid ?>')"><?= $linkText ?></a>
     <span class="fa fa-trash icon-white useCursorPointer" title="<?php echo __('Delete object reference');?>" role="button" tabindex="0" aria-label="<?php echo __('Delete object reference');?>" onClick="deleteObject('object_references', 'delete', '<?php echo h($idref); ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
-    <br />
+    <br>
 <?php
   endforeach;
 ?>