Merge remote-tracking branch 'origin/2.4' into galaxy-cluster2.0

.travis.yml

@@ -146,6 +146,7 @@ before_script:
     - popd
 
 script:
+    - ./app/Vendor/bin/parallel-lint --exclude app/Lib/cakephp/ --exclude app/Vendor/ --exclude app/Lib/random_compat/ -e php,ctp app/
     - ./app/Vendor/bin/phpunit app/Test/ComplexTypeToolTest.php
     - pushd tests
     - ./curl_tests.sh $AUTH
@@ -153,7 +154,7 @@ script:
     - pushd PyMISP
     - git submodule init
     - git submodule update
-    - poetry install -E fileobjects -E openioc -E virustotal -E docs -E pdfexport
+    - travis_retry poetry install -E fileobjects -E openioc -E virustotal -E docs -E pdfexport
     - poetry run python tests/testlive_comprehensive.py
     - poetry run python tests/test_mispevent.py
     - popd
@@ -161,12 +162,9 @@ script:
     - pushd PyMISP/examples/events/
     - poetry run python ./create_massive_dummy_events.py -l 5 -a 30
     - popd
-    - pushd app/files/feed-metadata
-    - jsonschema -i defaults.json schema.json
-    - popd
+    - python3 tools/misp-feed/validate.py
 
 after_failure:
-    - ls -aRl `pwd`
     - curl http://misp.local
     - cat /etc/apache2/sites-available/misp.local.conf
     - sudo tail -n +1 `pwd`/app/tmp/logs/*

AUTHORS

@@ -3,11 +3,13 @@ Developers
 
 * Christophe Vandeplas <christophe@vandeplas.com> (original author)
 * Andras Iklody <andras.iklody@gmail.com> (lead developer)
+* Many more
 
 Contributors
 ------------
 
 Aaron Kaplan
+Andreas Ziegler
 Airbus Group CERT (AiG CERT)
 Alexander Jaeger
 Alexandre Dulaunoy
@@ -17,25 +19,35 @@ Andreas Ziegler
 Andrzej Dereszowski
 Bâkır Emre
 Chris Clark
+Christian Studer
 Christophe Vandeplas
 David André
 Guilherme Capilé
 Gábor Molnár
 Iglocska
+Jakub Onderka
 Koen Van Impe
 L. Aaron Kaplan
 Noud de Brouwer
 Raphaël Vinot
 Richard van den Berg
+Sami Mokaddem
+Steve Clement
 nullprobe
 remg427
 
-Copyright (C) 2012 Christophe Vandeplas
-Copyright (C) 2012 Belgian Defence
-Copyright (C) 2012 NATO / NCIRC
-Copyright (C) 2013-2018 Andras Iklody
-Copyright (C) 2015-2018 CIRCL - Computer Incident Response Center Luxembourg
-Copyright (C) 2016 Andreas Ziegler
+
+* Copyright (C) 2012-2020 Christophe Vandeplas
+* Copyright (C) 2012 Belgian Defence
+* Copyright (C) 2012 NATO / NCIRC
+* Copyright (C) 2013-2020 Andras Iklody
+* Copyright (C) 2015-2020 CIRCL - Computer Incident Response Center Luxembourg
+* Copyright (C) 2016 Andreas Ziegler
+* Copyright (C) 2018-2020 Sami Mokaddem
+* Copyright (C) 2018-2020 Christian Studer
+* Copyright (C) 2015-2020 Alexandre Dulaunoy
+* Copyright (C) 2018-2020 Steve Clement
+* Copyright (C) 2020 Jakub Onderka
 
 MISP is licensed under the GNU AFFERO GENERAL PUBLIC LICENSE version 3.
 

PyMISP

@@ -1 +1 @@
-Subproject commit 0220f25f98c46fecf66152e9f0167cb45981c012
+Subproject commit 0d67babea2ba967f924978023159067aa88de2eb

README.md

@@ -104,11 +104,16 @@ License
 
 This software is licensed under [GNU Affero General Public License version 3](http://www.gnu.org/licenses/agpl-3.0.html)
 
-* Copyright (C) 2012 Christophe Vandeplas
+* Copyright (C) 2012-2020 Christophe Vandeplas
 * Copyright (C) 2012 Belgian Defence
 * Copyright (C) 2012 NATO / NCIRC
-* Copyright (C) 2013-2019 Andras Iklody
-* Copyright (C) 2015-2019 CIRCL - Computer Incident Response Center Luxembourg
+* Copyright (C) 2013-2020 Andras Iklody
+* Copyright (C) 2015-2020 CIRCL - Computer Incident Response Center Luxembourg
 * Copyright (C) 2016 Andreas Ziegler
+* Copyright (C) 2018-2020 Sami Mokaddem
+* Copyright (C) 2018-2020 Christian Studer
+* Copyright (C) 2015-2020 Alexandre Dulaunoy
+* Copyright (C) 2018-2020 Steve Clement
+* Copyright (C) 2020 Jakub Onderka
 
 For more information, [the list of authors and contributors](AUTHORS) is available.

VERSION.json

@@ -1 +1 @@
-{"major":2, "minor":4, "hotfix":132}
+{"major":2, "minor":4, "hotfix":134}

app/Config/config.default.php

@@ -22,6 +22,7 @@ $config = array(
 			'email_subject_include_tag_name' => true,
 			'background_jobs'                => true,
 			'cached_attachments'             => true,
+			'osuser'                         => 'www-data',
 			'email'                          => 'email@address.com',
 			'contact'                        => 'email@address.com',
 			'cveurl'                         => 'https://cve.circl.lu/cve/',

app/Config/core.default.php

@@ -181,9 +181,16 @@ Configure::write('Session', array(
 	'cookie_timeout' => 10080 ,  // Cookie timeout, default is 1 week
 	'defaults'       => 'php',
 	'autoRegenerate' => false,
-	'checkAgent' => false
+	'checkAgent' => false,
 ));
 
+// Set session cookie SameSite parameter to Lax if this param is supported and no default value is set
+if (PHP_VERSION_ID >= 70300 && empty(ini_get('session.cookie_samesite'))) {
+    Configure::write('Session.ini', [
+        'session.cookie_samesite' => 'Lax'
+    ]);
+}
+
 /**
  * The level of CakePHP security.
  */

app/Console/Command/AdminShell.php

@@ -351,13 +351,14 @@ class AdminShell extends AppShell
     {
         $this->ConfigLoad->execute();
         $whoami = exec('whoami');
-        if ($whoami === 'httpd' || $whoami === 'www-data' || $whoami === 'apache' || $whoami === 'wwwrun' || $whoami === 'travis' || $whoami === 'www') {
+        $osuser = Configure::read('MISP.osuser');
+        if ($whoami === 'httpd' || $whoami === 'www-data' || $whoami === 'apache' || $whoami === 'wwwrun' || $whoami === 'travis' || $whoami === 'www' || $whoami === $osuser) {
             echo 'Executing all updates to bring the database up to date with the current version.' . PHP_EOL;
             $processId = empty($this->args[0]) ? false : $this->args[0];
             $this->Server->runUpdates(true, false, $processId);
             echo 'All updates completed.' . PHP_EOL;
         } else {
-            die('This OS user is not allowed to run this command.'. PHP_EOL. 'Run it under `www-data` or `httpd` or `apache` or `wwwrun`.' . PHP_EOL . 'You tried to run this command as: ' . $whoami . PHP_EOL);
+            die('This OS user is not allowed to run this command.'. PHP_EOL. 'Run it under `www-data` or `httpd` or `apache` or `wwwrun` or set MISP.osuser in the configuration.' . PHP_EOL . 'You tried to run this command as: ' . $whoami . PHP_EOL);
         }
     }
 
@@ -582,7 +583,7 @@ class AdminShell extends AppShell
                 'db_version' => $dbVersion
             );
             $file = new File(ROOT . DS . 'db_schema.json', true);
-            $file->write(json_encode($data, JSON_PRETTY_PRINT));
+            $file->write(json_encode($data, JSON_PRETTY_PRINT) . "\n");
             $file->close();
             echo __("> Database schema dumped on disk") . PHP_EOL;
         } else {
@@ -639,4 +640,19 @@ class AdminShell extends AppShell
             PHP_EOL, PHP_EOL, $ip, PHP_EOL, PHP_EOL, $user['User']['id'], $user['User']['email'], PHP_EOL, PHP_EOL
         );
     }
+
+    public function scanAttachment()
+    {
+        $input = $this->args[0];
+        $attributeId = isset($this->args[1]) ? $this->args[1] : null;
+        $jobId = isset($this->args[2]) ? $this->args[2] : null;
+
+        $this->loadModel('AttachmentScan');
+        $result = $this->AttachmentScan->scan($input, $attributeId, $jobId);
+        if ($result === false) {
+            echo 'Job failed' . PHP_EOL;
+        } else {
+            echo $result . PHP_EOL;
+        }
+    }
 }

app/Console/Command/ServerShell.php

@@ -36,10 +36,15 @@ class ServerShell extends AppShell
         }
 
         $serverId = intval($this->args[0]);
-        $res = @$this->Server->runConnectionTest($serverId);
-        if (!empty($res['message']))
-            $res['message'] = json_decode($res['message']);
+        $server = $this->Server->find('first', [
+            'conditions' => ['Server.id' => $serverId],
+            'recursive' => -1,
+        ]);
+        if (!$server) {
+            die("Server with ID $serverId doesn't exists.");
+        }
 
+        $res = @$this->Server->runConnectionTest($server);
         echo json_encode($res) . PHP_EOL;
     }
 

app/Console/Command/StatisticsShell.php

@@ -1,6 +1,8 @@
 <?php
 class StatisticsShell extends AppShell {
 
+    public $uses = array('Event', 'User', 'Organisation', 'Log');
+
     public function contributors()
     {
         $from = empty($this->args[0]) ? null : $this->args[0];
@@ -133,4 +135,120 @@ class StatisticsShell extends AppShell {
         }
         echo "==================================\n\n";
     }
+
+    public function orgEngagement()
+    {
+        $orgs = $this->Organisation->find('list', [
+            'recursive' => -1,
+            'fields' => ['Organisation.id', 'Organisation.id'],
+            'conditions' => ['Organisation.local' => 1]
+        ]);
+        $orgs = array_values($orgs);
+        $total_orgs = count($orgs);
+        $data = [];
+        $orgCreations = $this->Log->find('list', [
+            'conditions' => [
+                'model' => 'Organisation',
+                'action' => 'add'
+            ],
+            'fields' => ['Log.model_id', 'Log.created']
+        ]);
+        $localOrgs = $this->Organisation->find('count', [
+            'conditions' => [
+                'Organisation.local' => 1
+            ]
+        ]);
+        foreach ($orgs as $k => $org) {
+            echo sprintf(__('Processing organisation %s / %s.%s', $k+1, $total_orgs, PHP_EOL));
+            $temp = [
+                'org_id' => $org
+            ];
+            if (empty($orgCreations[$org])) {
+                continue;
+            } else {
+                $temp['org_creation_timestamp'] = strtotime($orgCreations[$org]);
+            }
+            $first_event = $this->Event->find('first', [
+                'recursive' => -1,
+                'conditions' => [
+                    'orgc_id' => $org
+                ],
+                'order' => ['Event.id ASC'],
+                'fields' => ['Event.id']
+            ]);
+            if (empty($first_event)) {
+                continue;
+            }
+            $first_event_creation = $this->Log->find('first', [
+                'recursive' => -1,
+                'conditions' => [
+                    'model_id' => $first_event['Event']['id'],
+                    'model' => 'Event',
+                    'action' => 'add'
+                ]
+            ]);
+            if (empty($first_event_creation)) {
+                continue;
+            }
+            $temp['first_event_creation'] = strtotime($first_event_creation['Log']['created']);
+            $temp['time_until_first_event'] = $temp['first_event_creation'] - $temp['org_creation_timestamp'];
+            $data[] = $temp;
+        }
+        $average_time_to_first_event = 0;
+        foreach ($data as $org_data) {
+            $average_time_to_first_event += (int)$org_data['time_until_first_event'] / 60 / 60 / 24;
+        }
+        echo PHP_EOL . str_repeat('-', 63) . PHP_EOL;
+        echo __('Total local orgs: %s%s', $localOrgs, PHP_EOL);
+        echo __('Local orgs with event creations: %s%s', count($data), PHP_EOL);
+        echo __('Average days until first event: %s', (int)($average_time_to_first_event / count($data)));
+        echo PHP_EOL . str_repeat('-', 63) . PHP_EOL;
+    }
+
+    public function yearlyOrgGrowth()
+    {
+        $orgCreations = $this->Log->find('list', [
+            'conditions' => [
+                'model' => 'Organisation',
+                'action' => 'add'
+            ],
+            'fields' => ['Log.model_id', 'Log.created']
+        ]);
+        $localOnly = empty($this->args[0]) ? false : true;
+        if ($localOnly) {
+            $orgs = $this->Organisation->find('list', [
+                'recursive' => -1,
+                'fields' => ['Organisation.id', 'Organisation.local']
+            ]);
+            foreach ($orgs as $org_id => $local) {
+                if (!$local && isset($orgCreations[$org_id])) {
+                    unset($orgCreations[$org_id]);
+                }
+            }
+        }
+        $years = [];
+        foreach ($orgCreations as $orgCreation) {
+            $year = substr($orgCreation, 0, 4);
+            if (empty($years[$year])) {
+                $years[$year] = 0;
+            }
+            $years[$year] += 1;
+        }
+        ksort($years);
+        $yearOverYear = [];
+        $previous = 0;
+        echo PHP_EOL . str_repeat('-', 63) . PHP_EOL;
+        echo __('Year over year growth of organisation count.');
+        echo PHP_EOL . str_repeat('-', 63) . PHP_EOL;
+        $currentYear = date("Y");
+        foreach ($years as $year => $count) {
+            $prognosis = '';
+            if ($year == $currentYear) {
+                $percentage_passed = (strtotime(($year +1) . '-01-01') - strtotime(($year) . '-01-01')) / (time() - (strtotime($year . '-01-01')));
+                $prognosis = sprintf(' (%s by the end of the year at current rate)', round($percentage_passed * $count));
+            }
+            echo __('%s: %s %s%s', $year, $count - $previous, $prognosis, PHP_EOL);
+        }
+        echo str_repeat('-', 63) . PHP_EOL;
+    }
 }

app/Controller/AppController.php

@@ -37,6 +37,8 @@ App::uses('RequestRearrangeTool', 'Tools');
  * @link http://book.cakephp.org/2.0/en/controllers.html#the-app-controller
  *
  * @throws ForbiddenException // TODO Exception
+ * @property ACLComponent $ACL
+ * @property RestResponseComponent $RestResponse
  */
 class AppController extends Controller
 {
@@ -46,8 +48,8 @@ class AppController extends Controller
 
     public $helpers = array('Utility', 'OrgImg', 'FontAwesome', 'UserName', 'DataPathCollector');
 
-    private $__queryVersion = '111';
-    public $pyMispVersion = '2.4.131';
+    private $__queryVersion = '117';
+    public $pyMispVersion = '2.4.134';
     public $phpmin = '7.2';
     public $phprec = '7.4';
     public $pythonmin = '3.6';
@@ -462,15 +464,10 @@ class AppController extends Controller
             $this->set('isAclZmq', isset($role['perm_publish_zmq']) ? $role['perm_publish_zmq'] : false);
             $this->set('isAclKafka', isset($role['perm_publish_kafka']) ? $role['perm_publish_kafka'] : false);
             $this->set('isAclDecaying', isset($role['perm_decaying']) ? $role['perm_decaying'] : false);
+            $this->set('aclComponent', $this->ACL);
             $this->userRole = $role;
 
             $this->set('loggedInUserName', $this->__convertEmailToName($this->Auth->user('email')));
-            if ($this->request->params['controller'] === 'users' && $this->request->params['action'] === 'dashboard') {
-                $notifications = $this->{$this->modelClass}->populateNotifications($this->Auth->user());
-            } else {
-                $notifications = $this->{$this->modelClass}->populateNotifications($this->Auth->user(), 'fast');
-            }
-            $this->set('notifications', $notifications);
 
             if (
                 Configure::read('MISP.log_paranoid') ||
@@ -541,17 +538,28 @@ class AppController extends Controller
             }
         }
         $this->components['RestResponse']['sql_dump'] = $this->sql_dump;
-        $this->loadModel('UserSetting');
-        $homepage = $this->UserSetting->find('first', array(
-            'recursive' => -1,
-            'conditions' => array(
-                'UserSetting.user_id' => $this->Auth->user('id'),
-                'UserSetting.setting' => 'homepage'
-            ),
-            'contain' => array('User.id', 'User.org_id')
-        ));
-        if (!empty($homepage)) {
-            $this->set('homepage', $homepage['UserSetting']['value']);
+
+        // Notifications and homepage is not necessary for AJAX or REST requests
+        if ($this->Auth->user() && !$this->_isRest() && !$this->request->is('ajax')) {
+            if ($this->request->params['controller'] === 'users' && $this->request->params['action'] === 'dashboard') {
+                $notifications = $this->{$this->modelClass}->populateNotifications($this->Auth->user());
+            } else {
+                $notifications = $this->{$this->modelClass}->populateNotifications($this->Auth->user(), 'fast');
+            }
+            $this->set('notifications', $notifications);
+
+            $this->loadModel('UserSetting');
+            $homepage = $this->UserSetting->find('first', array(
+                'recursive' => -1,
+                'conditions' => array(
+                    'UserSetting.user_id' => $this->Auth->user('id'),
+                    'UserSetting.setting' => 'homepage'
+                ),
+                'contain' => array('User.id', 'User.org_id')
+            ));
+            if (!empty($homepage)) {
+                $this->set('homepage', $homepage['UserSetting']['value']);
+            }
         }
     }
 
@@ -1094,6 +1102,7 @@ class AppController extends Controller
                 if ($user['User']) {
                     unset($user['User']['gpgkey']);
                     unset($user['User']['certif_public']);
+                    $this->User->updateLoginTimes($user['User']);
                     $this->Session->renew();
                     $this->Session->write(AuthComponent::$sessionKey, $user['User']);
                     if (Configure::read('MISP.log_auth')) {
@@ -1182,6 +1191,7 @@ class AppController extends Controller
         if ($this->Auth->startup($this)) {
             $user = $this->Auth->user();
             if ($user) {
+                $this->User->updateLoginTimes($user);
                 // User found in the db, add the user info to the session
                 $this->Session->renew();
                 $this->Session->write(AuthComponent::$sessionKey, $user);

app/Controller/AttributesController.php

@@ -158,7 +158,6 @@ class AttributesController extends AppController
             if (!isset($attributes[0])) {
                 $attributes = array(0 => $attributes);
             }
-            $this->Warninglist = ClassRegistry::init('Warninglist');
             $fails = array();
             $successes = 0;
             $attributeCount = count($attributes);
@@ -957,6 +956,14 @@ class AttributesController extends AppController
 
     public function view($id)
     {
+        if ($this->request->is('head')) { // Just check if attribute exists
+            $attribute = $this->Attribute->fetchAttributesSimple($this->Auth->user(), [
+                'conditions' => $this->__idToConditions($id),
+                'fields' => ['Attribute.id'],
+            ]);
+            return new CakeResponse(['status' => $attribute ? 200 : 404]);
+        }
+
         $attribute = $this->__fetchAttribute($id);
         if (empty($attribute)) {
             throw new MethodNotAllowedException(__('Invalid attribute'));
@@ -1622,6 +1629,7 @@ class AttributesController extends AppController
         $this->Feed = ClassRegistry::init('Feed');
 
         $this->loadModel('Sighting');
+        $this->loadModel('AttachmentScan');
         $user = $this->Auth->user();
         foreach ($attributes as $k => $attribute) {
             $attributeId = $attribute['Attribute']['id'];
@@ -1634,6 +1642,10 @@ class AttributesController extends AppController
                 }
                 $attributes[$k] = $attribute;
             }
+            if ($attribute['Attribute']['type'] === 'attachment' && $this->AttachmentScan->isEnabled()) {
+                $infected = $this->AttachmentScan->isInfected(AttachmentScan::TYPE_ATTRIBUTE, $attribute['Attribute']['id']);
+                $attributes[$k]['Attribute']['infected'] = $infected;
+            }
 
             if ($attribute['Attribute']['distribution'] == 4) {
                 $attributes[$k]['Attribute']['SharingGroup'] = $attribute['SharingGroup'];
@@ -1947,15 +1959,19 @@ class AttributesController extends AppController
         if (!$this->request->is('ajax')) {
             throw new MethodNotAllowedException(__('This function can only be accessed via AJAX.'));
         }
+
+        $fieldsToFetch = ['id', $field];
+        if ($field === 'value') {
+            $fieldsToFetch[] = 'to_ids'; // for warninglist
+            $fieldsToFetch[] = 'type'; // for view
+            $fieldsToFetch[] = 'category'; // for view
+        }
+
         $params = array(
-                'conditions' => array('Attribute.id' => $id),
-                'fields' => array('id', 'distribution', 'event_id', $field),
-                'contain' => array(
-                        'Event' => array(
-                                'fields' => array('distribution', 'id', 'org_id'),
-                        )
-                ),
-                'flatten' => 1
+            'conditions' => array('Attribute.id' => $id),
+            'fields' => $fieldsToFetch,
+            'contain' => ['Event'],
+            'flatten' => 1,
         );
         $attribute = $this->Attribute->fetchAttributes($this->Auth->user(), $params);
         if (empty($attribute)) {
@@ -1973,8 +1989,14 @@ class AttributesController extends AppController
             } else {
                 echo '&nbsp';
             }
+        } elseif ($field === 'value') {
+            $this->loadModel('Warninglist');
+            $attribute['Attribute'] = $this->Warninglist->checkForWarning($attribute['Attribute']);
         }
+
         $this->set('value', $result);
+        $this->set('object', $attribute);
+        $this->set('field', $field);
         $this->layout = 'ajax';
         $this->render('ajax/attributeViewFieldForm');
     }
@@ -2403,23 +2425,20 @@ class AttributesController extends AppController
         if (empty($attribute)) {
             throw new NotFoundException(__('Invalid Attribute'));
         }
-        $this->loadModel('Server');
         $this->loadModel('Module');
         $modules = $this->Module->getEnabledModules($this->Auth->user());
         $validTypes = array();
         if (isset($modules['hover_type'][$attribute[0]['Attribute']['type']])) {
             $validTypes = $modules['hover_type'][$attribute[0]['Attribute']['type']];
         }
-        $url = Configure::read('Plugin.Enrichment_services_url') ? Configure::read('Plugin.Enrichment_services_url') : $this->Server->serverSettings['Plugin']['Enrichment_services_url']['value'];
-        $port = Configure::read('Plugin.Enrichment_services_port') ? Configure::read('Plugin.Enrichment_services_port') : $this->Server->serverSettings['Plugin']['Enrichment_services_port']['value'];
         $resultArray = array();
         foreach ($validTypes as $type) {
             $options = array();
             $found = false;
             foreach ($modules['modules'] as $temp) {
-                if ($temp['name'] == $type) {
+                if ($temp['name'] === $type) {
                     $found = true;
-                    $format = (isset($temp['mispattributes']['format']) ? $temp['mispattributes']['format'] : 'simplified');
+                    $format = isset($temp['mispattributes']['format']) ? $temp['mispattributes']['format'] : 'simplified';
                     if (isset($temp['meta']['config'])) {
                         foreach ($temp['meta']['config'] as $conf) {
                             $options[$conf] = Configure::read('Plugin.Enrichment_' . $type . '_' . $conf);
@@ -2443,33 +2462,35 @@ class AttributesController extends AppController
             } else {
                 $data[$attribute[0]['Attribute']['type']] = $attribute[0]['Attribute']['value'];
             }
-            $data = json_encode($data);
-            $result = $this->Module->queryModuleServer('/query', $data, true);
+            $result = $this->Module->queryModuleServer($data, true);
             if ($result) {
                 if (!is_array($result)) {
-                    $resultArray[$type][] = array($type => $result);
+                    $resultArray[$type] = ['error' => $result];
+                    continue;
                 }
             } else {
                 // TODO: i18n?
-                $resultArray[$type][] = array($type => 'Enrichment service not reachable.');
+                $resultArray[$type] = ['error' => 'Enrichment service not reachable.'];
                 continue;
             }
             $current_result = array();
             if (isset($result['results']['Object'])) {
                 if (!empty($result['results']['Object'])) {
                     $objects = array();
-                    foreach($result['results']['Object'] as $object) {
+                    foreach ($result['results']['Object'] as $object) {
                         if (isset($object['Attribute']) && !empty($object['Attribute'])) {
                             $object_attributes = array();
                             foreach($object['Attribute'] as $object_attribute) {
-                                array_push($object_attributes, array('object_relation' => $object_attribute['object_relation'], 'value' => $object_attribute['value']));
+                                $object_attributes[] = [
+                                    'object_relation' => $object_attribute['object_relation'],
+                                    'value' => $object_attribute['value'],
+                                    'type' => $object_attribute['type'],
+                                ];
                             }
-                            array_push($objects, array('name' => $object['name'], 'Attribute' => $object_attributes));
+                            $objects[] = array('name' => $object['name'], 'Attribute' => $object_attributes);
                         }
                     }
-                    if (!empty($objects)) {
-                        $current_result['Object'] = $objects;
-                    }
+                    $current_result['Object'] = $objects;
                 }
                 unset($result['results']['Object']);
             }
@@ -2503,6 +2524,7 @@ class AttributesController extends AppController
                 }
             }
         }
+        $this->set('persistent', $persistent);
         $this->set('results', $resultArray);
         $this->layout = 'ajax';
         $this->render('ajax/hover_enrichment');
@@ -2935,17 +2957,37 @@ class AttributesController extends AppController
                 'all',
                 array(
                     'conditions' => array('Attribute.type' => array('attachment', 'malware-sample')),
-                    'recursive' => -1)
+                    'contain' => ['Event.orgc_id', 'Event.org_id'],
+                    'recursive' => -1
+                )
             );
         $counter = 0;
         $attachmentTool = new AttachmentTool();
+        $results = [];
         foreach ($attributes as $attribute) {
             $exists = $attachmentTool->exists($attribute['Attribute']['event_id'], $attribute['Attribute']['id']);
             if (!$exists) {
+                $results['affectedEvents'][$attribute['Attribute']['event_id']] = $attribute['Attribute']['event_id'];
+                $results['affectedAttributes'][] = $attribute['Attribute']['id'];
+                foreach (['orgc', 'org'] as $type) {
+                    if (empty($results['affectedOrgs'][$type][$attribute['Event'][$type . '_id']])) {
+                        $results['affectedOrgs'][$type][$attribute['Event'][$type . '_id']] = 0;
+                    } else {
+                        $results['affectedOrgs'][$type][$attribute['Event'][$type . '_id']] += 1;
+                    }
+                }
                 $counter++;
             }
         }
-
+        if (!empty($results)) {
+            $results['affectedEvents'] = array_values($results['affectedEvents']);
+            rsort($results['affectedEvents']);
+            rsort($results['affectedAttributes']);
+            foreach (['orgc', 'org'] as $type) {
+                arsort($results['affectedOrgs'][$type]);
+            }
+        }
+        file_put_contents(APP . '/tmp/logs/missing_attachments.log', json_encode($results, JSON_PRETTY_PRINT));
         return new CakeResponse(array('body' => $counter, 'status' => 200));
     }
 

app/Controller/Component/ACLComponent.php

@@ -150,6 +150,20 @@ class ACLComponent extends Component
                     'index' => array('*'),
                     'view' => array('*'),
             ),
+            'eventReports' => array(
+                'add' => array('perm_add'),
+                'view' => array('*'),
+                'viewSummary' => array('*'),
+                'edit' => array('perm_add'),
+                'delete' => array('perm_add'),
+                'restore' => array('perm_add'),
+                'index' => array('*'),
+                'getProxyMISPElements' => array('*'),
+                'extractAllFromReport' => array('*'),
+                'extractFromReport' => array('*'),
+                'replaceSuggestionInReport' => array('*'),
+                'importReportFromUrl' => array('*'),
+            ),
             'events' => array(
                     'add' => array('perm_add'),
                     'addIOC' => array('perm_add'),
@@ -209,6 +223,7 @@ class ACLComponent extends Component
                     'reportValidationIssuesEvents' => array(),
                     'restoreDeletedEvents' => array('perm_site_admin'),
                     'restSearch' => array('*'),
+                    'runTaxonomyExclusivityCheck' => array('*'),
                     'saveFreeText' => array('perm_add'),
                     'stix' => array('*'),
                     'stix2' => array('*'),
@@ -222,7 +237,6 @@ class ACLComponent extends Component
                     'view' => array('*'),
                     'viewClusterRelations' => array('*'),
                     'viewEventAttributes' => array('*'),
-                    'viewEventGraph' => array('*'),
                     'viewGraph' => array('*'),
                     'viewGalaxyMatrix' => array('*'),
                     'xml' => array('*')
@@ -245,11 +259,17 @@ class ACLComponent extends Component
                     'fetchSelectedFromFreetextIndex' => array(),
                     'getEvent' => array(),
                     'importFeeds' => array(),
-                    'index' => array('*'),
+                    'index' => ['OR' => [
+                        'host_org_user',
+                        'perm_site_admin',
+                    ]],
                     'loadDefaultFeeds' => array('perm_site_admin'),
                     'previewEvent' => array('*'),
                     'previewIndex' => array('*'),
-                    'searchCaches' => array('*'),
+                    'searchCaches' => ['OR' => [
+                        'host_org_user',
+                        'perm_site_admin',
+                    ]],
                     'toggleSelected' => array('perm_site_admin'),
                     'view' => array('*'),
             ),
@@ -396,7 +416,6 @@ class ACLComponent extends Component
                     'fetchSGOrgRow' => array('*'),
                     'getUUIDs' => array('perm_sync'),
                     'index' => array('*'),
-                    'landingpage' => array('*'),
                     'view' => array('*'),
             ),
             'pages' => array(
@@ -454,7 +473,10 @@ class ACLComponent extends Component
                     'getSubmoduleQuickUpdateForm' => array(),
                     'getWorkers' => array(),
                     'getVersion' => array('*'),
-                    'idTranslator' => array('*'),
+                    'idTranslator' => ['OR' => [
+                        'host_org_user',
+                        'perm_site_admin',
+                    ]],
                     'import' => array(),
                     'index' => array(),
                     'ondemandAction' => array(),
@@ -777,7 +799,9 @@ class ACLComponent extends Component
         foreach ($aclList as $k => $v) {
             $aclList[$k] = array_change_key_case($v);
         }
-        $this->__checkLoggedActions($user, $controller, $action);
+        if (!$soft) {
+            $this->__checkLoggedActions($user, $controller, $action);
+        }
         if ($user && $user['Role']['perm_site_admin']) {
             return true;
         }
@@ -833,7 +857,6 @@ class ACLComponent extends Component
         switch ($code) {
             case 404:
                 throw new NotFoundException($message);
-                break;
             case 403:
                 throw new MethodNotAllowedException($message);
             default:

app/Controller/Component/RestResponseComponent.php

@@ -15,7 +15,10 @@ class RestResponseComponent extends Component
         )
     );
 
-    private $___setup = false;
+    private $__setup = false;
+
+    /** @var array */
+    private $__fieldsConstraint;
 
     private $__descriptions = array(
         'Attribute' => array(
@@ -83,6 +86,18 @@ class RestResponseComponent extends Component
                 'optional' => array('network_name')
             )
         ),
+        'EventReport' => array(
+            'add' => array(
+                'description' => "POST a report in JSON format to create a report for the provided event",
+                'mandatory' => array('name'),
+                'optional' => array('distribution', 'content')
+            ),
+            'edit' => array(
+                'description' => "POST a report in JSON format to update the report",
+                'mandatory' => array(),
+                'optional' => array('name', 'distribution', 'content')
+            )
+        ),
         'Feed' => array(
             'add' => array(
                 'description' => "POST a MISP Feed descriptor JSON to this API to add a Feed.",
@@ -260,6 +275,10 @@ class RestResponseComponent extends Component
             'attachTagToObject' => array(
                 'description' => "Attach a Tag to an object, refenced by an UUID. Tag can either be a tag id or a tag name.",
                 'mandatory' => array('uuid', 'tag'),
+            ),
+            'search' => array(
+                'description' => "GET or POST the tags to search for as a raw string or as a list. The strict_tag_name_only parameter only returns tags matching exactly the tag name (thus, skipping synonyms and cluster's value)",
+                'params' => array('tag_name', 'strict_tag_name_only')
             )
         ),
         'User' => array(
@@ -311,8 +330,11 @@ class RestResponseComponent extends Component
 
     private $__scopedFieldsConstraint = array();
 
-    public function initialize(Controller $controller) {
-        $this->__configureFieldConstraints();
+    /** @var Controller */
+    private $Controller;
+
+    public function initialize(Controller $controller)
+    {
         $this->Controller = $controller;
     }
 
@@ -418,7 +440,7 @@ class RestResponseComponent extends Component
         return '[]';
     }
 
-    public function saveFailResponse($controller, $action, $id = false, $validationErrors, $format = false)
+    public function saveFailResponse($controller, $action, $id = false, $validationErrors, $format = false, $data = null)
     {
         $this->autoRender = false;
         $response = array();
@@ -430,12 +452,15 @@ class RestResponseComponent extends Component
         $response['saved'] = false;
         $response['name'] = 'Could not ' . $stringifiedAction . ' ' . Inflector::singularize($controller);
         $response['message'] = $response['name'];
+        if (!is_null($data)) {
+            $response['data'] = $data;
+        }
         $response['url'] = $this->__generateURL($action, $controller, $id);
         $response['errors'] = $validationErrors;
         return $this->__sendResponse($response, 403, $format);
     }
 
-    public function saveSuccessResponse($controller, $action, $id = false, $format = false, $message = false)
+    public function saveSuccessResponse($controller, $action, $id = false, $format = false, $message = false, $data = null)
     {
         $action = $this->__dissectAdminRouting($action);
         if (!$message) {
@@ -445,13 +470,26 @@ class RestResponseComponent extends Component
         $response['success'] = true;
         $response['name'] = $message;
         $response['message'] = $response['name'];
+        if (!is_null($data)) {
+            $response['data'] = $data;
+        }
         $response['url'] = $this->__generateURL($action, $controller, $id);
         return $this->__sendResponse($response, 200, $format);
     }
 
+    /**
+     * @param mixed $response
+     * @param int $code
+     * @param string|false $format
+     * @param bool $raw
+     * @param bool $download
+     * @param array $headers
+     * @return CakeResponse
+     */
     private function __sendResponse($response, $code, $format = false, $raw = false, $download = false, $headers = array())
     {
-        if (strtolower($format) === 'application/xml' || strtolower($format) === 'xml') {
+        $format = strtolower($format);
+        if ($format === 'application/xml' || $format === 'xml') {
             if (!$raw) {
                 if (isset($response[0])) {
                     if (count(array_keys($response[0])) == 1) {
@@ -468,9 +506,9 @@ class RestResponseComponent extends Component
                 $response = $response->asXML();
             }
             $type = 'xml';
-        } elseif (strtolower($format) == 'openioc') {
+        } elseif ($format === 'openioc') {
             $type = 'xml';
-        } elseif (strtolower($format) == 'csv') {
+        } elseif ($format === 'csv') {
             $type = 'csv';
         } else {
             if (empty($format)) {
@@ -506,7 +544,7 @@ class RestResponseComponent extends Component
                 }
             }
         }
-        $cakeResponse = new CakeResponse(array('body'=> $response, 'status' => $code, 'type' => $type));
+        $cakeResponse = new CakeResponse(array('body' => $response, 'status' => $code, 'type' => $type));
 
         if (Configure::read('Security.allow_cors')) {
             $headers["Access-Control-Allow-Headers"] =  "Origin, Content-Type, Authorization, Accept";
@@ -515,14 +553,10 @@ class RestResponseComponent extends Component
             $headers["Access-Control-Expose-Headers"] = ["X-Result-Count"];
         }
         if (!empty($this->headers)) {
-            foreach ($this->headers as $key => $value) {
-                $cakeResponse->header($key, $value);
-            }
+            $cakeResponse->header($this->headers);
         }
         if (!empty($headers)) {
-            foreach ($headers as $key => $value) {
-                $cakeResponse->header($key, $value);
-            }
+            $cakeResponse->header($headers);
         }
         if (!empty($deprecationWarnings)) {
             $cakeResponse->header('X-Deprecation-Warning', $deprecationWarnings);
@@ -530,7 +564,6 @@ class RestResponseComponent extends Component
         if ($download) {
             $cakeResponse->download($download);
         }
-
         return $cakeResponse;
     }
 
@@ -619,13 +652,13 @@ class RestResponseComponent extends Component
                     'params' => $this->__descriptions[$scope]['restSearch']['params']
                 );
             }
+            $this->__configureFieldConstraints();
             $this->__setupFieldsConstraint();
+            $this->__setup = true;
         }
         return true;
     }
 
-    private $__fieldConstraint = array();
-
     // default value and input for API field
     private function __configureFieldConstraints()
     {
@@ -641,7 +674,7 @@ class RestResponseComponent extends Component
                 'input' => 'radio',
                 'type' => 'integer',
                 'values' => array(1 => 'True', 0 => 'False' ),
-                'help' => __('Is the sharing group selectable (active) when chosing distribution')
+                'help' => __('Is the sharing group selectable (active) when choosing distribution')
             ),
             'all' => array(
                 'input' => 'text',
@@ -871,7 +904,7 @@ class RestResponseComponent extends Component
                 'input' => 'radio',
                 'type' => 'integer',
                 'values' => array(1 => 'True', 0 => 'False' ),
-                'help' => __('When uploading malicious samples, set this flag to tell MISP to encrpyt the sample and extract the file hashes. This will create a MISP object with the appropriate attributes.')
+                'help' => __('When uploading malicious samples, set this flag to tell MISP to encrypt the sample and extract the file hashes. This will create a MISP object with the appropriate attributes.')
             ),
             //'enforceWarningList' => array(
             //    'input' => 'radio',
@@ -1456,6 +1489,12 @@ class RestResponseComponent extends Component
                 'operators' => array('equal'),
                 'values' => array( 'misp' => 'MISP Feed', 'freetext' => 'Freetext Parsed Feed', 'csv' => 'CSV Parsed Feed')
             ),
+            'strict_tag_name_only' => array(
+                'input' => 'radio',
+                'type' => 'integer',
+                'values' => array(1 => 'True', 0 => 'False' ),
+                'help' => __('Only returns tags matching exactly the tag name (thus skipping synonyms and cluster\'s value)')
+            ),
             'subject' => array(
                 'input' => 'text',
                 'type' => 'string',
@@ -1833,5 +1872,4 @@ class RestResponseComponent extends Component
             $field['help'] = __('Seen within the last x amount of time, where x can be defined in days, hours, minutes (for example 5d or 12h or 30m)');
         }
     }
-
 }

app/Controller/EventGraphController.php

@@ -13,7 +13,7 @@ class EventGraphController extends AppController
         parent::beforeFilter();
     }
 
-    public function view($event_id = false)
+    public function view($event_id = false, $graph_id = null)
     {
         if ($event_id === false) {
             throw new MethodNotAllowedException(__('No event ID set.'));
@@ -40,12 +40,16 @@ class EventGraphController extends AppController
         }
 
         // fetch eventGraphs
+        $conditions = [
+            'EventGraph.event_id' => $event_id,
+            'EventGraph.org_id' => $org_id
+        ];
+        if (!is_null($graph_id)) {
+            $conditions['EventGraph.id'] = $graph_id;
+        }
         $eventGraphs = $this->EventGraph->find('all', array(
             'order' => 'EventGraph.timestamp DESC',
-            'conditions' => array(
-                'EventGraph.event_id' => $event_id,
-                'EventGraph.org_id' => $org_id
-            ),
+            'conditions' => $conditions,
             'contain' => array(
                 'User' => array(
                     'fields' => array(

app/Controller/EventReportsController.php

@@ -0,0 +1,483 @@
+<?php
+App::uses('AppController', 'Controller');
+
+/**
+ * @property EventReport $EventReport
+ */
+class EventReportsController extends AppController
+{
+    public $components = array(
+        'Security',
+        'AdminCrud',
+        'RequestHandler'
+    );
+
+    public $paginate = array(
+        'limit' => 60,
+        'order' => array(
+                'EventReport.event_id' => 'ASC',
+                'EventReport.name' => 'ASC'
+        ),
+        'recursive' => -1,
+        'contain' => array(
+            'SharingGroup' => array('fields' => array('id', 'name', 'uuid')),
+            'Event' => array(
+                'fields' =>  array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.info', 'Event.user_id', 'Event.date'),
+                'Orgc' => array('fields' => array('Orgc.id', 'Orgc.name')),
+                'Org' => array('fields' => array('Org.id', 'Org.name'))
+            )
+        )
+    );
+
+    public function add($eventId = false)
+    {
+        if ($this->request->is('get') && $this->_isRest()) {
+            return $this->RestResponse->describe('EventReports', 'add', false, $this->response->type());
+        }
+        if ($eventId === false) {
+            throw new MethodNotAllowedException(__('No event ID set.'));
+        }
+        $event = $this->__canModifyReport($eventId);
+        if ($this->request->is('post') || $this->request->is('put')) {
+            if (!isset($this->request->data['EventReport'])) {
+                $this->request->data['EventReport'] = $this->request->data;
+            }
+            $report = $this->request->data;
+            $errors = $this->EventReport->addReport($this->Auth->user(), $report, $eventId);
+            $redirectTarget = array('controller' => 'events', 'action' => 'view', $eventId);
+            if (!empty($errors)) {
+                return $this->__getFailResponseBasedOnContext($errors, array(), 'add', $this->EventReport->id, $redirectTarget);
+            } else {
+                $successMessage = __('Report saved.');
+                $report = $this->EventReport->simpleFetchById($this->Auth->user(), $this->EventReport->id);
+                return $this->__getSuccessResponseBasedOnContext($successMessage, $report, 'add', false, $redirectTarget);
+            }
+        }
+        $this->set('event_id', $eventId);
+        $this->set('action', 'add');
+        $this->__injectDistributionLevelToViewContext();
+        $this->__injectSharingGroupsDataToViewContext();
+    }
+
+
+    public function view($reportId, $ajax=false)
+    {
+        $report = $this->EventReport->simpleFetchById($this->Auth->user(), $reportId);
+        if ($this->_isRest()) {
+            return $this->RestResponse->viewData($report, $this->response->type());
+        }
+        $this->set('ajax', $ajax);
+        $this->set('id', $reportId);
+        $this->set('report', $report);
+        $this->set('title_for_layout', __('Event report %s', $report['EventReport']['name']));
+        $this->__injectDistributionLevelToViewContext();
+        $this->__injectPermissionsToViewContext($this->Auth->user(), $report);
+    }
+
+    public function getProxyMISPElements($reportId)
+    {
+        if (!$this->_isRest()) {
+            throw new MethodNotAllowedException(__('This function can only be reached via the API.'));
+        }
+        $report = $this->EventReport->simpleFetchById($this->Auth->user(), $reportId);
+        $proxyMISPElements = $this->EventReport->getProxyMISPElements($this->Auth->user(), $report['EventReport']['event_id']);
+        return $this->RestResponse->viewData($proxyMISPElements, $this->response->type());
+    }
+
+    public function viewSummary($reportId)
+    {
+        $report = $this->EventReport->simpleFetchById($this->Auth->user(), $reportId);
+        $this->set('id', $reportId);
+        $this->set('report', $report);
+        $this->__injectDistributionLevelToViewContext();
+        $this->__injectPermissionsToViewContext($this->Auth->user(), $report);
+    }
+
+    public function edit($id)
+    {
+        $savedReport = $this->EventReport->fetchIfAuthorized($this->Auth->user(), $id, 'edit', $throwErrors=true, $full=true);
+        if ($this->request->is('post') || $this->request->is('put')) {
+            $newReport = $this->request->data;
+            $newReport = $this->__applyDataFromSavedReport($newReport, $savedReport);
+            $errors = $this->EventReport->editReport($this->Auth->user(), $newReport, $savedReport['EventReport']['event_id']);
+            $redirectTarget = array('controller' => 'eventReports', 'action' => 'view', $id);
+            if (!empty($errors)) {
+                return $this->__getFailResponseBasedOnContext($validationErrors, array(), 'edit', $id, $redirectTarget);
+            } else {
+                $successMessage = __('Report saved.');
+                $report = $this->EventReport->simpleFetchById($this->Auth->user(), $this->EventReport->id);
+                return $this->__getSuccessResponseBasedOnContext($successMessage, $report, 'edit', $id, $redirectTarget);
+            }
+        } else {
+            $this->request->data = $savedReport;
+        }
+
+        $this->set('id', $savedReport['EventReport']['id']);
+        $this->set('event_id', $savedReport['EventReport']['event_id']);
+        $this->set('action', 'edit');
+        $this->__injectDistributionLevelToViewContext();
+        $this->__injectSharingGroupsDataToViewContext();
+        $this->render('add');
+    }
+
+    public function delete($id, $hard=false)
+    {
+        $report = $this->EventReport->fetchIfAuthorized($this->Auth->user(), $id, 'delete', $throwErrors=true, $full=false);
+        if ($this->request->is('post')) {
+            $errors = $this->EventReport->deleteReport($this->Auth->user(), $report, $hard=$hard);
+            $redirectTarget = $this->referer();
+            if (empty($errors)) {
+                $successMessage = __('Event Report %s %s deleted', $id, $hard ? __('hard') : __('soft'));
+                $report = $hard ? null : $this->EventReport->simpleFetchById($this->Auth->user(), $id);
+                return $this->__getSuccessResponseBasedOnContext($successMessage, $report, 'delete', $id, $redirectTarget);
+            } else {
+                $errorMessage = __('Event Report %s could not be %s deleted.%sReasons: %s', $id, $hard ? __('hard') : __('soft'), PHP_EOL, json_encode($errors));
+                return $this->__getFailResponseBasedOnContext($errorMessage, array(), 'edit', $id, $redirectTarget);
+            }
+        } else {
+            if (!$this->request->is('ajax')) {
+                throw new MethodNotAllowedException(__('This function can only be reached via AJAX.'));
+            } else {
+                $this->layout = 'ajax';
+                $this->set('report', $report);
+                $this->render('ajax/delete');
+            }
+        }
+    }
+
+    public function restore($id)
+    {
+        $report = $this->EventReport->fetchIfAuthorized($this->Auth->user(), $id, 'edit', $throwErrors=true, $full=false);
+        if ($this->request->is('post')) {
+            $errors = $this->EventReport->restoreReport($this->Auth->user(), $id);
+            $redirectTarget = $this->referer();
+            if (empty($errors)) {
+                $successMessage = __('Event Report %s restored', $id);
+                $report = $this->EventReport->simpleFetchById($this->Auth->user(), $id);
+                return $this->__getSuccessResponseBasedOnContext($successMessage, $report, 'restore', $id, $redirectTarget);
+            } else {
+                $errorMessage = __('Event Report %s could not be %s restored.%sReasons: %s', $id, PHP_EOL, json_encode($errors));
+                return $this->__getFailResponseBasedOnContext($errorMessage, array(), 'restore', $id, $redirectTarget);
+            }
+        } else {
+            if (!$this->request->is('ajax')) {
+                throw new MethodNotAllowedException(__('This function can only be reached via AJAX.'));
+            } else {
+                $this->layout = 'ajax';
+                $this->set('report', $report);
+            }
+        }
+    }
+
+    public function index()
+    {
+        $filters = $this->IndexFilter->harvestParameters(['event_id', 'value', 'context', 'index_for_event', 'extended_event']);
+        $filters['embedded_view']  = $this->request->is('ajax');
+        $compiledConditions = $this->__generateIndexConditions($filters);
+        if ($this->_isRest()) {
+            $reports = $this->EventReport->find('all', [
+                'recursive' => -1,
+                'conditions' => $compiledConditions,
+                'contain' => $this->EventReport->defaultContain,
+            ]);
+            return $this->RestResponse->viewData($reports, $this->response->type());
+        } else {
+            $this->paginate['conditions']['AND'][] = $compiledConditions;
+            $reports = $this->paginate();
+            $this->set('reports', $reports);
+            $this->__injectIndexVariablesToViewContext($filters);
+            if (!empty($filters['index_for_event'])) {
+                if (empty($filters['event_id'])) {
+                    throw new MethodNotAllowedException("When requesting index for event, event ID must be provided.");
+                }
+                $this->set('canModify', $this->__canModifyReport($filters['event_id']));
+                $this->set('extendedEvent', !empty($filters['extended_event']));
+                $fetcherModule = $this->EventReport->isFetchURLModuleEnabled();
+                $this->set('importModuleEnabled', is_array($fetcherModule));
+                $this->render('ajax/indexForEvent');
+            }
+        }
+    }
+
+    public function extractAllFromReport($reportId)
+    {
+        if (!$this->request->is('ajax')) {
+            throw new MethodNotAllowedException(__('This function can only be reached via AJAX.'));
+        } else {
+            if ($this->request->is('post')) {
+                $report = $this->EventReport->fetchIfAuthorized($this->Auth->user(), $reportId, 'edit', $throwErrors=true, $full=false);
+                $results = $this->EventReport->getComplexTypeToolResultWithReplacements($this->Auth->user(), $report);
+                $report['EventReport']['content'] = $results['replacementResult']['contentWithReplacements'];
+                $contextResults = $this->EventReport->extractWithReplacements($this->Auth->user(), $report, ['replace' => true]);
+                $suggestionResult = $this->EventReport->transformFreeTextIntoSuggestion($contextResults['contentWithReplacements'], $results['complexTypeToolResult']);
+                $errors = $this->EventReport->applySuggestions($this->Auth->user(), $report, $suggestionResult['contentWithSuggestions'], $suggestionResult['suggestionsMapping']);
+                if (empty($errors)) {
+                    if (!empty($this->data['EventReport']['tag_event'])) {
+                        $this->EventReport->attachTagsAfterReplacements($this->Auth->User(), $contextResults['replacedContext'], $report['EventReport']['event_id']);
+                    }
+                    $report = $this->EventReport->simpleFetchById($this->Auth->user(), $reportId);
+                    $data = [ 'report' => $report ];
+                    $successMessage = __('Automatic extraction applied to Event Report %s', $reportId);
+                    return $this->__getSuccessResponseBasedOnContext($successMessage, $data, 'applySuggestions', $reportId);
+                } else {
+                    $errorMessage = __('Automatic extraction could not be applied to Event Report %s.%sReasons: %s', $reportId, PHP_EOL, json_encode($errors));
+                    return $this->__getFailResponseBasedOnContext($errorMessage, array(), 'applySuggestions', $reportId);
+                }
+            }
+            $this->layout = 'ajax';
+            $this->set('reportId', $reportId);
+            $this->render('ajax/extractAllFromReport');
+        }
+    }
+
+    public function extractFromReport($reportId)
+    {
+        if (!$this->request->is('ajax')) {
+            throw new MethodNotAllowedException(__('This function can only be reached via AJAX.'));
+        } else {
+            $report = $this->EventReport->fetchIfAuthorized($this->Auth->user(), $reportId, 'view', $throwErrors=true, $full=false);
+            $dataResults = $this->EventReport->getComplexTypeToolResultWithReplacements($this->Auth->user(), $report);
+            $report['EventReport']['content'] = $dataResults['replacementResult']['contentWithReplacements'];
+            $contextResults = $this->EventReport->extractWithReplacements($this->Auth->user(), $report);
+            $typeToCategoryMapping = $this->EventReport->Event->Attribute->typeToCategoryMapping();
+            $data = [
+                'complexTypeToolResult' => $dataResults['complexTypeToolResult'],
+                'typeToCategoryMapping' => $typeToCategoryMapping,
+                'replacementValues' => $dataResults['replacementResult']['replacedValues'],
+                'replacementContext' => $contextResults['replacedContext']
+            ];
+            return $this->RestResponse->viewData($data, $this->response->type());
+        }
+    }
+
+    public function replaceSuggestionInReport($reportId)
+    {
+        if (!$this->request->is('ajax')) {
+            throw new MethodNotAllowedException(__('This function can only be reached via AJAX.'));
+        } else {
+            $report = $this->EventReport->fetchIfAuthorized($this->Auth->user(), $reportId, 'edit', $throwErrors=true, $full=false);
+            if ($this->request->is('post')) {
+                $errors = [];
+                $suggestions = $this->EventReport->jsonDecode($this->data['EventReport']['suggestions']);
+                if (!empty($suggestions['content']) && !empty($suggestions['mapping'])) {
+                    $errors = $this->EventReport->applySuggestions($this->Auth->user(), $report, $suggestions['content'], $suggestions['mapping']);
+                } else {
+                    $errors[] = __('`content` and `mapping` key cannot be empty');
+                }
+                if (empty($errors)) {
+                    $report = $this->EventReport->simpleFetchById($this->Auth->user(), $reportId);
+                    $results = $this->EventReport->getComplexTypeToolResultWithReplacements($this->Auth->user(), $report);
+                    $contextResults = $this->EventReport->extractWithReplacements($this->Auth->user(), $report);
+                    $data = [
+                        'report' => $report,
+                        'complexTypeToolResult' => $results['complexTypeToolResult'],
+                        'replacementValues' => $results['replacementResult']['replacedValues'],
+                        'replacementContext' => $contextResults['replacedContext']
+                    ];
+                    $successMessage = __('Suggestions applied to Event Report %s', $reportId);
+                    return $this->__getSuccessResponseBasedOnContext($successMessage, $data, 'applySuggestions', $reportId);
+                } else {
+                    $errorMessage = __('Suggestions could not be applied to Event Report %s.%sReasons: %s', $reportId, PHP_EOL, json_encode($errors));
+                    return $this->__getFailResponseBasedOnContext($errorMessage, array(), 'applySuggestions', $reportId);
+                }
+            }
+            $this->layout = 'ajax';
+            $this->render('ajax/replaceSuggestionInReport');
+        }
+    }
+
+    public function importReportFromUrl($event_id)
+    {
+        if (!$this->request->is('ajax')) {
+            throw new MethodNotAllowedException(__('This function can only be reached via AJAX.'));
+        }
+        $fetcherModule = $this->EventReport->isFetchURLModuleEnabled();
+        if ($this->request->is('post')) {
+            if (empty($this->data['EventReport']['url'])) {
+                throw new MethodNotAllowedException(__('An URL must be provided'));
+            }
+            $url = $this->data['EventReport']['url'];
+            $markdown = $this->EventReport->downloadMarkdownFromURL($event_id, $url);
+            $errors = [];
+            if (!empty($markdown)) {
+                $report = [
+                    'name' => __('Report from - %s (%s)', $url, time()),
+                    'distribution' => 5,
+                    'content' => $markdown
+                ];
+                $errors = $this->EventReport->addReport($this->Auth->user(), $report, $event_id);
+            } else {
+                $errors[] = __('Could not fetch report from URL. Fetcher module not enabled or could not download the page');
+            }
+            $redirectTarget = array('controller' => 'events', 'action' => 'view', $event_id);
+            if (!empty($errors)) {
+                return $this->__getFailResponseBasedOnContext($errors, array(), 'addFromURL', $this->EventReport->id, $redirectTarget);
+            } else {
+                $successMessage = __('Report downloaded and created');
+                $report = $this->EventReport->simpleFetchById($this->Auth->user(), $this->EventReport->id);
+                return $this->__getSuccessResponseBasedOnContext($successMessage, $report, 'addFromURL', false, $redirectTarget);
+            }
+        }
+        $this->set('importModuleEnabled', is_array($fetcherModule));
+        $this->set('event_id', $event_id);
+        $this->layout = 'ajax';
+        $this->render('ajax/importReportFromUrl');
+    }
+
+    private function __generateIndexConditions($filters = [])
+    {
+        $aclConditions = $this->EventReport->buildACLConditions($this->Auth->user());
+        $eventConditions = [];
+        if (!empty($filters['event_id'])) {
+            $extendingEventIds = [];
+            if (!empty($filters['extended_event'])) {
+                $extendingEventIds = $this->EventReport->Event->getExtendingEventIdsFromEvent($this->Auth->user(), $filters['event_id']);
+            }
+            $eventConditions = ['EventReport.event_id' => array_merge([$filters['event_id']], $extendingEventIds)];
+        }
+
+        $contextConditions = [];
+        if (empty($filters['context'])) {
+            $filters['context'] = 'default';
+        }
+        if ($filters['context'] == 'deleted') {
+            $contextConditions['EventReport.deleted'] = true;
+        } elseif ($filters['context'] == 'default') {
+            $contextConditions['EventReport.deleted'] = false;
+        }
+        $searchConditions = [];
+        if (empty($filters['value'])) {
+            $filters['value'] = '';
+        } else {
+            $searchall = '%' . strtolower($filters['value']) . '%';
+            $searchConditions = array(
+                'OR' => array(
+                    'LOWER(EventReport.name) LIKE' => $searchall,
+                    'LOWER(EventReport.content) LIKE' => $searchall,
+                    'EventReport.id' => $searchall,
+                    'EventReport.uuid' => $searchall
+                )
+            );
+        }
+        $compiledConditions = [
+            'AND' => [
+                $aclConditions,
+                $eventConditions,
+                $contextConditions,
+                $searchConditions,
+            ]
+        ];
+        return $compiledConditions;
+    }
+
+    private function __getSuccessResponseBasedOnContext($message, $data = null, $action = '', $id = false, $redirect = array())
+    {
+        if ($this->_isRest()) {
+            if (!is_null($data)) {
+                return $this->RestResponse->viewData($data, $this->response->type());
+            } else {
+                return $this->RestResponse->saveSuccessResponse($this->EventReport->alias, $action, $id, false, $message);
+            }
+        } elseif ($this->request->is('ajax')) {
+            return $this->RestResponse->saveSuccessResponse($this->EventReport->alias, $action, $id, false, $message, $data);
+        } else {
+            $this->Flash->success($message);
+            $this->redirect($redirect);
+        }
+        return;
+    }
+
+    private function __getFailResponseBasedOnContext($message, $data = null, $action = '', $id = false, $redirect = array())
+    {
+        if (is_array($message)) {
+            $message = implode(', ', $message);
+        }
+        if ($this->_isRest()) {
+            if (!is_null($data)) {
+                return $this->RestResponse->viewData($data, $this->response->type());
+            } else {
+                return $this->RestResponse->saveFailResponse('EventReport', $action, $id, $message, false);
+            }
+        } elseif ($this->request->is('ajax')) {
+            return $this->RestResponse->saveFailResponse('EventReport', $action, $id, $message, false, $data);
+        } else {
+            $this->Flash->error($message);
+            $this->redirect($this->referer());
+        }
+        return;
+    }
+
+    private function __injectIndexVariablesToViewContext($filters)
+    {
+        if (!empty($filters['context'])) {
+            $this->set('context', $filters['context']);
+        } else {
+            $this->set('context', 'default');
+        }
+        if (!empty($filters['event_id'])) {
+            $this->set('event_id', $filters['event_id']);
+        }
+        if (isset($filters['embedded_view'])) {
+            $this->set('embedded_view', $filters['embedded_view']);
+        } else {
+            $this->set('embedded_view', false);
+        }
+        if (!empty($filters['value'])) {
+            $this->set('searchall', $filters['value']);
+        } else {
+            $this->set('searchall', '');
+        }
+        $this->__injectDistributionLevelToViewContext();
+    }
+
+    private function __injectDistributionLevelToViewContext()
+    {
+        $distributionLevels = $this->EventReport->Event->Attribute->distributionLevels;
+        $this->set('distributionLevels', $distributionLevels);
+        $initialDistribution = 5;
+        $configuredDistribution = Configure::check('MISP.default_attribute_distribution');
+        if ($configuredDistribution != null && $configuredDistribution != 'event') {
+            $initialDistribution = $configuredDistribution;
+        }
+        $this->set('initialDistribution', $initialDistribution);
+    }
+
+    private function __injectSharingGroupsDataToViewContext()
+    {
+        $sgs = $this->EventReport->Event->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', 1);
+        $this->set('sharingGroups', $sgs);
+    }
+
+    private function __injectPermissionsToViewContext($user, $report)
+    {
+        $canEdit = $this->EventReport->canEditReport($user, $report) === true;
+        $this->set('canEdit', $canEdit);
+    }
+
+    private function __canModifyReport($eventId)
+    {
+        $event = $this->EventReport->Event->fetchSimpleEvent($this->Auth->user(), $eventId, array());
+        if (empty($event)) {
+            throw new NotFoundException(__('Invalid event'));
+        }
+        if (!$this->__canModifyEvent($event)) {
+            throw new ForbiddenException(__('You do not have permission to do that.'));
+        }
+        return $event;
+    }
+
+    private function __applyDataFromSavedReport($newReport, $savedReport)
+    {
+        if (!isset($newReport['EventReport'])) {
+            $newReport = array('EventReport' => $newReport);
+        }
+        $fieldList = $this->EventReport->captureFields;
+        $ignoreFieldList = ['id', 'uuid', 'event_id', 'deleted'];
+        foreach ($fieldList as $field) {
+            if (!in_array($field, $ignoreFieldList) && isset($newReport['EventReport'][$field])) {
+                $savedReport['EventReport'][$field] = $newReport['EventReport'][$field];
+            }
+        }
+        return $savedReport;
+    }
+}

app/Controller/EventsController.php

@@ -627,7 +627,14 @@ class EventsController extends AppController
                         $v = $filterString;
                         break;
                     case 'minimal':
-                        $this->paginate['conditions']['AND'][] = array('NOT' => array('Event.attribute_count' => 0));
+                        $tableName = $this->Event->EventReport->table;
+                        $eventReportQuery = sprintf('EXISTS (SELECT id, deleted FROM %s WHERE %s.event_id = Event.id and %s.deleted = 0)', $tableName, $tableName, $tableName);
+                        $this->paginate['conditions']['AND'][] = [
+                            'OR' => [
+                                ['Event.attribute_count >' => 0],
+                                [$eventReportQuery]
+                            ]
+                        ];
                         break;
                     default:
                         continue 2;
@@ -984,48 +991,41 @@ class EventsController extends AppController
         $this->layout = 'ajax';
     }
 
-    /*
+    /**
      * Search for a value on an attribute level for a specific field.
-     * $attribute : (array) an attribute
-     * $fields : (array) list of keys in attribute to search in
-     * $searchValue : Values to search ( '|' is the separator)
-     * returns true on match
+     *
+     * @param array $attribute An attribute
+     * @param array $fields List of keys in attribute to search in
+     * @param string $searchValue Values to search ( '|' is the separator)
+     * @return bool Returns true on match
      */
     private function __valueInFieldAttribute($attribute, $fields, $searchValue)
     {
-        foreach ($attribute as $k => $v) { // look in attributes line
-            if (is_string($v)) {
-                foreach ($fields as $field) {
-                    if (strpos(".", $field) === false) { // check sub array after
-                        // check for key in attribut
-                        if (isset($attribute[$field])) {
-                            $temp_value = strtolower($attribute[$field]);
-                            $temp_search = strtolower($searchValue);
-                            $temp_searches = explode('|', $temp_search);
-                            foreach ($temp_searches as $s) {
-                                if (strpos($temp_value, $s) !==false) {
-                                    return true;
-                                }
-                            }
+        $searchParts = explode('|', mb_strtolower($searchValue));
+
+        foreach ($fields as $field) {
+            if (strpos($field, 'Tag') === 0) {
+                if (empty($attribute['AttributeTag'])) {
+                    continue;
+                }
+                $fieldValues = Hash::extract($attribute, 'AttributeTag.{n}.' . $field);
+                foreach ($fieldValues as $fieldValue) {
+                    $fieldValue = mb_strtolower($fieldValue);
+                    foreach ($searchParts as $s) {
+                        if (strpos($fieldValue, $s) !== false) {
+                            return true;
                         }
                     }
                 }
             } else {
-                // check for tag in attribut maybe for other thing later
-                if ($k === 'AttributeTag') {
-                    foreach ($v as $tag) {
-                        foreach ($fields as $field) {
-                            if (strpos(strtolower($field), "tag.") !== false) { // check sub array
-                                $tagKey = explode('tag.', strtolower($field))[1];
-                                if (isset($tag['Tag'][$tagKey])) {
-                                    $temp_value = strtolower($tag['Tag'][$tagKey]);
-                                    $temp_search = strtolower($searchValue);
-                                    if (strpos($temp_value, $temp_search) !==false) {
-                                        return true;
-                                    }
-                                }
-                            }
-                        }
+                $fieldValue = isset($attribute[$field]) ? $attribute[$field] : null;
+                if (empty($fieldValue)) {
+                    continue;
+                }
+                $fieldValue = mb_strtolower($fieldValue);
+                foreach ($searchParts as $s) {
+                    if (strpos($fieldValue, $s) !== false) {
+                        return true;
                     }
                 }
             }
@@ -1070,6 +1070,7 @@ class EventsController extends AppController
             $conditions['to_ids'] = $filters['toIDS'] == 2 ? 0 : 1;
         }
         $conditions['includeFeedCorrelations'] = true;
+        $conditions['includeWarninglistHits'] = true;
         if (!isset($filters['includeServerCorrelations'])) {
             $conditions['includeServerCorrelations'] = 1;
             if ($this->_isRest()) {
@@ -1081,6 +1082,7 @@ class EventsController extends AppController
         $conditions['includeAllTags'] = true;
         $conditions['includeGranularCorrelations'] = 1;
         $conditions['includeEventCorrelations'] = false;
+        $conditions['noEventReports'] = true; // event reports for view are loaded dynamically
         if (!empty($filters['includeRelatedTags'])) {
             $this->set('includeRelatedTags', 1);
             $conditions['includeRelatedTags'] = 1;
@@ -1253,10 +1255,6 @@ class EventsController extends AppController
         $warningTagConflicts = array();
         $filters = $this->_harvestParameters($filterData, $exception);
 
-        $this->loadModel('GalaxyCluster');
-        if (!$this->_isRest()) {
-            //$attack = $this->GalaxyCluster->Galaxy->constructAttackReport($event);
-        }
         $emptyEvent = (empty($event['Object']) && empty($event['Attribute']));
         $this->set('emptyEvent', $emptyEvent);
         $attributeCount = isset($event['Attribute']) ? count($event['Attribute']) : 0;
@@ -1278,6 +1276,7 @@ class EventsController extends AppController
         $this->set('object_count', $objectCount);
         // set the data for the contributors / history field
         $contributors = $this->Event->ShadowAttribute->getEventContributors($event['Event']['id']);
+        $this->set('contributors', $contributors);
         if ($this->userRole['perm_publish'] && $event['Event']['orgc_id'] == $this->Auth->user('org_id')) {
             $proposalStatus = false;
             if (isset($event['ShadowAttribute']) && !empty($event['ShadowAttribute'])) {
@@ -1476,7 +1475,6 @@ class EventsController extends AppController
             $cortex_modules = $this->Module->getEnabledModules($this->Auth->user(), false, 'Cortex');
             $this->set('cortex_modules', $cortex_modules);
         }
-        $this->set('contributors', $contributors);
         $this->set('typeGroups', array_keys($this->Event->Attribute->typeGroupings));
         $attributeUri = $this->baseurl . '/events/viewEventAttributes/' . $event['Event']['id'];
         foreach ($this->params->named as $k => $v) {
@@ -1507,13 +1505,17 @@ class EventsController extends AppController
         $this->set('advancedFilteringActive', $advancedFiltering['active'] ? 1 : 0);
         $this->set('advancedFilteringActiveRules', $advancedFiltering['activeRules']);
         $this->set('defaultFilteringRules', $this->defaultFilteringRules);
-        $this->set('mitreAttackGalaxyId', $this->Event->GalaxyCluster->Galaxy->getMitreAttackGalaxyId());
         $this->set('modificationMapCSV', $modificationMapCSV);
         $this->set('title_for_layout', __('Event #%s', $event['Event']['id']));
     }
 
     public function view($id = null, $continue = false, $fromEvent = null)
     {
+        if ($this->request->is('head')) { // Just check if event exists
+            $exists = $this->Event->fetchSimpleEvent($this->Auth->user(), $id, ['fields' => ['id']]);
+            return new CakeResponse(['status' => $exists ? 200 : 404]);
+        }
+
         if (is_numeric($id)) {
             $conditions = array('eventid' => $id);
         } else if (Validation::uuid($id)) {
@@ -1526,6 +1528,7 @@ class EventsController extends AppController
             $conditions['includeAttachments'] = true;
         } else {
             $conditions['includeAllTags'] = true;
+            $conditions['noEventReports'] = true; // event reports for view are loaded dynamically
         }
         $deleted = 0;
         if (isset($this->params['named']['deleted'])) {
@@ -1570,6 +1573,7 @@ class EventsController extends AppController
             $this->set('extended', 0);
         }
         $conditions['excludeLocalTags'] = false;
+        $conditions['includeWarninglistHits'] = true;
         if (isset($this->params['named']['excludeLocalTags'])) {
             $conditions['excludeLocalTags'] = $this->params['named']['excludeLocalTags'];
         }
@@ -1588,32 +1592,31 @@ class EventsController extends AppController
             $conditions['includeServerCorrelations'] = $this->params['named']['includeServerCorrelations'];
         }
         $results = $this->Event->fetchEvent($this->Auth->user(), $conditions);
-        if (!empty($this->params['named']['includeGranularCorrelations'])) {
-            foreach ($results as $k => $event) {
-                if (!empty($event['RelatedAttribute'])) {
-                    foreach ($event['RelatedAttribute'] as $attribute_id => $relation) {
-                        foreach ($event['Attribute'] as $k2 => $attribute) {
-                            if ((int)$attribute['id'] == $attribute_id) {
-                                $results[$k]['Attribute'][$k2]['RelatedAttribute'][] = $relation;
-                                break 2;
-                            }
-                        }
-                        foreach ($event['Object'] as $k2 => $object) {
-                            foreach ($object['Attribute'] as $k3 => $attribute) {
-                                if ((int)$attribute['id'] == $attribute_id) {
-                                    $results[$k]['Object'][$k2]['Attribute'][$k3]['RelatedAttribute'][] = $relation;
-                                    break 3;
-                                }
-                            }
+        if (empty($results)) {
+            throw new NotFoundException(__('Invalid event'));
+        }
+        $event = $results[0];
+
+        // Attach related attributes to proper attribute
+        if (!empty($this->params['named']['includeGranularCorrelations']) && !empty($event['RelatedAttribute'])) {
+            foreach ($event['RelatedAttribute'] as $attribute_id => $relation) {
+                foreach ($event['Attribute'] as $k2 => $attribute) {
+                    if ((int)$attribute['id'] == $attribute_id) {
+                        $event['Attribute'][$k2]['RelatedAttribute'][] = $relation;
+                        break 2;
+                    }
+                }
+                foreach ($event['Object'] as $k2 => $object) {
+                    foreach ($object['Attribute'] as $k3 => $attribute) {
+                        if ((int)$attribute['id'] == $attribute_id) {
+                            $event['Object'][$k2]['Attribute'][$k3]['RelatedAttribute'][] = $relation;
+                            break 3;
                         }
                     }
                 }
             }
         }
-        if (empty($results)) {
-            throw new NotFoundException(__('Invalid event'));
-        }
-        $event = $results[0];
+
         $this->Event->id = $event['Event']['id'];
         if (isset($this->params['named']['searchFor']) && $this->params['named']['searchFor'] !== '') {
             $this->__applyQueryString($event, $this->params['named']['searchFor']);
@@ -1627,11 +1630,11 @@ class EventsController extends AppController
 
         if ($this->_isRest()) {
             $this->set('event', $event);
-        }
-        $this->set('deleted', isset($deleted) ? ($deleted == 2 ? 0 : 1) : 0);
-        $this->set('includeRelatedTags', (!empty($this->params['named']['includeRelatedTags'])) ? 1 : 0);
-        $this->set('includeDecayScore', (!empty($this->params['named']['includeDecayScore'])) ? 1 : 0);
-        if (!$this->_isRest()) {
+        } else {
+            $this->set('deleted', isset($deleted) ? ($deleted == 2 ? 0 : 1) : 0);
+            $this->set('includeRelatedTags', (!empty($this->params['named']['includeRelatedTags'])) ? 1 : 0);
+            $this->set('includeDecayScore', (!empty($this->params['named']['includeDecayScore'])) ? 1 : 0);
+
             if ($this->_isSiteAdmin() && $event['Event']['orgc_id'] !== $this->Auth->user('org_id')) {
                 $this->Flash->info(__('You are currently logged in as a site administrator and about to edit an event not belonging to your organisation. This goes against the sharing model of MISP. Use a normal user account for day to day work.'));
             }
@@ -1721,12 +1724,13 @@ class EventsController extends AppController
         $this->redirect(array('controller' => 'events', 'action' => 'view', $eventId, true, $eventId));
     }
 
-    private function __applyQueryString(&$event, $searchFor, $filterColumnsOverwrite=false) {
+    private function __applyQueryString(&$event, $searchFor, $filterColumnsOverwrite=false)
+    {
         // filtering on specific columns is specified
         if ($filterColumnsOverwrite !== false) {
             $filterValue = array_map('trim', explode(",", $filterColumnsOverwrite));
         } else {
-            $filterColumnsOverwrite = empty(Configure::read('MISP.event_view_filter_fields')) ? 'id, uuid, value, comment, type, category, Tag.name' : Configure::read('MISP.event_view_filter_fields');
+            $filterColumnsOverwrite = Configure::read('MISP.event_view_filter_fields') ?: 'id,uuid,value,comment,type,category,Tag.name';
             $filterValue = array_map('trim', explode(",", $filterColumnsOverwrite));
             $validFilters = array('id', 'uuid', 'value', 'comment', 'type', 'category', 'Tag.name');
             foreach ($filterValue as $k => $v) {
@@ -1759,7 +1763,7 @@ class EventsController extends AppController
                     unset($event['Object'][$k]['Attribute'][$k2]);
                 }
             }
-            if (count($event['Object'][$k]['Attribute']) == 0) {
+            if (empty($event['Object'][$k]['Attribute'])) {
                 // remove object if empty
                 unset($event['Object'][$k]);
             } else {
@@ -1771,7 +1775,7 @@ class EventsController extends AppController
 
     // look in the parameters if we are doing advanced filtering or not
     private function __checkIfAdvancedFiltering($filters) {
-        $advancedFilteringActive = array_diff_key($filters, array('sort'=>0, 'direction'=>0, 'focus'=>0, 'extended'=>0, 'overrideLimit'=>0, 'filterColumnsOverwrite'=>0, 'attributeFilter'=>0, 'extended' => 0, 'page' => 0));
+        $advancedFilteringActive = array_diff_key($filters, array('sort'=>0, 'direction'=>0, 'focus'=>0, 'overrideLimit'=>0, 'filterColumnsOverwrite'=>0, 'attributeFilter'=>0, 'extended' => 0, 'page' => 0));
 
         if (count($advancedFilteringActive) > 0) {
             if (count(array_diff_key($advancedFilteringActive, array('deleted', 'includeRelatedTags', 'includeDecayScore'))) > 0) {
@@ -1921,7 +1925,8 @@ class EventsController extends AppController
                             throw new ForbiddenException(__('Event blocked by local blocklist.'));
                         }
                         // REST users want to see the newly created event
-                        $results = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $created_id));
+                        $metadata = $this->request->param('named.metadata');
+                        $results = $this->Event->fetchEvent($this->Auth->user(), ['eventid' => $created_id, 'metadata' => $metadata]);
                         $event = $results[0];
                         if (!empty($validationErrors)) {
                             $event['errors'] = $validationErrors;
@@ -2048,11 +2053,19 @@ class EventsController extends AppController
                     throw new MethodNotAllowedException(__('File upload failed or file does not have the expected extension (.xml / .json).'));
                 }
                 if (isset($this->data['Event']['submittedfile'])) {
-                    if (Configure::read('MISP.take_ownership_xml_import')
-                        && (isset($this->data['Event']['takeownership']) && $this->data['Event']['takeownership'] == 1)) {
-                        $results = $this->_addMISPExportFile($ext, true, $this->data['Event']['publish']);
-                    } else {
-                        $results = $this->_addMISPExportFile($ext, false, $this->data['Event']['publish']);
+                    $isXml = strtolower($ext) === 'xml';
+                    App::uses('FileAccessTool', 'Tools');
+                    $data = (new FileAccessTool())->readFromFile($this->data['Event']['submittedfile']['tmp_name'], $this->data['Event']['submittedfile']['size']);
+                    $takeOwnership = Configure::read('MISP.take_ownership_xml_import')
+                        && (isset($this->data['Event']['takeownership']) && $this->data['Event']['takeownership'] == 1);
+
+                    try {
+                        $results = $this->Event->addMISPExportFile($this->Auth->user(), $data, $isXml, $takeOwnership, $this->data['Event']['publish']);
+                    } catch (Exception $e) {
+                        $filename = $this->data['Event']['submittedfile']['name'];
+                        $this->log("Exception during processing MISP file import '$filename': {$e->getMessage()}");
+                        $this->Flash->error(__('Could not process MISP export file. Probably file content is invalid.'));
+                        $this->redirect(['controller' => 'events', 'action' => 'add_misp_export']);
                     }
                 }
             }
@@ -2244,22 +2257,13 @@ class EventsController extends AppController
         if ($this->request->is('get') && $this->_isRest()) {
             return $this->RestResponse->describe('Events', 'edit', false, $this->response->type());
         }
-        if (Validation::uuid($id)) {
-            $temp = $this->Event->find('first', array('recursive' => -1, 'fields' => array('Event.id'), 'conditions' => array('Event.uuid' => $id)));
-            if (empty($temp)) {
-                throw new NotFoundException(__('Invalid event'));
-            }
-            $id = $temp['Event']['id'];
-        } elseif (!is_numeric($id)) {
+        $event = $this->Event->fetchSimpleEvent($this->Auth->user(), $id);
+        if (!$event) {
             throw new NotFoundException(__('Invalid event'));
         }
-        $this->Event->id = $id;
-        if (!$this->Event->exists()) {
-            throw new NotFoundException(__('Invalid event'));
-        }
-        $this->Event->read(null, $id);
+        $id = $event['Event']['id']; // change possible event UUID with real ID
         // check if private and user not authorised to edit
-        if (!$this->__canModifyEvent($this->Event->data) && !($this->userRole['perm_sync'] && $this->_isRest())) {
+        if (!$this->__canModifyEvent($event) && !($this->userRole['perm_sync'] && $this->_isRest())) {
             $message = __('You are not authorised to do that.');
             if ($this->_isRest()) {
                 throw new ForbiddenException($message);
@@ -2272,6 +2276,7 @@ class EventsController extends AppController
             $this->Event->insertLock($this->Auth->user(), $id);
         }
         if ($this->request->is('post') || $this->request->is('put')) {
+            $this->Event->set($event);
             if ($this->_isRest()) {
                 if (isset($this->request->data['response'])) {
                     $this->request->data = $this->Event->updateXMLArray($this->request->data, true);
@@ -2288,7 +2293,8 @@ class EventsController extends AppController
                 $result = $this->Event->_edit($this->request->data, $this->Auth->user(), $id);
                 if ($result === true) {
                     // REST users want to see the newly created event
-                    $results = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $id));
+                    $metadata = $this->request->param('named.metadata');
+                    $results = $this->Event->fetchEvent($this->Auth->user(), ['eventid' => $id, 'metadata' => $metadata]);
                     $event = $results[0];
                     $this->set('event', $event);
                     $this->render('view');
@@ -2296,8 +2302,6 @@ class EventsController extends AppController
                 } else {
                     $message = 'Error';
                     if ($this->_isRest()) {
-                        App::uses('JSONConverterTool', 'Tools');
-                        $converter = new JSONConverterTool();
                         if (isset($result['error'])) {
                             $errors = $result['error'];
                         } else {
@@ -2314,11 +2318,10 @@ class EventsController extends AppController
             // say what fields are to be updated
             $fieldList = array('date', 'threat_level_id', 'analysis', 'info', 'published', 'distribution', 'timestamp', 'sharing_group_id', 'extends_uuid');
 
-            $this->Event->read();
             // always force the org, but do not force it for admins
             if (!$this->_isSiteAdmin()) {
                 // set the same org as existed before
-                $this->request->data['Event']['org_id'] = $this->Event->data['Event']['org_id'];
+                $this->request->data['Event']['org_id'] = $event['Event']['org_id'];
             }
             // we probably also want to remove the published flag
             $this->request->data['Event']['published'] = 0;
@@ -2331,10 +2334,7 @@ class EventsController extends AppController
                 $this->Flash->error(__('The event could not be saved. Please, try again.'));
             }
         } else {
-            if (!$this->userRole['perm_modify']) {
-                $this->redirect(array('controller' => 'events', 'action' => 'index', 'admin' => false));
-            }
-            $this->request->data = $this->Event->read(null, $id);
+            $this->request->data = $event;
         }
 
         // combobox for distribution
@@ -2372,7 +2372,7 @@ class EventsController extends AppController
         $this->set('analysisLevels', $analysisLevels);
         $this->set('fieldDesc', $fieldDesc);
         $this->set('eventDescriptions', $this->Event->fieldDescriptions);
-        $this->set('event', $this->Event->data);
+        $this->set('event', $event);
         $this->render('add');
     }
 
@@ -2952,18 +2952,6 @@ class EventsController extends AppController
         return $this->restSearch();
     }
 
-    // Grab an event or a list of events for the event view or any of the XML exports. The returned object includes an array of events (or an array that only includes a single event if an ID was given)
-    // Included with the event are the attached attributes, shadow attributes, related events, related attribute information for the event view and the creating user's email address where appropriate
-    private function __fetchEvent($eventid = false, $idList = false, $user = false, $tags = false, $from = false, $to = false)
-    {
-        // if we come from automation, we may not be logged in - instead we used an auth key in the URL.
-        if (empty($user)) {
-            $user = $this->Auth->user();
-        }
-        $results = $this->Event->fetchEvent($user, array('eventid' => $eventid, 'idList' => $idList, 'tags' => $tags, 'from' => $from, 'to' => $to));
-        return $results;
-    }
-
     public function nids()
     {
         $this->_legacyAPIRemap(array(
@@ -3123,66 +3111,6 @@ class EventsController extends AppController
         }
     }
 
-    private function _addMISPExportFile($ext, $take_ownership = false, $publish = false)
-    {
-        App::uses('FileAccessTool', 'Tools');
-        $data = (new FileAccessTool())->readFromFile($this->data['Event']['submittedfile']['tmp_name'], $this->data['Event']['submittedfile']['size']);
-
-        if ($ext == 'xml') {
-            App::uses('Xml', 'Utility');
-            $dataArray = Xml::toArray(Xml::build($data));
-        } else {
-            $dataArray = json_decode($data, true);
-            if (isset($dataArray['response'][0])) {
-                foreach ($dataArray['response'] as $k => $temp) {
-                    $dataArray['Event'][] = $temp['Event'];
-                    unset($dataArray['response'][$k]);
-                }
-            }
-        }
-        // In case we receive an event that is not encapsulated in a response. This should never happen (unless it's a copy+paste fail),
-        // but just in case, let's clean it up anyway.
-        if (isset($dataArray['Event'])) {
-            $dataArray['response']['Event'] = $dataArray['Event'];
-            unset($dataArray['Event']);
-        }
-        if (!isset($dataArray['response']) || !isset($dataArray['response']['Event'])) {
-            throw new Exception(__('This is not a valid MISP XML file.'));
-        }
-        $dataArray = $this->Event->updateXMLArray($dataArray);
-        $results = array();
-        $validationIssues = array();
-        if (isset($dataArray['response']['Event'][0])) {
-            foreach ($dataArray['response']['Event'] as $k => $event) {
-                $result = array('info' => $event['info']);
-                if ($take_ownership) {
-                    $event['orgc_id'] = $this->Auth->user('org_id');
-                    unset($event['Orgc']);
-                }
-                $event = array('Event' => $event);
-                $created_id = 0;
-                $event['Event']['locked'] = 1;
-                $event['Event']['published'] = $publish;
-                $result['result'] = $this->Event->_add($event, true, $this->Auth->user(), '', null, false, null, $created_id, $validationIssues);
-                $result['id'] = $created_id;
-                $result['validationIssues'] = $validationIssues;
-                $results[] = $result;
-            }
-        } else {
-            $temp['Event'] = $dataArray['response']['Event'];
-            if ($take_ownership) {
-                $temp['Event']['orgc_id'] = $this->Auth->user('org_id');
-                unset($temp['Event']['Orgc']);
-            }
-            $created_id = 0;
-            $temp['Event']['locked'] = 1;
-            $temp['Event']['published'] = $publish;
-            $result = $this->Event->_add($temp, true, $this->Auth->user(), '', null, false, null, $created_id, $validationIssues);
-            $results = array(0 => array('info' => $temp['Event']['info'], 'result' => $result, 'id' => $created_id, 'validationIssues' => $validationIssues));
-        }
-        return $results;
-    }
-
     public function downloadOpenIOCEvent($key, $eventid, $enforceWarninglist = false)
     {
         // return a downloadable text file called misp.openIOC.<eventId>.ioc for individual events
@@ -3687,19 +3615,7 @@ class EventsController extends AppController
             $this->set('importComment', '');
             $this->set('title', 'Freetext Import Results');
             $this->loadModel('Warninglist');
-            $tldLists = $this->Warninglist->getTldLists();
-            $missingTldLists = array();
-            foreach ($tldLists as $tldList) {
-                $temp = $this->Warninglist->find('first', array(
-                    'recursive' => -1,
-                    'conditions' => array('Warninglist.name' => $tldList),
-                    'fields' => array('Warninglist.id')
-                ));
-                if (empty($temp)) {
-                    $missingTldLists[] = $tldList;
-                }
-            }
-            $this->set('missingTldLists', $missingTldLists);
+            $this->set('missingTldLists', $this->Warninglist->missingTldLists());
             $this->render('resolved_attributes');
         }
     }
@@ -3748,20 +3664,25 @@ class EventsController extends AppController
         if (!$this->userRole['perm_add']) {
             throw new MethodNotAllowedException(__('Event not found or you don\'t have permissions to create attributes'));
         }
+        if (!$this->request->is('post')) {
+            throw new MethodNotAllowedException('This endpoint requires a POST request.');
+        }
         $event = $this->Event->fetchSimpleEvent($this->Auth->user(), $id);
         if (!$event) {
             throw new NotFoundException(__('Invalid event.'));
         }
-        if ($this->request->is('post')) {
-            $this->Event->insertLock($this->Auth->user(), $id);
-            $attributes = json_decode($this->request->data['Attribute']['JsonObject'], true);
-            $default_comment = $this->request->data['Attribute']['default_comment'];
-            $proposals = !$this->__canModifyEvent($event) || (isset($this->request->data['Attribute']['force']) && $this->request->data['Attribute']['force']);
-            $flashMessage = $this->Event->processFreeTextDataRouter($this->Auth->user(), $attributes, $id, $default_comment, $proposals);
-            $this->Flash->info($flashMessage);
-            $this->redirect(array('controller' => 'events', 'action' => 'view', $id));
+
+        $this->Event->insertLock($this->Auth->user(), $id);
+        $attributes = json_decode($this->request->data['Attribute']['JsonObject'], true);
+        $default_comment = $this->request->data['Attribute']['default_comment'];
+        $proposals = !$this->__canModifyEvent($event) || (isset($this->request->data['Attribute']['force']) && $this->request->data['Attribute']['force']);
+        $flashMessage = $this->Event->processFreeTextDataRouter($this->Auth->user(), $attributes, $id, $default_comment, $proposals);
+        $this->Flash->info($flashMessage);
+
+        if ($this->request->is('ajax')) {
+            return $this->RestResponse->viewData($flashMessage, $this->response->type());
         } else {
-            throw new MethodNotAllowedException('This endpoint requires a POST request.');
+            $this->redirect(array('controller' => 'events', 'action' => 'view', $id));
         }
     }
 
@@ -4371,21 +4292,6 @@ class EventsController extends AppController
         $this->set('id', $id);
     }
 
-    public function viewEventGraph()
-    {
-        $event = $this->Event->fetchEvent($this->Auth->user(), array(
-            'eventid' => $id
-        ));
-        if (empty($event)) {
-            throw new MethodNotAllowedException(__('Invalid Event.'));
-        }
-        $this->set('event', $event[0]);
-        $this->set('scope', 'event');
-        $this->set('id', $id);
-    }
-
-
-
     /*
         public function deleteNode($id) {
             if (!$this->request->is('post')) throw new MethodNotAllowedException(__('Only POST requests are allowed.'));
@@ -4418,16 +4324,17 @@ class EventsController extends AppController
         return new CakeResponse(array('body' => json_encode($json), 'status' => 200, 'type' => 'json'));
     }
 
-    private function genDistributionGraph($id, $type = 'event', $extended = 0) {
+    private function genDistributionGraph($id, $type = 'event', $extended = 0)
+    {
         $validTools = array('event');
         if (!in_array($type, $validTools)) {
             throw new MethodNotAllowedException(__('Invalid type.'));
         }
-        $this->loadModel('Server');
-        $this->loadModel('Organisation');
+
         App::uses('DistributionGraphTool', 'Tools');
         $grapher = new DistributionGraphTool();
 
+        $this->loadModel('Server');
         $servers = $this->Server->find('list', array(
             'fields' => array('name'),
         ));
@@ -4605,11 +4512,16 @@ class EventsController extends AppController
 
     public function viewGalaxyMatrix($scope_id, $galaxy_id, $scope='event', $disable_picking=false)
     {
-        $local = !empty($this->params['named']['local']);
-        $this->set('local', $local);
         $this->loadModel('Galaxy');
         $mitreAttackGalaxyId = $this->Galaxy->getMitreAttackGalaxyId();
-        $matrixData = $this->Galaxy->getMatrix($galaxy_id);
+        if ($galaxy_id === 'mitre-attack') { // specific case for MITRE ATTACK matrix
+            $galaxy_id = $mitreAttackGalaxyId;
+        }
+
+        $matrixData = $this->Galaxy->getMatrix($galaxy_id); // throws exception if matrix not found
+
+        $local = !empty($this->params['named']['local']);
+        $this->set('local', $local);
 
         $tabs = $matrixData['tabs'];
         $matrixTags = $matrixData['matrixTags'];
@@ -4880,8 +4792,7 @@ class EventsController extends AppController
         if (!empty($options)) {
             $data['config'] = $options;
         }
-        $data = json_encode($data);
-        $result = $this->Module->queryModuleServer('/query', $data, false, $type);
+        $result = $this->Module->queryModuleServer($data, false, $type);
         if (!$result) {
             throw new MethodNotAllowedException(__('%s service not reachable.', $type));
         }
@@ -4926,8 +4837,7 @@ class EventsController extends AppController
         if (!empty($options)) {
             $data['config'] = $options;
         }
-        $data = json_encode($data);
-        $result = $this->Module->queryModuleServer('/query', $data, false, $type);
+        $result = $this->Module->queryModuleServer($data, false, $type);
         if (!$result) {
             throw new MethodNotAllowedException(__('%s service not reachable.', $type));
         }
@@ -5003,8 +4913,8 @@ class EventsController extends AppController
             throw new ForbiddenException(__('You don\'t have permission to do that.'));
         }
 
-        $resolved_data = json_decode($this->request->data['Event']['JsonObject'], true);
-        $data = json_decode($this->request->data['Event']['data'], true);
+        $resolved_data = $this->Event->jsonDecode($this->request->data['Event']['JsonObject']);
+        $data = $this->Event->jsonDecode($this->request->data['Event']['data']);
         if (!empty($data['initialObject'])) {
             $resolved_data['initialObject'] = $data['initialObject'];
         }
@@ -5012,7 +4922,12 @@ class EventsController extends AppController
         $default_comment = $this->request->data['Event']['default_comment'];
         $flashMessage = $this->Event->processModuleResultsDataRouter($this->Auth->user(), $resolved_data, $event['Event']['id'], $default_comment);
         $this->Flash->info($flashMessage);
-        $this->redirect(array('controller' => 'events', 'action' => 'view', $event['Event']['id']));
+
+        if ($this->request->is('ajax')) {
+            return $this->RestResponse->viewData($flashMessage, $this->response->type());
+        } else {
+            $this->redirect(array('controller' => 'events', 'action' => 'view', $event['Event']['id']));
+        }
     }
 
     public function importModule($module, $eventId)
@@ -5051,25 +4966,27 @@ class EventsController extends AppController
                     $this->request->data['Event']['config']['special_delimiter'] = ' ';
                 }
             }
-            foreach ($module['mispattributes']['userConfig'] as $configName => $config) {
-                if (!$fail) {
-                    if (isset($config['validation'])) {
-                        if ($config['validation'] === '0' && $config['type'] == 'String') {
-                            $validation = true;
-                        }
-                    } else {
-                        $validation = call_user_func_array(array($this->Module, $this->Module->configTypes[$config['type']]['validation']), array($this->request->data['Event']['config'][$configName]));
-                    }
-                    if ($validation !== true) {
-                        $fail = ucfirst($configName) . ': ' . $validation;
-                    } else {
-                        if (isset($config['regex']) && !empty($config['regex'])) {
-                            $fail = preg_match($config['regex'], $this->request->data['Event']['config'][$configName]) ? false : ucfirst($configName) . ': ' . 'Invalid setting' . ($config['errorMessage'] ? ' - ' . $config['errorMessage'] : '');
-                            if (!empty($fail)) {
-                                $modulePayload['config'][$configName] = $this->request->data['Event']['config'][$configName];
+            if (isset($module['mispattributes']['userConfig'])) {
+                foreach ($module['mispattributes']['userConfig'] as $configName => $config) {
+                    if (!$fail) {
+                        if (isset($config['validation'])) {
+                            if ($config['validation'] === '0' && $config['type'] == 'String') {
+                                $validation = true;
                             }
                         } else {
-                            $modulePayload['config'][$configName] = $this->request->data['Event']['config'][$configName];
+                            $validation = call_user_func_array(array($this->Module, $this->Module->configTypes[$config['type']]['validation']), array($this->request->data['Event']['config'][$configName]));
+                        }
+                        if ($validation !== true) {
+                            $fail = ucfirst($configName) . ': ' . $validation;
+                        } else {
+                            if (isset($config['regex']) && !empty($config['regex'])) {
+                                $fail = preg_match($config['regex'], $this->request->data['Event']['config'][$configName]) ? false : ucfirst($configName) . ': ' . 'Invalid setting' . ($config['errorMessage'] ? ' - ' . $config['errorMessage'] : '');
+                                if (!empty($fail)) {
+                                    $modulePayload['config'][$configName] = $this->request->data['Event']['config'][$configName];
+                                }
+                            } else {
+                                $modulePayload['config'][$configName] = $this->request->data['Event']['config'][$configName];
+                            }
                         }
                     }
                 }
@@ -5110,7 +5027,7 @@ class EventsController extends AppController
                     if (!empty($filename)) {
                         $modulePayload['filename'] = $filename;
                     }
-                    $result = $this->Module->queryModuleServer('/query', json_encode($modulePayload), false, $moduleFamily = 'Import');
+                    $result = $this->Module->queryModuleServer($modulePayload, false, $moduleFamily = 'Import');
                     if (!$result) {
                         throw new Exception(__('Import service not reachable.'));
                     }
@@ -5181,13 +5098,15 @@ class EventsController extends AppController
                     $this->set('importComment', $importComment);
                     $this->render($render_name);
                 }
-            } else {
+            }
+            if ($fail) {
                 $this->Flash->error($fail);
             }
         }
         $this->set('configTypes', $this->Module->configTypes);
         $this->set('module', $module);
         $this->set('eventId', $eventId);
+        $this->set('event', $event);
     }
 
     public function exportModule($module, $id, $standard = false)
@@ -5739,4 +5658,53 @@ class EventsController extends AppController
             $this->redirect(['action' => 'restoreDeletedEvents']);
         }
     }
+
+    public function runTaxonomyExclusivityCheck($id)
+    {
+        $conditions = [];
+        if (is_numeric($id)) {
+            $conditions = array('eventid' => $id);
+        } else if (Validation::uuid($id)) {
+            $conditions = array('event_uuid' => $id);
+        } else {
+            throw new NotFoundException(__('Invalid event'));
+        }
+        $conditions['excludeLocalTags'] = false;
+        $conditions['excludeGalaxy'] = true;
+        $event = $this->Event->fetchEvent($this->Auth->user(), $conditions);
+        if (empty($event)) {
+            throw new NotFoundException(__('Invalid event'));
+        }
+        $event = $event[0];
+        $this->loadModel('Taxonomy');
+        $allConflicts = [];
+        $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($event['EventTag']);
+        if (!empty($tagConflicts['global']) || !empty($tagConflicts['local'])) {
+            $tagConflicts['Event'] = $event['Event'];
+            $allConflicts[] = $tagConflicts;
+        }
+        foreach ($event['Object'] as $k => $object) {
+            if (isset($object['Attribute'])) {
+                foreach ($object['Attribute'] as $k2 => $attribute) {
+                    $this->Event->Attribute->removeGalaxyClusterTags($event['Object'][$k]['Attribute'][$k2]);
+                    $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']);
+                    if (!empty($tagConflicts['global']) || !empty($tagConflicts['local'])) {
+                        $tagConflicts['Attribute'] = $event['Object'][$k]['Attribute'][$k2];
+                        unset($tagConflicts['Attribute']['AttributeTag'], $tagConflicts['Attribute']['Galaxy'], $tagConflicts['Attribute']['ShadowAttribute']);
+                        $allConflicts[] = $tagConflicts;
+                    }
+                }
+            }
+        }
+        foreach ($event['Attribute'] as $k => $attribute) {
+            $this->Event->Attribute->removeGalaxyClusterTags($event['Attribute'][$k]);
+            $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attribute['AttributeTag']);
+            if (!empty($tagConflicts['global']) || !empty($tagConflicts['local'])) {
+                $tagConflicts['Attribute'] = $event['Attribute'][$k];
+                unset($tagConflicts['Attribute']['AttributeTag'], $tagConflicts['Attribute']['Galaxy'], $tagConflicts['Attribute']['ShadowAttribute']);
+                $allConflicts[] = $tagConflicts;
+            }
+        }
+        return $this->RestResponse->viewData($allConflicts);
+    }
 }

app/Controller/FeedsController.php

@@ -773,6 +773,9 @@ class FeedsController extends AppController
             return $this->RestResponse->viewData($event, $this->response->type());
         }
         if (is_array($event)) {
+            $this->loadModel('Warninglist');
+            $this->Warninglist->attachWarninglistToAttributes($event['Event']['Attribute']);
+
             $this->loadModel('Event');
             $params = $this->Event->rearrangeEventForView($event, $this->passedArgs, $all);
             $this->params->params['paging'] = array('Feed' => $params);

app/Controller/GalaxyClustersController.php

@@ -1,9 +1,6 @@
 <?php
 App::uses('AppController', 'Controller');
 
-/**
- * @property GalaxyCluster $GalaxyCluster
- */
 class GalaxyClustersController extends AppController
 {
     public $components = array('Session', 'RequestHandler');
@@ -18,6 +15,14 @@ class GalaxyClustersController extends AppController
             'contain' => array(
                 'Tag' => array(
                     'fields' => array('Tag.id'),
+                    /*
+                    'EventTag' => array(
+                        'fields' => array('EventTag.event_id')
+                    ),
+                    'AttributeTag' => array(
+                        'fields' => array('AttributeTag.event_id', 'AttributeTag.attribute_id')
+                    )
+                    */
                 ),
                 'GalaxyElement' => array(
                     'conditions' => array('GalaxyElement.key' => 'synonyms'),

app/Controller/LogsController.php

@@ -23,9 +23,9 @@ class LogsController extends AppController
     {
         parent::beforeFilter();
 
-        // permit reuse of CSRF tokens on the search page.
+        // No need for CSRF tokens for a search
         if ('admin_search' == $this->request->params['action']) {
-            $this->Security->csrfUseOnce = false;
+            $this->Security->csrfCheck = false;
         }
     }
 
@@ -68,7 +68,7 @@ class LogsController extends AppController
                 'request' => $this->request,
                 'named_params' => $this->params['named'],
                 'paramArray' => $paramArray,
-                'ordered_url_params' => compact($paramArray)
+                'ordered_url_params' => func_get_args()
             );
             $exception = false;
             $filters = $this->_harvestParameters($filterData, $exception);

app/Controller/ModulesController.php

@@ -42,7 +42,7 @@ class ModulesController extends AppController
         }
 
         // Query
-        $result = $this->Module->queryModuleServer('/query', json_encode($data), true);
+        $result = $this->Module->queryModuleServer($data, true);
         if (!$result) {
             $result = array('error' => 'Something went wrong, no response from module.');
         }

app/Controller/ObjectsController.php

@@ -87,6 +87,15 @@ class ObjectsController extends AppController
             $similar_object_similarity_amount[$obj['Attribute']['object_id']] = $obj[0]['similarity_amount'];
         }
 
+        if (isset($this->request->data['Attribute'])) {
+            foreach ($this->request->data['Attribute'] as &$attribute) {
+                $validation = $this->MispObject->Attribute->validateAttribute($attribute, false);
+                if ($validation !== true) {
+                    $attribute['validation'] = $validation;
+                }
+            }
+        }
+
         $this->set('distributionLevels', $this->MispObject->Attribute->distributionLevels);
         $this->set('action', $action);
         $this->set('template', $template);
@@ -225,11 +234,13 @@ class ObjectsController extends AppController
                     $object['Attribute'][$k]['event_id'] = $eventId;
                     $this->MispObject->Event->Attribute->set($object['Attribute'][$k]);
                     if (!$this->MispObject->Event->Attribute->validates()) {
-                        if ($this->MispObject->Event->Attribute->validationErrors['value'][0] !== 'Composite type found but the value not in the composite (value1|value2) format.') {
+                        $validationErrors = $this->MispObject->Event->Attribute->validationErrors;
+                        $isCompositeError = isset($validationErrors['value']) && $validationErrors['value'][0] === 'Composite type found but the value not in the composite (value1|value2) format.';
+                        if (!$isCompositeError) {
                             $error = sprintf(
                                 'Could not save object as at least one attribute has failed validation (%s). %s',
                                 isset($attribute['object_relation']) ? $attribute['object_relation'] : 'No object_relation',
-                                json_encode($this->MispObject->Event->Attribute->validationErrors)
+                                json_encode($validationErrors)
                             );
                         }
                     }
@@ -539,7 +550,7 @@ class ObjectsController extends AppController
         }
         $date = new DateTime();
         $object['Object']['timestamp'] = $date->getTimestamp();
-        $object = $this->MispObject->syncObjectAndAttributeSeen($object, $forcedSeenOnElements);
+        $object = $this->MispObject->syncObjectAndAttributeSeen($object, $forcedSeenOnElements, false);
         if ($this->MispObject->save($object)) {
             $this->MispObject->Event->unpublishEvent($object['Event']['id']);
             if ($seen_changed) {
@@ -876,22 +887,32 @@ class ObjectsController extends AppController
 
     public function view($id)
     {
-        if ($this->_isRest()) {
-            $objects = $this->MispObject->fetchObjects($this->Auth->user(), array(
+        if ($this->request->is('head')) { // Just check if object exists
+            $exists = $this->MispObject->fetchObjects($this->Auth->user(), [
                 'conditions' => $this->__objectIdToConditions($id),
-            ));
-            if (!empty($objects)) {
-                $object = $objects[0];
-                if (!empty($object['Event'])) {
-                    $object['Object']['Event'] = $object['Event'];
-                }
-                if (!empty($object['Attribute'])) {
-                    $object['Object']['Attribute'] = $object['Attribute'];
-                }
-                return $this->RestResponse->viewData(array('Object' => $object['Object']), $this->response->type());
-            }
+                'metadata' => true,
+            ]);
+            return new CakeResponse(['status' => $exists ? 200 : 404]);
+        }
+
+        $objects = $this->MispObject->fetchObjects($this->Auth->user(), array(
+            'conditions' => $this->__objectIdToConditions($id),
+        ));
+        if (empty($objects)) {
             throw new NotFoundException(__('Invalid object.'));
         }
+        $object = $objects[0];
+        if ($this->_isRest()) {
+            if (!empty($object['Event'])) {
+                $object['Object']['Event'] = $object['Event'];
+            }
+            if (!empty($object['Attribute'])) {
+                $object['Object']['Attribute'] = $object['Attribute'];
+            }
+            return $this->RestResponse->viewData(array('Object' => $object['Object']), $this->response->type());
+        } else {
+            $this->redirect('/events/view/' . $object['Object']['event_id']);
+        }
     }
 
     public function orphanedObjectDiagnostics()

app/Controller/OrganisationsController.php

@@ -1,6 +1,9 @@
 <?php
 App::uses('AppController', 'Controller');
 
+/**
+ * @property Organisation $Organisation
+ */
 class OrganisationsController extends AppController
 {
     public $components = array('Session', 'RequestHandler');
@@ -87,6 +90,10 @@ class OrganisationsController extends AppController
         if ($this->_isRest()) {
             return $this->RestResponse->viewData($orgs, $this->response->type());
         } else {
+            foreach ($orgs as &$org) {
+                $org['Organisation']['country_code'] = $this->Organisation->getCountryCode($org['Organisation']['nationality']);
+            }
+
             $this->set('named', $this->params['named']);
             $this->set('scope', $scope);
             $this->set('orgs', $orgs);
@@ -152,7 +159,8 @@ class OrganisationsController extends AppController
                 }
             }
         }
-        $this->set('countries', $this->_arrayToValuesIndexArray($this->Organisation->countries));
+        $countries = array_merge(['' => __('Not specified')], $this->_arrayToValuesIndexArray($this->Organisation->getCountries()));
+        $this->set('countries', $countries);
     }
 
     public function admin_edit($id)
@@ -230,7 +238,17 @@ class OrganisationsController extends AppController
             $this->Organisation->read(null, $id);
             $this->request->data = $this->Organisation->data;
         }
-        $this->set('countries', $this->_arrayToValuesIndexArray($this->Organisation->countries));
+
+        $countries = array_merge(['' => __('Not specified')], $this->_arrayToValuesIndexArray($this->Organisation->getCountries()));
+        if (!empty($this->Organisation->data['Organisation']['nationality'])) {
+            $currentCountry = $this->Organisation->data['Organisation']['nationality'];
+            if (!isset($countries[$currentCountry])) {
+                // Append old country name to list to keep backward compatibility
+                $countries[$currentCountry] = $currentCountry;
+            }
+        }
+
+        $this->set('countries', $countries);
         $this->set('orgId', $id);
         if (is_array($this->request->data['Organisation']['restricted_to_domain'])) {
             $this->request->data['Organisation']['restricted_to_domain'] = implode("\n", $this->request->data['Organisation']['restricted_to_domain']);
@@ -355,28 +373,13 @@ class OrganisationsController extends AppController
             $org['Organisation']['user_count'] = $this->Organisation->User->getMembersCount($org['Organisation']['id']);
             return $this->RestResponse->viewData($org, $this->response->type());
         } else {
+            $org['Organisation']['country_code'] = $this->Organisation->getCountryCode($org['Organisation']['nationality']);
             $this->set('fullAccess', $fullAccess);
             $this->set('org', $org);
             $this->set('id', $id);
         }
     }
 
-    public function landingpage($id)
-    {
-        $this->Organisation->id = $id;
-        if (!$this->Organisation->exists()) {
-            throw new NotFoundException(__('Invalid organisation'));
-        }
-        $org = $this->Organisation->find('first', array('conditions' => array('id' => $id), 'fields' => array('landingpage', 'name')));
-        $landingpage = $org['Organisation']['landingpage'];
-        if (empty($landingpage)) {
-            $landingpage = __('No landing page has been created for this organisation.');
-        }
-        $this->set('landingPage', $landingpage);
-        $this->set('org', $org['Organisation']['name']);
-        $this->render('ajax/landingpage');
-    }
-
     public function fetchOrgsForSG($idList = '{}', $type)
     {
         if ($type === 'local') {

app/Controller/ServersController.php

@@ -153,25 +153,36 @@ class ServersController extends AppController
         if (!$this->_isSiteAdmin()) {
             throw new MethodNotAllowedException('You are not authorised to do that.');
         }
-        $server = $this->Server->find('first', array('conditions' => array('Server.id' => $serverId), 'recursive' => -1, 'fields' => array('Server.id', 'Server.url', 'Server.name')));
+        $server = $this->Server->find('first', array(
+            'conditions' => array('Server.id' => $serverId),
+            'recursive' => -1,
+            'fields' => array('Server.id', 'Server.url', 'Server.name'))
+        );
         if (empty($server)) {
             throw new NotFoundException('Invalid server ID.');
         }
         try {
             $event = $this->Server->previewEvent($serverId, $eventId);
         } catch (NotFoundException $e) {
-            throw new NotFoundException(__("Event '$eventId' not found."));
+            throw new NotFoundException(__("Event '%s' not found.", $eventId));
         } catch (Exception $e) {
-            $this->Flash->error(__('Download failed.') . ' ' . $e->getMessage());
+            $this->Flash->error(__('Download failed. %s', $e->getMessage()));
             $this->redirect(array('action' => 'previewIndex', $serverId));
         }
 
+        $this->loadModel('Warninglist');
+        if (isset($event['Event']['Attribute'])) {
+            $this->Warninglist->attachWarninglistToAttributes($event['Event']['Attribute']);
+        }
+        if (isset($event['Event']['ShadowAttribute'])) {
+            $this->Warninglist->attachWarninglistToAttributes($event['Event']['ShadowAttribute']);
+        }
+
         $this->loadModel('Event');
         $params = $this->Event->rearrangeEventForView($event, $this->passedArgs, $all);
         $this->params->params['paging'] = array('Server' => $params);
         $this->set('event', $event);
         $this->set('server', $server);
-        $this->loadModel('Event');
         $dataForView = array(
                 'Attribute' => array('attrDescriptions' => 'fieldDescriptions', 'distributionDescriptions' => 'distributionDescriptions', 'distributionLevels' => 'distributionLevels'),
                 'Event' => array('eventDescriptions' => 'fieldDescriptions', 'analysisLevels' => 'analysisLevels'),
@@ -796,7 +807,7 @@ class ServersController extends AppController
                 }
             }
             if ($this->_isRest()) {
-                return $this->RestResponse->saveSuccessResponse('Servers', 'push', sprintf(__('Push complete. %s events pushed, %s events could not be pushed.', count($result[0]), count($result[1]))), $this->response->type());
+                return $this->RestResponse->saveSuccessResponse('Servers', 'push', __('Push complete. %s events pushed, %s events could not be pushed.', count($result[0]), count($result[1])), $this->response->type());
             } else {
                 $this->set('successes', $result[0]);
                 $this->set('fails', $result[1]);
@@ -1115,7 +1126,14 @@ class ServersController extends AppController
                 $sessionStatus = $this->Server->sessionDiagnostics($diagnostic_errors, $sessionCount);
                 $this->set('sessionCount', $sessionCount);
 
-                $additionalViewVars = array('gpgStatus', 'sessionErrors', 'proxyStatus', 'sessionStatus', 'zmqStatus', 'stixVersion', 'cyboxVersion', 'mixboxVersion', 'maecVersion', 'stix2Version', 'pymispVersion', 'moduleStatus', 'yaraStatus', 'gpgErrors', 'proxyErrors', 'zmqErrors', 'stixOperational', 'stix', 'moduleErrors', 'moduleTypes', 'dbDiagnostics', 'dbSchemaDiagnostics', 'redisInfo');
+                $this->loadModel('AttachmentScan');
+                try {
+                    $attachmentScan = ['status' => true, 'software' => $this->AttachmentScan->diagnostic()];
+                } catch (Exception $e) {
+                    $attachmentScan = ['status' => false, 'error' => $e->getMessage()];
+                }
+
+                $additionalViewVars = array('gpgStatus', 'sessionErrors', 'proxyStatus', 'sessionStatus', 'zmqStatus', 'stixVersion', 'cyboxVersion', 'mixboxVersion', 'maecVersion', 'stix2Version', 'pymispVersion', 'moduleStatus', 'yaraStatus', 'gpgErrors', 'proxyErrors', 'zmqErrors', 'stixOperational', 'stix', 'moduleErrors', 'moduleTypes', 'dbDiagnostics', 'dbSchemaDiagnostics', 'redisInfo', 'attachmentScan');
             }
             // check whether the files are writeable
             $writeableDirs = $this->Server->writeableDirsDiagnostics($diagnostic_errors);
@@ -1240,8 +1258,12 @@ class ServersController extends AppController
 
     public function getWorkers()
     {
-        $issues = 0;
-        $worker_array = $this->Server->workerDiagnostics($issues);
+        if (Configure::read('MISP.background_jobs')) {
+            $workerIssueCount = 0;
+            $worker_array = $this->Server->workerDiagnostics($workerIssueCount);
+        } else {
+            $worker_array = [__('Background jobs not enabled')];
+        }
         return $this->RestResponse->viewData($worker_array);
     }
 
@@ -1649,34 +1671,33 @@ class ServersController extends AppController
         if (!$this->Auth->user('Role')['perm_sync'] && !$this->Auth->user('Role')['perm_site_admin']) {
             throw new MethodNotAllowedException('You don\'t have permission to do that.');
         }
-        $this->Server->id = $id;
-        if (!$this->Server->exists()) {
+
+        $server = $this->Server->find('first', ['Server.id' => $id]);
+        if (!$server) {
             throw new NotFoundException(__('Invalid server'));
         }
-        $result = $this->Server->runConnectionTest($id);
+        $result = $this->Server->runConnectionTest($server);
         if ($result['status'] == 1) {
-            $version = json_decode($result['message'], true);
-            if (isset($version['version']) && preg_match('/^[0-9]+\.+[0-9]+\.[0-9]+$/', $version['version'])) {
+            if (isset($result['info']['version']) && preg_match('/^[0-9]+\.+[0-9]+\.[0-9]+$/', $result['info']['version'])) {
                 $perm_sync = false;
-                if (isset($version['perm_sync'])) {
-                    $perm_sync = $version['perm_sync'];
+                if (isset($result['info']['perm_sync'])) {
+                    $perm_sync = $result['info']['perm_sync'];
                 }
                 $perm_sighting = false;
-                if (isset($version['perm_sighting'])) {
-                    $perm_sighting = $version['perm_sighting'];
+                if (isset($result['info']['perm_sighting'])) {
+                    $perm_sighting = $result['info']['perm_sighting'];
                 }
                 App::uses('Folder', 'Utility');
                 $file = new File(ROOT . DS . 'VERSION.json', true);
                 $local_version = json_decode($file->read(), true);
                 $file->close();
-                $version = explode('.', $version['version']);
+                $version = explode('.', $result['info']['version']);
                 $mismatch = false;
                 $newer = false;
                 $parts = array('major', 'minor', 'hotfix');
                 if ($version[0] == 2 && $version[1] == 4 && $version[2] > 68) {
-                    $post = $this->Server->runPOSTTest($id);
+                    $post = $this->Server->runPOSTTest($server);
                 }
-                $testPost = false;
                 foreach ($parts as $k => $v) {
                     if (!$mismatch) {
                         if ($version[$k] > $local_version[$v]) {
@@ -1708,7 +1729,8 @@ class ServersController extends AppController
                                 'version' => implode('.', $version),
                                 'mismatch' => $mismatch,
                                 'newer' => $newer,
-                                'post' => isset($post) ? $post : 'too old'
+                                'post' => isset($post) ? $post : 'too old',
+                                'client_certificate' => $result['client_certificate'],
                                 )
                             ),
                             'type' => 'json'
@@ -1990,7 +2012,7 @@ class ServersController extends AppController
             'body' => empty($request['body']) ? '' : $request['body'],
             'url' => $request['url'],
             'http_method' => $request['method'],
-            'use_full_path' => $request['use_full_path'],
+            'use_full_path' => empty($request['use_full_path']) ? false : $request['use_full_path'],
             'show_result' => $request['show_result'],
             'skip_ssl' => $request['skip_ssl_validation'],
             'bookmark' => $request['bookmark'],
@@ -1998,9 +2020,9 @@ class ServersController extends AppController
             'timestamp' => $date->getTimestamp()
         );
         if (!empty($request['url'])) {
-            if (empty($request['use_full_path'])) {
+            if (empty($request['use_full_path']) || empty(Configure::read('Security.rest_client_enable_arbitrary_urls'))) {
                 $path = preg_replace('#^(://|[^/?])+#', '', $request['url']);
-                $url = Configure::read('MISP.baseurl') . $path;
+                $url = empty(Configure::read('Security.rest_client_baseurl')) ? (Configure::read('MISP.baseurl') . $path) : (Configure::read('Security.rest_client_baseurl') . $path);
                 unset($request['url']);
             } else {
                 $url = $request['url'];
@@ -2070,6 +2092,7 @@ class ServersController extends AppController
         }
         $view_data['duration'] = microtime(true) - $start;
         $view_data['duration'] = round($view_data['duration'] * 1000, 2) . 'ms';
+        $view_data['url'] = $url;
         $view_data['code'] =  $response->code;
         $view_data['headers'] = $response->headers;
         if (!empty($request['show_result'])) {

app/Controller/TagsController.php

@@ -2,9 +2,6 @@
 
 App::uses('AppController', 'Controller');
 
-/**
- * @property Tag $Tag
- */
 class TagsController extends AppController
 {
     public $components = array('Security' ,'RequestHandler');
@@ -30,6 +27,12 @@ class TagsController extends AppController
 
     public $helpers = array('TextColour');
 
+    public function beforeFilter()
+    {
+        parent::beforeFilter();
+        $this->Security->unlockedActions[] = 'search';
+    }
+
     public function index($favouritesOnly = false)
     {
         $this->loadModel('Attribute');
@@ -38,6 +41,14 @@ class TagsController extends AppController
         if ($this->_isSiteAdmin()) {
             $this->paginate['contain']['User'] = array('fields' => array('id', 'email'));
         }
+        $filterData = array(
+            'request' => $this->request,
+            'named_params' => $this->params['named'],
+            'paramArray' => ['favouritesOnly', 'filter', 'searchall', 'name', 'search', 'exclude_statistics'],
+            'ordered_url_params' => @compact($paramArray)
+        );
+        $exception = false;
+        $passedArgsArray = $this->_harvestParameters($filterData, $exception);
         $taxonomies = $this->Taxonomy->listTaxonomies(array('full' => false, 'enabled' => true));
         $taxonomyNamespaces = array();
         if (!empty($taxonomies)) {
@@ -46,9 +57,8 @@ class TagsController extends AppController
             }
         }
         $taxonomyTags = array();
-        $passedArgsArray = array();
         $this->Event->recursive = -1;
-        if ($favouritesOnly) {
+        if (!empty($passedArgsArray['favouritesOnly'])) {
             $tag_id_list = $this->Tag->FavouriteTag->find('list', array(
                     'conditions' => array('FavouriteTag.user_id' => $this->Auth->user('id')),
                     'fields' => array('FavouriteTag.tag_id')
@@ -58,47 +68,48 @@ class TagsController extends AppController
             }
             $this->paginate['conditions']['AND']['Tag.id'] = $tag_id_list;
         }
-        if (isset($this->params['named']['searchall'])) {
-            $passedArgsArray['all'] = $this->params['named']['searchall'];
-        } elseif ($this->request->is('post')) {
-            $validNames = array('filter', 'searchall', 'name', 'search');
-            foreach ($validNames as $vn) {
-                if (!empty($this->request->data[$vn])) {
-                    $passedArgsArray['all'] = $this->request->data[$vn];
-                    continue;
-                }
-            }
+        if (!empty($passedArgsArray['searchall'])) {
+            $this->paginate['conditions']['AND'][] = ['LOWER(Tag.name) LIKE' => '%' . strtolower($passedArgsArray['searchall']) . '%'];
         }
-        if (!empty($passedArgsArray['all'])) {
-            $this->paginate['conditions']['AND']['LOWER(Tag.name) LIKE'] = '%' . strtolower($passedArgsArray['all']) . '%';
+        foreach (['name', 'filter', 'search'] as $f) {
+            if (!empty($passedArgsArray['name'])) {
+                $this->paginate['conditions']['AND'][] = ['LOWER(Tag.name)' => strtolower($passedArgsArray[$f])];
+            }
         }
         if ($this->_isRest()) {
             unset($this->paginate['limit']);
+            unset($this->paginate['contain']['EventTag']);
+            unset($this->paginate['contain']['AttributeTag']);
             $paginated = $this->Tag->find('all', $this->paginate);
         } else {
+            if (!empty($passedArgsArray['exclude_statistics'])) {
+                unset($this->paginate['contain']['EventTag']);
+                unset($this->paginate['contain']['AttributeTag']);
+                $this->set('exclude_statistics', true);
+            }
             $paginated = $this->paginate();
         }
         $tagList = array();
         $csv = array();
+        $sgs = $this->Tag->EventTag->Event->SharingGroup->fetchAllAuthorised($this->Auth->user());
         foreach ($paginated as $k => $tag) {
             $tagList[] = $tag['Tag']['id'];
-            $paginated[$k]['Tag']['count'] = $this->Tag->EventTag->countForTag($tag, $this->Auth->user());
-            $paginated[$k]['Tag']['attribute_count'] = $this->Tag->AttributeTag->countForTag($tag, $this->Auth->user());
-
-            if (!$this->_isRest()) {
-                $paginated[$k]['event_ids'] = array();
-                foreach ($paginated[$k]['EventTag'] as $et) {
-                    $paginated[$k]['event_ids'][] = $et['event_id'];
-                }
-                $paginated[$k]['attribute_ids'] = array();
-                foreach ($paginated[$k]['AttributeTag'] as $at) {
-                    $paginated[$k]['attribute_ids'][] = $at['attribute_id'];
+            if (empty($passedArgsArray['exclude_statistics'])) {
+                $paginated[$k]['Tag']['count'] = $this->Tag->EventTag->countForTag($tag['Tag']['id'], $this->Auth->user(), $sgs);
+                if (!$this->_isRest()) {
+                    $paginated[$k]['event_ids'] = array();
+                    $paginated[$k]['attribute_ids'] = array();
+                    foreach ($paginated[$k]['EventTag'] as $et) {
+                        $paginated[$k]['event_ids'][] = $et['event_id'];
+                    }
+                    unset($paginated[$k]['EventTag']);
+                    foreach ($paginated[$k]['AttributeTag'] as $at) {
+                        $paginated[$k]['attribute_ids'][] = $at['attribute_id'];
+                    }
+                    unset($paginated[$k]['AttributeTag']);
                 }
+                $paginated[$k]['Tag']['attribute_count'] = $this->Tag->AttributeTag->countForTag($tag['Tag']['id'], $this->Auth->user(), $sgs);
             }
-
-            unset($paginated[$k]['EventTag']);
-            unset($paginated[$k]['AttributeTag']);
-
             if (!empty($tag['FavouriteTag'])) {
                 foreach ($tag['FavouriteTag'] as $ft) {
                     if ($ft['user_id'] == $this->Auth->user('id')) {
@@ -125,7 +136,7 @@ class TagsController extends AppController
                 }
             }
         }
-        if (!$this->_isRest()) {
+        if (!$this->_isRest() && empty($passedArgsArray['exclude_statistics'])) {
             $this->loadModel('Sighting');
             $sightings['event'] = $this->Sighting->getSightingsForObjectIds($this->Auth->user(), $tagList);
             $sightings['attribute'] = $this->Sighting->getSightingsForObjectIds($this->Auth->user(), $tagList, 'attribute');
@@ -149,13 +160,13 @@ class TagsController extends AppController
                     $startDate = date('Y-m-d', strtotime("-3 days", strtotime($startDate)));
                     $to = date('Y-m-d', time());
                     for ($date = $startDate; strtotime($date) <= strtotime($to); $date = date('Y-m-d', strtotime("+1 day", strtotime($date)))) {
-                        if (!isset($csv[$k])) {
-                            $csv[$k] = 'Date,Close\n';
+                        if (!isset($paginated[$k]['Tag']['csv'])) {
+                            $paginated[$k]['Tag']['csv'] = 'Date,Close\n';
                         }
                         if (isset($tag['sightings'][$date])) {
-                            $csv[$k] .= $date . ',' . $tag['sightings'][$date] . '\n';
+                            $paginated[$k]['Tag']['csv'] .= $date . ',' . $tag['sightings'][$date] . '\n';
                         } else {
-                            $csv[$k] .= $date . ',0\n';
+                            $paginated[$k]['Tag']['csv'] .= $date . ',0\n';
                         }
                     }
                 }
@@ -398,13 +409,58 @@ class TagsController extends AppController
             if (empty($tag['EventTag'])) {
                 $tag['Tag']['count'] = 0;
             } else {
-                $tag['Tag']['count'] = $this->Tag->EventTag->countForTag($tag, $this->Auth->user());
+                $eventIDs = array();
+                foreach ($tag['EventTag'] as $eventTag) {
+                    $eventIDs[] = $eventTag['event_id'];
+                }
+                $conditions = array('Event.id' => $eventIDs);
+                if (!$this->_isSiteAdmin()) {
+                    $conditions = array_merge(
+                        $conditions,
+                        array('OR' => array(
+                                array('AND' => array(
+                                        array('Event.distribution >' => 0),
+                                        array('Event.published =' => 1)
+                                )),
+                                array('Event.orgc_id' => $this->Auth->user('org_id'))
+                        ))
+                );
+                }
+                $events = $this->Tag->EventTag->Event->find('all', array(
+                        'fields' => array('Event.id', 'Event.distribution', 'Event.orgc_id'),
+                        'conditions' => $conditions
+                ));
+                $tag['Tag']['count'] = count($events);
             }
             unset($tag['EventTag']);
             if (empty($tag['AttributeTag'])) {
                 $tag['Tag']['attribute_count'] = 0;
             } else {
-                $tag['Tag']['attribute_count'] = $this->Tag->AttributeTag->countForTag($tag, $this->Auth->user());;
+                $attributeIDs = array();
+                foreach ($tag['AttributeTag'] as $attributeTag) {
+                    $attributeIDs[] = $attributeTag['attribute_id'];
+                }
+                $conditions = array('Attribute.id' => $attributeIDs);
+                if (!$this->_isSiteAdmin()) {
+                    $conditions = array_merge(
+                        $conditions,
+                        array('OR' => array(
+                            array('AND' => array(
+                                array('Attribute.deleted =' => 0),
+                                array('Attribute.distribution >' => 0),
+                                array('Event.distribution >' => 0),
+                                array('Event.published =' => 1)
+                            )),
+                            array('Event.orgc_id' => $this->Auth->user('org_id'))
+                        ))
+                    );
+                }
+                $attributes = $this->Tag->AttributeTag->Attribute->find('all', array(
+                    'fields'     => array('Attribute.id', 'Attribute.deleted', 'Attribute.distribution', 'Event.id', 'Event.distribution', 'Event.orgc_id'),
+                    'contain'    => array('Event' => array('fields' => array('id', 'distribution', 'orgc_id'))),
+                    'conditions' => $conditions
+                ));
+                $tag['Tag']['attribute_count'] = count($attributes);
             }
             unset($tag['AttributeTag']);
             $this->set('Tag', $tag['Tag']);
@@ -885,7 +941,7 @@ class TagsController extends AppController
             if (is_numeric($tag)) {
                 $conditions = array('Tag.id' => $tag);
             } else {
-                $conditions = array('LOWER(Tag.name) LIKE' => strtolower(trim($tag)));
+                $conditions = array('Tag.name LIKE' => trim($tag));
             }
             if (empty($local)) {
                 if (!empty($this->request->data['local'])) {
@@ -1067,7 +1123,7 @@ class TagsController extends AppController
         $this->render('/Events/view_graph');
     }
 
-    public function search($tag = false)
+    public function search($tag = false, $strictTagNameOnly = false, $searchIfTagExists = true)
     {
         if (isset($this->request->data['Tag'])) {
             $this->request->data = $this->request->data['Tag'];
@@ -1080,41 +1136,60 @@ class TagsController extends AppController
         if (!is_array($tag)) {
             $tag = array($tag);
         }
-        $conditions = array();
-        foreach ($tag as $k => $t) {
-            $tag[$k] = strtolower($t);
-            $conditions['OR'][] = array('LOWER(GalaxyCluster.value)' => $tag[$k]);
-        }
-        foreach ($tag as $k => $t) {
-            $conditions['OR'][] = array('AND' => array('GalaxyElement.key' => 'synonyms', 'LOWER(GalaxyElement.value) LIKE' => $t));
-        }
         $this->loadModel('GalaxyCluster');
-        $elements = $this->GalaxyCluster->GalaxyElement->find('all', array(
-            'recursive' => -1,
-            'conditions' => $conditions,
-            'contain' => array('GalaxyCluster.tag_name')
-        ));
-        foreach ($elements as $element) {
-            $tag[] = strtolower($element['GalaxyCluster']['tag_name']);
-        }
         $conditions = array();
-        foreach ($tag as $k => $t) {
-            $conditions['OR'][] = array('LOWER(Tag.name) LIKE' => $t);
+        if (!$strictTagNameOnly) {
+            $conditionsCluster = [];
+            foreach ($tag as $k => $t) {
+                $tag[$k] = strtolower($t);
+                $conditionsCluster['OR'][] = array('LOWER(GalaxyCluster.value)' => $tag[$k]);
+            }
+            foreach ($tag as $k => $t) {
+                $conditionsCluster['OR'][] = array('AND' => array('GalaxyElement.key' => 'synonyms', 'LOWER(GalaxyElement.value) LIKE' => $t));
+            }
+            $elements = $this->GalaxyCluster->GalaxyElement->find('all', array(
+                'recursive' => -1,
+                'conditions' => $conditionsCluster,
+                'contain' => array('GalaxyCluster.tag_name')
+            ));
+            foreach ($elements as $element) {
+                $tag[] = strtolower($element['GalaxyCluster']['tag_name']);
+            }
+            foreach ($tag as $k => $t) {
+                $conditions['OR'][] = array('LOWER(Tag.name) LIKE' => $t);
+            }
+        } else {
+            foreach ($tag as $k => $t) {
+                $conditions['OR'][] = array('Tag.name' => $t);
+            }
         }
         $tags = $this->Tag->find('all', array(
             'conditions' => $conditions,
             'recursive' => -1
         ));
+        if (!$searchIfTagExists && empty($tags)) {
+            $tags = [];
+            foreach ($tag as $i => $tagName) {
+                $tags[] = ['Tag' => ['name' => $tagName], 'simulatedTag' => true];
+            }
+        }
         $this->loadModel('Taxonomy');
         foreach ($tags as $k => $t) {
-            $taxonomy = $this->Taxonomy->getTaxonomyForTag($t['Tag']['name'], true);
-            if (!empty($taxonomy)) {
+            $dataFound = false;
+            $taxonomy = $this->Taxonomy->getTaxonomyForTag($t['Tag']['name'], false);
+            if (!empty($taxonomy) && !empty($taxonomy['TaxonomyPredicate'][0])) {
+                $dataFound = true;
                 $tags[$k]['Taxonomy'] = $taxonomy['Taxonomy'];
+                $tags[$k]['TaxonomyPredicate'] = $taxonomy['TaxonomyPredicate'][0];
             }
             $cluster = $this->GalaxyCluster->getCluster($t['Tag']['name'], $this->Auth->user);
             if (!empty($cluster)) {
+                $dataFound = true;
                 $tags[$k]['GalaxyCluster'] = $cluster['GalaxyCluster'];
             }
+            if (!$searchIfTagExists && !$dataFound && !empty($t['simulatedTag'])) {
+                unset($tags[$k]);
+            }
         }
         return $this->RestResponse->viewData($tags, $this->response->type());
     }

app/Controller/UsersController.php

@@ -751,7 +751,9 @@ class UsersController extends AppController
                             $password = isset($this->request->data['User']['password']) ? $this->request->data['User']['password'] : false;
                             $result = $this->User->initiatePasswordReset($user, true, true, $password);
                             if ($result && empty(Configure::read('MISP.disable_emailing'))) {
-                                $notification_message .= ' User notified of new credentials.';
+                                $notification_message .= ' ' . __('User notified of new credentials.');
+                            } else {
+                                $notification_message .= ' ' . __('User notification of new credentials could not be send.');
                             }
                         }
                         if ($this->_isRest()) {
@@ -762,7 +764,7 @@ class UsersController extends AppController
                             $user['User']['password'] = '******';
                             return $this->RestResponse->viewData($user, $this->response->type());
                         } else {
-                            $this->Flash->success(__('The user has been saved.' . $notification_message));
+                            $this->Flash->success(__('The user has been saved.') . $notification_message);
                             $this->redirect(array('action' => 'index'));
                         }
                     } else {
@@ -1229,13 +1231,7 @@ class UsersController extends AppController
       ));
       $lastUserLogin = $user['User']['last_login'];
       unset($user['User']['password']);
-      $user['User']['action'] = 'login';
-      $user['User']['last_login'] = $this->Auth->user('current_login');
-      $user['User']['current_login'] = time();
-      $this->User->save($user['User'], true, array('id', 'last_login', 'current_login'));
-      if (empty($this->Auth->authenticate['Form']['passwordHasher']) && !empty($passwordToSave)) {
-          $this->User->saveField('password', $passwordToSave);
-      }
+      $this->User->updateLoginTimes($user['User']);
       $this->User->Behaviors->enable('SysLogLogable.SysLogLogable');
       if ($lastUserLogin) {
           $readableDatetime = (new DateTime())->setTimestamp($lastUserLogin)->format('D, d M y H:i:s O'); // RFC822

app/Controller/WarninglistsController.php

@@ -1,6 +1,9 @@
 <?php
 App::uses('AppController', 'Controller');
 
+/**
+ * @property Warninglist $Warninglist
+ */
 class WarninglistsController extends AppController
 {
     public $components = array('Session', 'RequestHandler');
@@ -104,9 +107,9 @@ class WarninglistsController extends AppController
             $message = __('Could not update any of the warning lists');
         } else {
             $flashType = 'success';
-            $message = __('Successfully updated ' . $successes . ' warninglists.');
+            $message = __('Successfully updated %s warninglists.', $successes);
             if ($fails != 0) {
-                $message . __(' However, could not update ') . $fails . ' warning list.'; // TODO: non-SVO languages need to be considered
+                $message .= __(' However, could not update %s warninglists.', $fails); // TODO: non-SVO languages need to be considered
             }
         }
         if ($this->_isRest()) {
@@ -186,7 +189,6 @@ class WarninglistsController extends AppController
             $this->Warninglist->regenerateWarninglistCaches($warningList['Warninglist']['id']);
         }
         if ($success) {
-            $this->Warninglist->regenerateWarninglistCaches($id);
             return new CakeResponse(array('body'=> json_encode(array('saved' => true, 'success' => $success . __(' warninglist(s) ') . $message)), 'status' => 200, 'type' => 'json')); // TODO: non-SVO lang considerations
         } else {
             return new CakeResponse(array('body'=> json_encode(array('saved' => false, 'errors' => __('Warninglist(s) could not be toggled.'))), 'status' => 200, 'type' => 'json'));
@@ -267,7 +269,6 @@ class WarninglistsController extends AppController
     public function checkValue()
     {
         if ($this->request->is('post')) {
-            $warninglists = $this->Warninglist->getWarninglists(array());
             if (empty($this->request->data)) {
                 throw new NotFoundException(__('No valid data received.'));
             }
@@ -278,13 +279,14 @@ class WarninglistsController extends AppController
             if (array_key_exists('[]', $data)) {
                 $data = $data['[]'];
             }
+
             $hits = array();
+            $warninglists = $this->Warninglist->getEnabled();
             foreach ($data as $dataPoint) {
                 foreach ($warninglists as $warninglist) {
-                    $listValues = $this->Warninglist->getWarninglistEntries($warninglist['Warninglist']['id']);
-                    $listValues = array_combine($listValues, $listValues);
-                    $result = $this->Warninglist->quickCheckValue($listValues, $dataPoint, $warninglist['Warninglist']['type']);
-                    if ($result) {
+                    $values = $this->Warninglist->getFilteredEntries($warninglist);
+                    $result = $this->Warninglist->quickCheckValue($values, $dataPoint, $warninglist['Warninglist']['type']);
+                    if ($result !== false) {
                         $hits[$dataPoint][] = array('id' => $warninglist['Warninglist']['id'], 'name' => $warninglist['Warninglist']['name']);
                     }
                 }

app/Lib/Dashboard/ButtonWidget.php

@@ -0,0 +1,34 @@
+<?php
+
+class ButtonWidget
+{
+    public $title = 'Button Widget';
+    public $render = 'Button';
+    public $width = 3;
+    public $height = 2;
+    public $cacheLifetime = false;
+    public $autoRefreshDelay = false;
+    public $params = array(
+        'url' => 'URL (after base url) to redirect to',
+        'text' => 'Text to display on the button'
+    );
+    public $description = 'Simple button to allow shortcuts';
+    public $placeholder =
+'{
+    "url": "/events/index",
+    "text": "Go to events"
+}';
+
+    public function handler($user, $options = array())
+    {
+        $data = array();
+        if(isset($options['url'])) {
+            $data['url'] = $options['url'];
+        }
+        if(isset($options['text'])) {
+            $data['text'] = $options['text'];
+        }
+
+        return $data;
+    }
+}

app/Lib/Dashboard/MispAdminResourceWidget.php

@@ -16,14 +16,15 @@ class MispAdminResourceWidget
 	{
         $this->Server = ClassRegistry::init('Server');
         $data = array();
-        $redis = $this->Server->setupRedis();
-        if ($redis) {
-            $memory_stats = round($redis->rawCommand('memory', 'stats')[3] / 1024 / 1024) . 'M';
+
+        $redisInfo = $this->Server->redisInfo();
+        if ($redisInfo['connection']) {
+            $memory_stats = round($redisInfo['used_memory'] / 1024 / 1024) . 'M';
             $data[] = array(
                 'title' => __('Current Redis memory usage'),
                 'value' => h($memory_stats)
             );
-            $memory_stats = round($redis->rawCommand('memory', 'stats')[1] / 1024 / 1024) . 'M';
+            $memory_stats = round($redisInfo['used_memory_peak'] / 1024 / 1024) . 'M';
             $data[] = array(
                 'title' => __('Peak Redis memory usage'),
                 'value' => h($memory_stats)

app/Lib/Dashboard/MispAdminSyncTestWidget.php

@@ -15,7 +15,7 @@ class MispAdminSyncTestWidget
 	{
         $this->Server = ClassRegistry::init('Server');
         $servers = $this->Server->find('all', array(
-            'fields' => array('id', 'url', 'name', 'pull', 'push', 'caching_enabled'),
+            'fields' => array('id', 'url', 'name', 'pull', 'push', 'caching_enabled', 'authkey', 'cert_file', 'client_cert_file', 'self_signed'),
             'conditions' => array('OR' => array('pull' => 1, 'push' => 1, 'caching_enabled' => 1)),
             'recursive' => -1
         ));
@@ -25,16 +25,15 @@ class MispAdminSyncTestWidget
         }
         $syncTestErrorCodes = $this->Server->syncTestErrorCodes;
         foreach ($servers as $server) {
-            $result = $this->Server->runConnectionTest($server['Server']['id']);
+            $result = $this->Server->runConnectionTest($server);
             if ($result['status'] === 1) {
                 $message = __('Connected.');
                 $colour = 'green';
-                $flags = json_decode($result['message'], true);
-                if (empty($flags['perm_sync'])) {
+                if (empty($result['info']['perm_sync'])) {
                     $colour = 'orange';
                     $message .= ' ' . __('No sync access.');
                 }
-                if (empty($flags['perm_sighting'])) {
+                if (empty($result['info']['perm_sighting'])) {
                     $colour = 'orange';
                     $message .= ' ' . __('No sighting access.');
                 }

app/Lib/Tools/AttachmentTool.php

@@ -33,7 +33,6 @@ class AttachmentTool
         return $this->_exists(true, $eventId, $attributeId, $path_suffix);
     }
 
-
     /**
      * @param bool $shadow
      * @param int $eventId
@@ -297,6 +296,9 @@ class AttachmentTool
     }
 
     /**
+     * It is not possible to use PHP extensions for compressing. The reason is, that extensions support just AES encrypted
+     * files, but these files are not supported in Windows and in Python. So the only solution is to use 'zip' command.
+     *
      * @param string $originalFilename
      * @param string $content
      * @param string $md5
@@ -304,23 +306,6 @@ class AttachmentTool
      * @throws Exception
      */
     public function encrypt($originalFilename, $content, $md5)
-    {
-        if (method_exists("ZipArchive", "setEncryptionName")) {
-            // When PHP zip extension is installed and supports creating encrypted archives.
-            return $this->encryptByExtension($originalFilename, $content, $md5);
-        } else {
-            return $this->encryptByCommand($originalFilename, $content, $md5);
-        }
-    }
-
-    /**
-     * @param string $originalFilename
-     * @param string $content
-     * @param string $md5
-     * @return string Content of zipped file
-     * @throws Exception
-     */
-    private function encryptByCommand($originalFilename, $content, $md5)
     {
         $tempDir = $this->tempDir();
 
@@ -367,43 +352,6 @@ class AttachmentTool
         }
     }
 
-    /**
-     * @param string $originalFilename
-     * @param string $content
-     * @param string $md5
-     * @return string Content of zipped file
-     * @throws Exception
-     */
-    private function encryptByExtension($originalFilename, $content, $md5)
-    {
-        $zipFilePath = $this->tempFileName();
-
-        $zip = new ZipArchive();
-        $result = $zip->open($zipFilePath, ZipArchive::CREATE);
-        if ($result === true) {
-            $zip->setPassword(self::ZIP_PASSWORD);
-
-            $zip->addFromString($md5, $content);
-            $zip->setEncryptionName($md5, ZipArchive::EM_AES_128);
-
-            $zip->addFromString("$md5.filename.txt", $originalFilename);
-            $zip->setEncryptionName("$md5.filename.txt", ZipArchive::EM_AES_128);
-
-            $zip->close();
-        } else {
-            throw new Exception("Could not create encrypted ZIP file '$zipFilePath'. Error code: $result");
-        }
-
-        $zipFile = new File($zipFilePath);
-        $zipContent = $zipFile->read();
-        if ($zipContent === false) {
-            throw new Exception("Could not read content of newly created ZIP file.");
-        }
-        $zipFile->delete();
-
-        return $zipContent;
-    }
-
     /**
      * @param string $content
      * @param array $hashTypes
@@ -475,7 +423,7 @@ class AttachmentTool
      * Naive way to detect if we're working in S3
      * @return bool
      */
-    private function attachmentDirIsS3()
+    public function attachmentDirIsS3()
     {
         return substr(Configure::read('MISP.attachments_dir'), 0, 2) === "s3";
     }

app/Lib/Tools/ComplexTypeTool.php

@@ -3,7 +3,7 @@ require_once __DIR__ . '/TmpFileTool.php';
 
 class ComplexTypeTool
 {
-    private $__refangRegexTable = array(
+    private static $__refangRegexTable = array(
         array(
             'from' => '/^(hxxp|hxtp|htxp|meow|h\[tt\]p)/i',
             'to' => 'http',
@@ -38,9 +38,9 @@ class ComplexTypeTool
 
     private $__tlds = null;
 
-    public function refangValue($value, $type)
+    public static function refangValue($value, $type)
     {
-        foreach ($this->__refangRegexTable as $regex) {
+        foreach (self::$__refangRegexTable as $regex) {
             if (in_array($type, $regex['types'])) {
                 $value = preg_replace($regex['from'], $regex['to'], $value);
             }
@@ -248,12 +248,23 @@ class ComplexTypeTool
         128 => array('single' => array('sha512'), 'composite' => array('filename|sha512'))
     );
 
-    // algorithms to run through in order
-    private $__checks = array('Hashes', 'Email', 'IP', 'DomainOrFilename', 'SimpleRegex', 'AS', 'BTC');
+    // algorithms to run through in order, without Hashes that are checked separately
+    private $__checks = array('Email', 'IP', 'DomainOrFilename', 'SimpleRegex', 'AS', 'BTC');
 
+    /**
+     * @param string $raw_input Trimmed value
+     * @return array|false
+     */
     private function __resolveType($raw_input)
     {
-        $input = array('raw' => trim($raw_input));
+        $input = array('raw' => $raw_input);
+
+        // Check hashes before refang and port extracting, it is not necessary for hashes. This speedups parsing
+        // freetexts or CSVs with a lot of hashes.
+        $hashes = $this->__checkForHashes($input);
+        if ($hashes) {
+            return $hashes;
+        }
 
         $input = $this->__refangInput($input);
         $input = $this->__extractPort($input);
@@ -320,7 +331,7 @@ class ComplexTypeTool
             if ($this->__checkForBTC($input)) {
                 $types[] = 'btc';
             }
-            return array('types' => $types, 'to_ids' => true, 'default_type' => $hash['single'][0], 'value' => $input['raw']);
+            return array('types' => $types, 'to_ids' => true, 'default_type' => $types[0], 'value' => $input['raw']);
         }
         // ssdeep has a different pattern
         if ($this->__resolveSsdeep($input['raw'])) {
@@ -347,7 +358,7 @@ class ComplexTypeTool
     private function __refangInput($input)
     {
         $input['refanged'] = $input['raw'];
-        foreach ($this->__refangRegexTable as $regex) {
+        foreach (self::$__refangRegexTable as $regex) {
             $input['refanged'] = preg_replace($regex['from'], $regex['to'], $input['refanged']);
         }
         $input['refanged'] = rtrim($input['refanged'], ".");
@@ -365,7 +376,13 @@ class ComplexTypeTool
     {
         // CVE numbers
         if (preg_match("#^cve-[0-9]{4}-[0-9]{4,9}$#i", $input['raw'])) {
-            return array('types' => array('vulnerability'), 'categories' => array('External analysis'), 'to_ids' => false, 'default_type' => 'vulnerability', 'value' => $input['raw']);
+            return [
+                'types' => ['vulnerability'],
+                'categories' => ['External analysis'],
+                'to_ids' => false,
+                'default_type' => 'vulnerability',
+                'value' => strtoupper($input['raw']), // 'CVE' must be uppercase
+            ];
         }
         // Phone numbers - for automatic recognition, needs to start with + or include dashes
         if ($input['raw'][0] === '+' || strpos($input['raw'], '-')) {
@@ -496,7 +513,7 @@ class ComplexTypeTool
     private function __resolveHash($value)
     {
         $strlen = strlen($value);
-        if (isset($this->__hexHashTypes[$strlen]) && preg_match("#[0-9a-f]{" . $strlen . "}$#i", $value)) {
+        if (isset($this->__hexHashTypes[$strlen]) && ctype_xdigit($value)) {
             return $this->__hexHashTypes[$strlen];
         }
         return false;

app/Lib/Tools/CryptGpgExtended.php

@@ -1,6 +1,16 @@
 <?php
 class CryptGpgExtended extends Crypt_GPG
 {
+    public function __construct(array $options = array())
+    {
+        if (!method_exists($this, '_prepareInput')) {
+            $reflector = new \ReflectionClass('Crypt_GPG');
+            $classPath = $reflector->getFileName();
+            throw new Exception("Crypt_GPG class from '$classPath' is too old, at least version 1.6.1 is required.");
+        }
+        parent::__construct($options);
+    }
+
     /**
      * Export the smallest public key possible from the keyring.
      *
@@ -61,4 +71,91 @@ class CryptGpgExtended extends Crypt_GPG
 
         return $keyData;
     }
+
+    /**
+     * Return key info without importing it.
+     *
+     * @param string $key
+     * @return Crypt_GPG_Key[]
+     * @throws Crypt_GPG_Exception
+     * @throws Crypt_GPG_InvalidOperationException
+     */
+    public function keyInfo($key)
+    {
+        $input = $this->_prepareInput($key, false, false);
+
+        $output = '';
+        $this->engine->reset();
+        $this->engine->setInput($input);
+        $this->engine->setOutput($output);
+        $this->engine->setOperation('--import', ['--import-options', 'show-only', '--with-colons']);
+        $this->engine->run();
+
+        $keys   = array();
+        $key    = null; // current key
+        $subKey = null; // current sub-key
+
+        foreach (explode(PHP_EOL, $output) as $line) {
+            $lineExp = explode(':', $line);
+
+            if ($lineExp[0] === 'pub') {
+                // new primary key means last key should be added to the array
+                if ($key !== null) {
+                    $keys[] = $key;
+                }
+
+                $key = new Crypt_GPG_Key();
+
+                $subKey = Crypt_GPG_SubKey::parse($line);
+                $key->addSubKey($subKey);
+
+            } elseif ($lineExp[0] === 'sub') {
+                $subKey = Crypt_GPG_SubKey::parse($line);
+                $key->addSubKey($subKey);
+
+            } elseif ($lineExp[0] === 'fpr') {
+                $fingerprint = $lineExp[9];
+
+                // set current sub-key fingerprint
+                $subKey->setFingerprint($fingerprint);
+
+            } elseif ($lineExp[0] === 'uid') {
+                $string = stripcslashes($lineExp[9]); // as per documentation
+                $userId = new Crypt_GPG_UserId($string);
+
+                if ($lineExp[1] === 'r') {
+                    $userId->setRevoked(true);
+                }
+
+                $key->addUserId($userId);
+            }
+        }
+
+        // add last key
+        if ($key !== null) {
+            $keys[] = $key;
+        }
+
+        return $keys;
+    }
+
+    /**
+     * @param string $key
+     * @return string
+     * @throws Crypt_GPG_Exception
+     * @throws Crypt_GPG_InvalidOperationException
+     */
+    public function enarmor($key)
+    {
+        $input = $this->_prepareInput($key, false, false);
+
+        $armored = '';
+        $this->engine->reset();
+        $this->engine->setInput($input);
+        $this->engine->setOutput($armored);
+        $this->engine->setOperation('--enarmor');
+        $this->engine->run();
+
+        return $armored;
+    }
 }

app/Lib/Tools/DistributionGraphTool.php

@@ -1,223 +1,221 @@
 <?php
-    class DistributionGraphTool
+class DistributionGraphTool
+{
+    private $__user = false;
+    private $__json = array();
+    /** @var Event */
+    private $__eventModel;
+    /** @var Organisation */
+    private $__organisationModel;
+
+    public function construct(Event $eventModel, array $servers, array $user, $extended_view=0)
     {
-        private $__user = false;
-        private $__json = array();
-        private $__eventModel = false;
+        $this->__eventModel = $eventModel;
+        $this->__serverList = $servers;
+        $this->__organisationModel = $eventModel->Orgc;
+        $this->__user = $user;
+        $this->__json = array();
+        $this->__extended_view = $extended_view;
 
-        public function construct($eventModel, $servers, $user, $extended_view=0)
-        {
-            $this->__eventModel = $eventModel;
-            $this->__serverList = $servers;
-            $this->__organisationModel = $eventModel->Orgc;
-            $this->__user = $user;
-            $this->__json = array();
-            $this->__extended_view = $extended_view;
+        // construct distribution info
+        $this->__json['distributionInfo'] = array();
+        $sgs = $this->__eventModel->SharingGroup->fetchAllAuthorised($this->__user, 'simplified', 1);
+        $this->__json['allSharingGroup'] = h($sgs);
+        $distributionLevels = $this->__eventModel->distributionLevels;
+        foreach ($distributionLevels as $key => $value) {
+            $this->__json['distributionInfo'][$key] = array('key' => h($value), 'desc' => h($this->__eventModel->distributionDescriptions[$key]['formdesc']), 'value' => h($key));
+        }
+        $this->__json['distributionInfo'][5] = ""; // inherit event. Will be deleted afterward
 
-            // construct distribution info
-            $this->__json['distributionInfo'] = array();
-            $sgs = $this->__eventModel->SharingGroup->fetchAllAuthorised($this->__user, 'simplified', 1);
-            $this->__json['allSharingGroup'] = h($sgs);
-            $distributionLevels = $this->__eventModel->distributionLevels;
-            foreach ($distributionLevels as $key => $value) {
-                $this->__json['distributionInfo'][$key] = array('key' => h($value), 'desc' => h($this->__eventModel->distributionDescriptions[$key]['formdesc']), 'value' => h($key));
+        return true;
+    }
+
+    private function __fetchAndAddDistributionInfo($elem)
+    {
+        $distributionLevel = $elem['distribution'];
+
+        if ($distributionLevel == 5) { // inherit -> convert it to the event distribution level
+            $elem['distribution'] = $this->__eventDistribution;
+            $this->__fetchAndAddDistributionInfo($elem);
+        } elseif ($distributionLevel == 4) { // sharing group
+            if (isset($elem['SharingGroup'])) {
+                $sg_name = $elem['SharingGroup']['name'];
+                $this->__addAdditionalDistributionInfo($distributionLevel, $sg_name);
+            } elseif ($this->__eventDistribution == 4) { // event is distributed for sg
+                $sg_name = $this->__eventSharingGroupName;
+                $this->__addAdditionalDistributionInfo($distributionLevel, $sg_name);
             }
-            $this->__json['distributionInfo'][5] = ""; // inherit event. Will be deleted afterward
+        } else {
+            return false;
+        }
+        return true;
+    }
 
-            return true;
+    private function __addAdditionalDistributionInfo($distributionLevel, $data)
+    {
+        if (empty($this->__json['additionalDistributionInfo'][$distributionLevel])) {
+            $this->__json['additionalDistributionInfo'][$distributionLevel] = array();
+        }
+        $this->__json['additionalDistributionInfo'][$distributionLevel][h($data)] = 0; // set-alike
+        if ($distributionLevel == 4) {
+            if (!isset($this->__json['sharingGroupRepartition'][h($data)])) {
+                $this->__json['sharingGroupRepartition'][h($data)] = 0;
+            }
+            $this->__json['sharingGroupRepartition'][h($data)]++;
+        }
+    }
+
+    private function __addOtherDistributionInfo()
+    {
+        // all comm
+        $this->__addAdditionalDistributionInfo(3, "This community"); // add current community
+        $this->__addAdditionalDistributionInfo(3, "All other communities"); // add current community
+
+        // connected
+        $servers = $this->__serverList;
+        $this->__addAdditionalDistributionInfo(2, "This community"); // add current community
+        foreach ($servers as $server) {
+            $this->__addAdditionalDistributionInfo(2, $server);
         }
 
-        private function __extract_sharing_groups_names($sharingArray)
-        {
-            return $sharingArray['name'];
-        }
-
-        private function __fetchAndAddDistributionInfo($elem)
-        {
-            $distributionLevel = $elem['distribution'];
-
-            if ($distributionLevel == 5) { // inherit -> convert it to the event distribution level
-                $elem['distribution'] = $this->__eventDistribution;
-                $this->__fetchAndAddDistributionInfo($elem);
-            } elseif ($distributionLevel == 4) { // sharing group
-                if (isset($elem['SharingGroup'])) {
-                    $sg_name = $this->__extract_sharing_groups_names($elem['SharingGroup']);
-                    $this->__addAdditionalDistributionInfo($distributionLevel, $sg_name);
-                } elseif ($this->__eventDistribution == 4) { // event is distributed for sg
-                    $sg_name = $this->__eventSharingGroupName;
-                    $this->__addAdditionalDistributionInfo($distributionLevel, $sg_name);
-                }
-            } else {
-                return false;
-            }
-            return true;
-        }
-
-        private function __addAdditionalDistributionInfo($distributionLevel, $data)
-        {
-            if (empty($this->__json['additionalDistributionInfo'][$distributionLevel])) {
-                $this->__json['additionalDistributionInfo'][$distributionLevel] = array();
-            }
-            $this->__json['additionalDistributionInfo'][$distributionLevel][h($data)] = 0; // set-alike
-            if ($distributionLevel == 4) {
-                if (!isset($this->__json['sharingGroupRepartition'][h($data)])) {
-                    $this->__json['sharingGroupRepartition'][h($data)] = 0;
-                }
-                $this->__json['sharingGroupRepartition'][h($data)]++;
+        // community
+        $orgs = $this->__organisationModel->find('list', array(
+            'fields' => array('name'),
+            'conditions' => array('local' => true)
+        ));
+        $thisOrg = $this->__user['Organisation']['name'];
+        $this->__addAdditionalDistributionInfo(1, $thisOrg); // add current community
+        foreach ($orgs as $org) {
+            if ($thisOrg != $org) {
+                $this->__addAdditionalDistributionInfo(1, $org);
             }
         }
 
-        private function __addOtherDistributionInfo()
-        {
-            // all comm
-            $this->__addAdditionalDistributionInfo(3, "This community"); // add current community
-            $this->__addAdditionalDistributionInfo(3, "All other communities"); // add current community
-
-            // connected
-            $servers = $this->__serverList;
-            $this->__addAdditionalDistributionInfo(2, "This community"); // add current community
-            foreach ($servers as $server) {
-                $this->__addAdditionalDistributionInfo(2, $server);
-            }
-
-            // community
-            $orgs = $this->__organisationModel->find('list', array(
-                'fields' => array('name'),
-                'conditions' => array('local' => true)
-            ));
-            $thisOrg = $this->__user['Organisation']['name'];
-            $this->__addAdditionalDistributionInfo(1, $thisOrg); // add current community
-            foreach ($orgs as $org) {
-                if ($thisOrg != $org) {
-                    $this->__addAdditionalDistributionInfo(1, $org);
-                }
-            }
-
-            // org only
-            $thisOrg = $this->__user['Organisation']['name'];
-            $this->__addAdditionalDistributionInfo(0, $thisOrg); // add current community
-        }
-
-        private function __get_event($id)
-        {
-            $fullevent = $this->__eventModel->fetchEvent($this->__user, array('eventid' => $id, 'flatten' => 0, 'includeTagRelations' => 1, 'extended' => $this->__extended_view));
-            $event = array();
-            if (empty($fullevent)) {
-                return $event;
-            }
-
-            $fullevent = $fullevent[0];
-            if (!empty($fullevent['Object'])) {
-                $event['Object'] = $fullevent['Object'];
-            } else {
-                $event['Object'] = array();
-            }
-
-            if (!empty($fullevent['Attribute'])) {
-                $event['Attribute'] = $fullevent['Attribute'];
-            } else {
-                $event['Attribute'] = array();
-            }
-            $event['distribution'] = $fullevent['Event']['distribution'];
-
-            if (isset($fullevent['SharingGroup'])) {
-                $event['SharingGroupName'] = $fullevent['SharingGroup']['name'];
-            } else {
-                $event['SharingGroupName'] = "?";
-            }
+        // org only
+        $thisOrg = $this->__user['Organisation']['name'];
+        $this->__addAdditionalDistributionInfo(0, $thisOrg); // add current community
+    }
 
+    /**
+     * Fetch event containing just 'Attribute', 'Object', 'SharingGroup' and 'distribution'
+     * @param int $id
+     * @return array
+     * @throws Exception
+     */
+    private function __get_event($id)
+    {
+        $fullevent = $this->__eventModel->fetchEvent($this->__user, array(
+            'eventid' => $id,
+            'flatten' => 0,
+            'noShadowAttributes' => true,
+            'noEventReports' => true,
+            'noSightings' => true,
+            'extended' => $this->__extended_view,
+        ));
+        $event = array();
+        if (empty($fullevent)) {
             return $event;
         }
 
-        public function get_distributions_graph($id)
-        {
-            $this->__json['event'] = $this->init_array_distri();
-            $this->__json['attribute'] = $this->init_array_distri();
-            $this->__json['object'] = $this->init_array_distri();
-            $this->__json['obj_attr'] = $this->init_array_distri();
-            $this->__json['additionalDistributionInfo'] = $this->init_array_distri(array());
-            $this->__json['sharingGroupRepartition'] = array();
+        $fullevent = $fullevent[0];
+        if (isset($fullevent['Object'])) {
+            $event['Object'] = $fullevent['Object'];
+        } else {
+            $event['Object'] = array();
+        }
 
-            $this->__addOtherDistributionInfo();
+        if (isset($fullevent['Attribute'])) {
+            $event['Attribute'] = $fullevent['Attribute'];
+        } else {
+            $event['Attribute'] = array();
+        }
+        $event['distribution'] = $fullevent['Event']['distribution'];
 
-            // transform set into array
-            foreach (array_keys($this->__json['additionalDistributionInfo']) as $d) {
-                $this->__json['additionalDistributionInfo'][$d] = array_keys($this->__json['additionalDistributionInfo'][$d]);
-            }
+        if (isset($fullevent['SharingGroup'])) {
+            $event['SharingGroupName'] = $fullevent['SharingGroup']['name'];
+        } else {
+            $event['SharingGroupName'] = "?";
+        }
 
-            if ($id === -1) {
-                return $this->__json;
-            }
-            $event = $this->__get_event($id);
-            $eventDist = $event['distribution'];
-            $eventSGName = $event['SharingGroupName'];
-            $this->__eventDistribution = $eventDist;
-            $this->__eventSharingGroupName = $eventSGName;
+        return $event;
+    }
 
-            if (empty($event)) {
-                return $this->__json;
-            }
+    public function get_distributions_graph($id)
+    {
+        $this->__json['event'] = $this->init_array_distri();
+        $this->__json['attribute'] = $this->init_array_distri();
+        $this->__json['object'] = $this->init_array_distri();
+        $this->__json['obj_attr'] = $this->init_array_distri();
+        $this->__json['additionalDistributionInfo'] = $this->init_array_distri(array());
+        $this->__json['sharingGroupRepartition'] = array();
 
-            if (!empty($event['Object'])) {
-                $object = $event['Object'];
-            } else {
-                $object = array();
-            }
+        $this->__addOtherDistributionInfo();
 
-            if (!empty($event['Attribute'])) {
-                $attribute = $event['Attribute'];
-            } else {
-                $attribute = array();
-            }
-
-            // extract distribution
-            foreach ($attribute as $attr) {
-                $distri = $attr['distribution'];
-                $this->__json['event'][$distri] += 1;
-                $this->__json['attribute'][$distri] += 1;
-                $this->__fetchAndAddDistributionInfo($attr);
-            }
-
-            foreach ($object as $obj) {
-                $distri = $obj['distribution'];
-                $this->__json['event'][$distri] += 1;
-                $this->__json['object'][$distri] += 1;
-                $this->__fetchAndAddDistributionInfo($obj);
-
-                $added_value = array();
-                if (!empty($obj['Attribute'])) {
-                    foreach ($obj['Attribute'] as $objAttr) {
-                        $distri = $objAttr['distribution'];
-                        $this->__json['event'][$distri] += 1;
-                        $this->__json['obj_attr'][$distri] += 1;
-                        $this->__fetchAndAddDistributionInfo($objAttr);
-                    }
-                }
-            }
-            // distribution 5 is inherit event, apply this fact on values
-            $this->__json['event'][$eventDist] += $this->__json['event'][5];
-            unset($this->__json['event'][5]);
-            $this->__json['attribute'][$eventDist] += $this->__json['attribute'][5];
-            unset($this->__json['attribute'][5]);
-            $this->__json['object'][$eventDist] += $this->__json['object'][5];
-            unset($this->__json['object'][5]);
-            $this->__json['obj_attr'][$eventDist] += $this->__json['obj_attr'][5];
-            unset($this->__json['obj_attr'][5]);
-
-            unset($this->__json['distributionInfo'][5]); // inherit event.
-
-            // transform set into array for SG (others are already done)
-            $this->__json['additionalDistributionInfo'][4] = array_keys($this->__json['additionalDistributionInfo'][4]);
+        // transform set into array
+        foreach (array_keys($this->__json['additionalDistributionInfo']) as $d) {
+            $this->__json['additionalDistributionInfo'][$d] = array_keys($this->__json['additionalDistributionInfo'][$d]);
+        }
 
+        if ($id === -1) {
+            return $this->__json;
+        }
+        $event = $this->__get_event($id);
+        if (empty($event)) {
             return $this->__json;
         }
 
-        public function init_array_distri($default=0)
-        {
-            $ret = array();
-            foreach ($this->__json['distributionInfo'] as $d => $v) {
-                $ret[h($d)] = $default;
-            }
-            return $ret;
+        $eventDist = $event['distribution'];
+        $eventSGName = $event['SharingGroupName'];
+        $this->__eventDistribution = $eventDist;
+        $this->__eventSharingGroupName = $eventSGName;
+
+        // extract distribution
+        foreach ($event['Attribute'] as $attr) {
+            $distri = $attr['distribution'];
+            $this->__json['event'][$distri] += 1;
+            $this->__json['attribute'][$distri] += 1;
+            $this->__fetchAndAddDistributionInfo($attr);
         }
+
+        foreach ($event['Object'] as $obj) {
+            $distri = $obj['distribution'];
+            $this->__json['event'][$distri] += 1;
+            $this->__json['object'][$distri] += 1;
+            $this->__fetchAndAddDistributionInfo($obj);
+
+            if (!empty($obj['Attribute'])) {
+                foreach ($obj['Attribute'] as $objAttr) {
+                    $distri = $objAttr['distribution'];
+                    $this->__json['event'][$distri] += 1;
+                    $this->__json['obj_attr'][$distri] += 1;
+                    $this->__fetchAndAddDistributionInfo($objAttr);
+                }
+            }
+        }
+        // distribution 5 is inherit event, apply this fact on values
+        $this->__json['event'][$eventDist] += $this->__json['event'][5];
+        unset($this->__json['event'][5]);
+        $this->__json['attribute'][$eventDist] += $this->__json['attribute'][5];
+        unset($this->__json['attribute'][5]);
+        $this->__json['object'][$eventDist] += $this->__json['object'][5];
+        unset($this->__json['object'][5]);
+        $this->__json['obj_attr'][$eventDist] += $this->__json['obj_attr'][5];
+        unset($this->__json['obj_attr'][5]);
+
+        unset($this->__json['distributionInfo'][5]); // inherit event.
+
+        // transform set into array for SG (others are already done)
+        $this->__json['additionalDistributionInfo'][4] = array_keys($this->__json['additionalDistributionInfo'][4]);
+
+        return $this->__json;
     }
+
+    public function init_array_distri($default=0)
+    {
+        $ret = array();
+        foreach ($this->__json['distributionInfo'] as $d => $v) {
+            $ret[h($d)] = $default;
+        }
+        return $ret;
+    }
+}

app/Lib/Tools/GpgTool.php

@@ -5,7 +5,7 @@ class GpgTool
      * @return CryptGpgExtended
      * @throws Exception
      */
-    public function initializeGpg()
+    public static function initializeGpg()
     {
         if (!class_exists('Crypt_GPG')) {
             // 'Crypt_GPG' class cannot be autoloaded, try to require from include_path.
@@ -31,6 +31,14 @@ class GpgTool
         return new CryptGpgExtended($options);
     }
 
+    /** @var CryptGpgExtended */
+    private $gpg;
+
+    public function __construct($gpg)
+    {
+        $this->gpg = $gpg;
+    }
+
     /**
      * @param string $search
      * @return array
@@ -65,9 +73,41 @@ class GpgTool
 
         $key = $response->body;
 
+        if ($this->gpg) {
+            $fetchedFingerprint = $this->validateGpgKey($key);
+            if (strtolower($fingerprint) !== strtolower($fetchedFingerprint)) {
+                throw new Exception("Requested fingerprint do not match with fetched key fingerprint ($fingerprint != $fetchedFingerprint)");
+            }
+        }
+
         return $key;
     }
 
+    /**
+     * Validates PGP key
+     * @param string $keyData
+     * @return string Primary key fingerprint
+     * @throws Exception
+     */
+    public function validateGpgKey($keyData)
+    {
+        if (!$this->gpg instanceof CryptGpgExtended) {
+            throw new InvalidArgumentException("Valid CryptGpgExtended instance required.");
+        }
+        $fetchedKeyInfo = $this->gpg->keyInfo($keyData);
+        if (empty($fetchedKeyInfo)) {
+            throw new Exception("No key found");
+        }
+        if (count($fetchedKeyInfo) !== 1) {
+            throw new Exception("Multiple keys found");
+        }
+        $primaryKey = $fetchedKeyInfo[0]->getPrimaryKey();
+        if (empty($primaryKey)) {
+            throw new Exception("No primary key found");
+        }
+        return $primaryKey->getFingerprint();
+    }
+
     /**
      * @param string $body
      * @return array
@@ -107,6 +147,91 @@ class GpgTool
         return $final;
     }
 
+    /**
+     * @see https://tools.ietf.org/html/draft-koch-openpgp-webkey-service-10
+     * @param string $email
+     * @return string
+     * @throws Exception
+     */
+    public function wkd($email)
+    {
+        if (!$this->gpg instanceof CryptGpgExtended) {
+            throw new InvalidArgumentException("Valid CryptGpgExtended instance required.");
+        }
+
+        $parts = explode('@', $email);
+        if (count($parts) !== 2) {
+            throw new InvalidArgumentException("Invalid e-mail address provided.");
+        }
+
+        list($localPart, $domain) = $parts;
+        $localPart = strtolower($localPart);
+        $localPartHash = $this->zbase32(sha1($localPart, true));
+
+        $advancedUrl = "https://openpgpkey.$domain/.well-known/openpgpkey/" . strtolower($domain) . "/hu/$localPartHash";
+        try {
+            $response = $this->keyServerLookup($advancedUrl);
+            return $this->processWkdResponse($response);
+        } catch (Exception $e) {
+            // pass, continue to direct method
+        }
+
+        $directUrl = "https://$domain/.well-known/openpgpkey/hu/$localPartHash";
+        $response = $this->keyServerLookup($directUrl);
+        return $this->processWkdResponse($response);
+    }
+
+    /**
+     * @param HttpSocketResponse $response
+     * @return string
+     * @throws Crypt_GPG_Exception
+     * @throws Crypt_GPG_InvalidOperationException
+     */
+    private function processWkdResponse(HttpSocketResponse $response)
+    {
+        if ($response->code == 404) {
+            throw new NotFoundException("Key not found");
+        } else if (!$response->isOk()) {
+            throw new Exception("Fetching the WKD failed with HTTP error {$response->code}: {$response->reasonPhrase}");
+        }
+
+        return $this->gpg->enarmor($response->body());
+    }
+
+    /**
+     * Converts data to zbase32 string.
+     *
+     * @see http://philzimmermann.com/docs/human-oriented-base-32-encoding.txt
+     * @param string $data
+     * @return string
+     */
+    private function zbase32($data)
+    {
+        $chars = 'ybndrfg8ejkmcpqxot1uwisza345h769'; // lower-case
+        $res = '';
+        $remainder = 0;
+        $remainderSize = 0;
+
+        for ($i = 0; $i < strlen($data); $i++) {
+            $b = ord($data[$i]);
+            $remainder = ($remainder << 8) | $b;
+            $remainderSize += 8;
+            while ($remainderSize > 4) {
+                $remainderSize -= 5;
+                $c = $remainder & (31 << $remainderSize);
+                $c >>= $remainderSize;
+                $res .= $chars[$c];
+            }
+        }
+        if ($remainderSize > 0) {
+            // remainderSize < 5:
+            $remainder <<= (5 - $remainderSize);
+            $c = $remainder & 31;
+            $res .= $chars[$c];
+        }
+        return $res;
+    }
+
     /**
      * @param string $uri
      * @return HttpSocketResponse

app/Lib/Tools/JSONConverterTool.php

@@ -53,23 +53,23 @@ class JSONConverterTool
 
     public function convert($event, $isSiteAdmin=false, $raw = false)
     {
-        $toRearrange = array('Org', 'Orgc', 'SharingGroup', 'Attribute', 'ShadowAttribute', 'RelatedAttribute', 'RelatedEvent', 'Galaxy', 'Object');
+        $toRearrange = array('Org', 'Orgc', 'SharingGroup', 'Attribute', 'ShadowAttribute', 'RelatedAttribute', 'RelatedEvent', 'Galaxy', 'Object', 'EventReport');
         foreach ($toRearrange as $object) {
             if (isset($event[$object])) {
                 $event['Event'][$object] = $event[$object];
                 unset($event[$object]);
             }
-            if ($object == 'SharingGroup' && isset($event['Event']['SharingGroup']) && empty($event['Event']['SharingGroup'])) {
-                unset($event['Event']['SharingGroup']);
-            }
-            if ($object == 'Galaxy') {
-                if (!empty($event['Event']['Galaxy'])) {
-                    foreach ($event['Event']['Galaxy'] as $k => $galaxy) {
-                        foreach ($galaxy['GalaxyCluster'] as $k2 => $cluster) {
-                            if (empty($cluster['meta'])) {
-                                $event['Event']['Galaxy'][$k]['GalaxyCluster'][$k2]['meta'] = new stdclass();
-                            }
-                        }
+        }
+
+        if (isset($event['Event']['SharingGroup']) && empty($event['Event']['SharingGroup'])) {
+            unset($event['Event']['SharingGroup']);
+        }
+
+        if (!empty($event['Event']['Galaxy'])) {
+            foreach ($event['Event']['Galaxy'] as $k => $galaxy) {
+                foreach ($galaxy['GalaxyCluster'] as $k2 => $cluster) {
+                    if (empty($cluster['meta'])) {
+                        $event['Event']['Galaxy'][$k]['GalaxyCluster'][$k2]['meta'] = new stdclass();
                     }
                 }
             }
@@ -82,9 +82,7 @@ class JSONConverterTool
             }
         }
 
-        //
         // cleanup the array from things we do not want to expose
-        //
         $tempSightings = array();
         if (!empty($event['Sighting'])) {
             foreach ($event['Sighting'] as $sighting) {
@@ -99,10 +97,7 @@ class JSONConverterTool
         if (isset($event['Event']['Object'])) {
             $event['Event']['Object'] = $this->__cleanObjects($event['Event']['Object'], $tempSightings);
         }
-        if (!empty($event['Sighting'])) {
-            unset($event['Sighting']);
-        }
-
+        unset($tempSightings);
         unset($event['Event']['RelatedAttribute']);
         if (isset($event['Event']['RelatedEvent'])) {
             foreach ($event['Event']['RelatedEvent'] as $key => $value) {
@@ -128,21 +123,14 @@ class JSONConverterTool
             }
             unset($attributes[$key]['value1']);
             unset($attributes[$key]['value2']);
-            unset($attributes[$key]['category_order']);
-            if (isset($event['RelatedAttribute'][$attribute['id']])) {
-                $attributes[$key]['RelatedAttribute'] = $event['Event']['RelatedAttribute'][$attribute['id']];
-                foreach ($attributes[$key]['RelatedAttribute'] as &$ra) {
-                    $ra = array('Attribute' => $ra);
-                }
-            }
             if (isset($attributes[$key]['AttributeTag'])) {
-                foreach ($attributes[$key]['AttributeTag'] as $atk => $tag) {
+                foreach ($attribute['AttributeTag'] as $atk => $tag) {
                     unset($tag['Tag']['org_id']);
                     $attributes[$key]['Tag'][$atk] = $tag['Tag'];
                 }
                 unset($attributes[$key]['AttributeTag']);
             }
-            if (!empty($tempSightings[$attribute['id']])) {
+            if (isset($tempSightings[$attribute['id']])) {
                 $attributes[$key]['Sighting'] = $tempSightings[$attribute['id']];
             }
         }

app/Lib/Tools/SendEmail.php

@@ -279,7 +279,7 @@ class SendEmail
      */
     public function sendExternal(array $params)
     {
-        foreach (array('body', 'reply-to', 'to', 'subject', 'text') as $requiredParam) {
+        foreach (array('body', 'reply-to', 'to', 'subject') as $requiredParam) {
             if (!isset($params[$requiredParam])) {
                 throw new InvalidArgumentException("Param '$requiredParam' is required, but not provided.");
             }
@@ -396,7 +396,7 @@ class SendEmail
             try {
                 $gnupgEmail = Configure::read('GnuPG.email');
                 if (empty($gnupgEmail)) {
-                    throw new Exception("Email signing is enabled but variable 'GnuPG.email' is not set.");
+                    throw new Exception("GPG email signing is enabled but variable 'GnuPG.email' is not set.");
                 }
 
                 $this->gpg->addSignKey($gnupgEmail, Configure::read('GnuPG.password'));
@@ -406,7 +406,7 @@ class SendEmail
 
                 $signed = true;
             } catch (Exception $e) {
-                throw new SendEmailException("The message could not be signed.", 0, $e);
+                throw new SendEmailException("The message could not be signed by GPG.", 0, $e);
             }
         }
 
@@ -419,11 +419,11 @@ class SendEmail
             try {
                 $fingerprint = $this->importAndValidateGpgPublicKey($user['User']['gpgkey']);
             } catch (Crypt_GPG_NoDataException $e) {
-                throw new SendEmailException("The message could not be encrypted because the provided key is invalid.", 0, $e);
+                throw new SendEmailException("The message could not be encrypted because the provided GPG key is invalid.", 0, $e);
             }
 
             if (!$fingerprint) {
-                throw new SendEmailException("The message could not be encrypted because the provided key is either expired or cannot be used for encryption.");
+                throw new SendEmailException("The message could not be encrypted because the provided GPG key is either expired or cannot be used for encryption.");
             }
 
             try {
@@ -441,7 +441,7 @@ class SendEmail
 
                 $encrypted = true;
             } catch (Exception $e) {
-                throw new SendEmailException('The message could not be encrypted.', 0, $e);
+                throw new SendEmailException('The message could not be encrypted by GPG.', 0, $e);
             }
         }
 
@@ -472,24 +472,23 @@ class SendEmail
             // Try to encrypt empty message
             $this->encryptTextBySmime($certificate, '');
         } catch (SendEmailException $e) {
-            throw new Exception('This certificate cannot be used to encrypt email.', 0, $e);
+            throw new Exception('This S/MIME certificate cannot be used to encrypt email.', 0, $e);
         }
 
         $parsed = openssl_x509_parse($certificate);
 
         if (!$parsed) {
-            throw new Exception('Could not parse certificate');
+            throw new Exception('Could not parse S/MIME certificate');
         }
 
-        // Purpose '5' should be 'smimeencrypt'
-        if (!($parsed['purposes'][5][0] === 1 && $parsed['purposes'][5][2] === 'smimeencrypt')) {
-            throw new Exception('This certificate cannot be used to encrypt email.');
+        if ($parsed['purposes'][X509_PURPOSE_SMIME_ENCRYPT][0] !== true) {
+            throw new Exception('This S/MIME certificate cannot be used to encrypt email.');
         }
 
         $now = new DateTime();
         $validToTime = new DateTime("@{$parsed['validTo_time_t']}");
         if ($validToTime <= $now) {
-            throw new Exception('This certificate is expired.');
+            throw new Exception('This S/MIME certificate expired at ' . $validToTime->format('c'));
         }
 
         return true;
@@ -775,6 +774,7 @@ class SendEmail
      * @param string $content
      * @return File[]
      * @throws SendEmailException
+     * @throws MethodNotAllowedException
      */
     private function createInputOutputFiles($content)
     {

app/Lib/Tools/SyncTool.php

@@ -60,4 +60,115 @@ class SyncTool
         }
         return $HttpSocket;
     }
+
+    /**
+     * @param array $server
+     * @return array|void
+     * @throws Exception
+     */
+    public static function getServerClientCertificateInfo(array $server)
+    {
+        if (!$server['Server']['client_cert_file']) {
+            return;
+        }
+
+        $clientCertificate = new File(APP . "files" . DS . "certs" . DS . $server['Server']['id'] . '_client.pem');
+        if (!$clientCertificate->exists()) {
+            throw new Exception("Certificate file '{$clientCertificate->pwd()}' doesn't exists.");
+        }
+
+        $certificateContent = $clientCertificate->read();
+        if ($certificateContent === false) {
+            throw new Exception("Could not read '{$clientCertificate->pwd()}' file with client certificate.");
+        }
+
+        return self::getClientCertificateInfo($certificateContent);
+    }
+
+    /**
+     * @param string $certificateContent PEM encoded certificate and private key.
+     * @return array
+     * @throws Exception
+     */
+    private static function getClientCertificateInfo($certificateContent)
+    {
+        $certificate = openssl_x509_read($certificateContent);
+        if (!$certificate) {
+            throw new Exception("Could't parse certificate: " . openssl_error_string());
+        }
+        $privateKey = openssl_pkey_get_private($certificateContent);
+        if (!$privateKey) {
+            throw new Exception("Could't get private key from certificate: " . openssl_error_string());
+        }
+        $verify = openssl_x509_check_private_key($certificate, $privateKey);
+        if (!$verify) {
+            throw new Exception('Public and private key do not match.');
+        }
+        return self::parseCertificate($certificate);
+    }
+
+    /**
+     * @param mixed $certificate
+     * @return array
+     * @throws Exception
+     */
+    private static function parseCertificate($certificate)
+    {
+        $parsed = openssl_x509_parse($certificate);
+        if (!$parsed) {
+            throw new Exception("Could't get parse X.509 certificate: " . openssl_error_string());
+        }
+        $currentTime = new DateTime();
+        $output = [
+            'serial_number' => $parsed['serialNumberHex'],
+            'signature_type' => $parsed['signatureTypeSN'],
+            'valid_from' => isset($parsed['validFrom_time_t']) ? new DateTime("@{$parsed['validFrom_time_t']}") : null,
+            'valid_to' => isset($parsed['validTo_time_t']) ? new DateTime("@{$parsed['validTo_time_t']}") : null,
+            'public_key_size' => null,
+            'public_key_type' => null,
+            'public_key_size_ok' => null,
+        ];
+
+        $output['valid_from_ok'] = $output['valid_from'] ? ($output['valid_from'] <= $currentTime) : null;
+        $output['valid_to_ok'] = $output['valid_to'] ? ($output['valid_to'] >= $currentTime) : null;
+
+        $subject = [];
+        foreach ($parsed['subject'] as $type => $value) {
+            $subject[] = "$type=$value";
+        }
+        $output['subject'] = implode(', ', $subject);
+
+        $issuer = [];
+        foreach ($parsed['issuer'] as $type => $value) {
+            $issuer[] = "$type=$value";
+        }
+        $output['issuer'] = implode(', ', $issuer);
+
+        $publicKey = openssl_pkey_get_public($certificate);
+        if ($publicKey) {
+            $publicKeyDetails = openssl_pkey_get_details($publicKey);
+            if ($publicKeyDetails) {
+                $output['public_key_size'] = $publicKeyDetails['bits'];
+                switch ($publicKeyDetails['type']) {
+                    case OPENSSL_KEYTYPE_RSA:
+                        $output['public_key_type'] = 'RSA';
+                        $output['public_key_size_ok'] = $output['public_key_size'] >= 2048;
+                        break;
+                    case OPENSSL_KEYTYPE_DSA:
+                        $output['public_key_type'] = 'DSA';
+                        $output['public_key_size_ok'] = $output['public_key_size'] >= 2048;
+                        break;
+                    case OPENSSL_KEYTYPE_DH:
+                        $output['public_key_type'] = 'DH';
+                        break;
+                    case OPENSSL_KEYTYPE_EC:
+                        $output['public_key_type'] = "EC ({$publicKeyDetails['ec']['curve_name']})";
+                        $output['public_key_size_ok'] = $output['public_key_size'] >= 224;
+                        break;
+                }
+            }
+        }
+
+        return $output;
+    }
 }

app/Lib/Tools/TmpFileTool.php

@@ -1,11 +1,12 @@
 <?php
 class TmpFileTool
 {
-    /**
-     * @var resource
-     */
+    /** @var resource */
     private $tmpfile;
 
+    /** @var string */
+    private $separator;
+
     /**
      * @param int $maxInMemory How many bytes should keep in memory before creating file on disk. By default is is 2 MB.
      * @throws Exception
@@ -21,6 +22,22 @@ class TmpFileTool
         }
     }
 
+    /**
+     * Write data to stream with separator. Separator will be prepend to content for next call.
+     * @param string $content
+     * @param string $separator
+     * @throws Exception
+     */
+    public function writeWithSeparator($content, $separator)
+    {
+        if (isset($this->separator)) {
+            $this->write($this->separator . $content);
+        } else {
+            $this->write($content);
+        }
+        $this->separator = $separator;
+    }
+
     /**
      * @param string $content
      * @throws Exception

app/Model/AppModel.php

@@ -86,7 +86,7 @@ class AppModel extends Model
         39 => false, 40 => false, 41 => false, 42 => false, 43 => false, 44 => false,
         45 => false, 46 => false, 47 => false, 48 => false, 49 => false, 50 => false,
         51 => false, 52 => false, 53 => false, 54 => false, 55 => false, 56 => false,
-        57 => false, 58 => false,
+        57 => false, 58 => false, 59 => false, 60 => false, 61 => false,
     );
 
     public $advanced_updates_description = array(
@@ -1418,6 +1418,37 @@ class AppModel extends Model
                 $sqlArray[] = sprintf("INSERT INTO `admin_settings` (`setting`, `value`) VALUES ('fix_login', %s);", time());
                 break;
             case 58:
+                $sqlArray[] = "ALTER TABLE `warninglists` MODIFY COLUMN `warninglist_entry_count` int(11) unsigned NOT NULL DEFAULT 0;";
+                break;
+            case 59:
+                $sqlArray[] = "CREATE TABLE IF NOT EXISTS event_reports (
+                    `id` int(11) NOT NULL AUTO_INCREMENT,
+                    `uuid` varchar(40) COLLATE utf8_bin NOT NULL ,
+                    `event_id` int(11) NOT NULL,
+                    `name` varchar(255) NOT NULL,
+                    `content` text,
+                    `distribution` tinyint(4) NOT NULL DEFAULT 0,
+                    `sharing_group_id` int(11),
+                    `timestamp` int(11) NOT NULL,
+                    `deleted` tinyint(1) NOT NULL DEFAULT 0,
+                    PRIMARY KEY (id),
+                    CONSTRAINT u_uuid UNIQUE (uuid),
+                    INDEX `name` (`name`)
+                ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;";
+                break;
+            case 60:
+                $sqlArray[] = "CREATE TABLE IF NOT EXISTS `attachment_scans` (
+                    `id` int(11) NOT NULL AUTO_INCREMENT,
+                    `type` varchar(40) COLLATE utf8_bin NOT NULL,
+                    `attribute_id` int(11) NOT NULL,
+                    `infected` tinyint(1) NOT NULL,
+                    `malware_name`  varchar(191) NULL,
+                    `timestamp` int(11) NOT NULL,
+                    PRIMARY KEY (`id`),
+                    INDEX `index` (`type`, `attribute_id`)
+                ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;";
+                break;
+            case 61:
                 $sqlArray[] = "ALTER TABLE `galaxy_clusters` ADD `distribution` tinyint(4) NOT NULL DEFAULT 0;";
                 $sqlArray[] = "ALTER TABLE `galaxy_clusters` ADD `sharing_group_id` int(11);";
                 $sqlArray[] = "ALTER TABLE `galaxy_clusters` ADD `org_id` int(11) NOT NULL;";
@@ -1792,14 +1823,6 @@ class AppModel extends Model
         }
     }
 
-    public function checkMISPVersion()
-    {
-        $file = new File(ROOT . DS . 'VERSION.json', true);
-        $version_array = json_decode($file->read(), true);
-        $file->close();
-        return $version_array;
-    }
-
     public function getPythonVersion()
     {
         if (!empty(Configure::read('MISP.python_bin'))) {
@@ -1962,9 +1985,7 @@ class AppModel extends Model
                     $workerType = '';
                     if (isset($workerDiagnostic['update']['ok']) && $workerDiagnostic['update']['ok']) {
                         $workerType = 'update';
-                    } elseif (isset($workerDiagnostic['prio']['ok']) && $workerDiagnostic['prio']['ok']) {
-                        $workerType = 'prio';
-                    } else { // no worker running, doing inline update
+                    } else { // update worker not running, doing the update inline
                         return $this->runUpdates($verbose, false);
                     }
                     $this->Job->create();
@@ -2640,35 +2661,67 @@ class AppModel extends Model
         return $HttpSocket;
     }
 
-    public function setupSyncRequest($server)
+    /**
+     * @param array $server
+     * @return array[]
+     * @throws JsonException
+     */
+    protected function setupSyncRequest(array $server)
     {
+        $version = implode('.', $this->checkMISPVersion());
         $request = array(
-                'header' => array(
-                        'Authorization' => $server['Server']['authkey'],
-                        'Accept' => 'application/json',
-                        'Content-Type' => 'application/json'
-                )
+            'header' => array(
+                'Authorization' => $server['Server']['authkey'],
+                'Accept' => 'application/json',
+                'Content-Type' => 'application/json',
+                'MISP-version' => $version,
+            )
         );
-        $request = $this->addHeaders($request);
-        return $request;
-    }
 
-    public function addHeaders($request)
-    {
-        $version = $this->checkMISPVersion();
-        $version = implode('.', $version);
-        try {
-            $commit = trim(shell_exec('git log --pretty="%H" -n1 HEAD'));
-        } catch (Exception $e) {
-            $commit = false;
-        }
-        $request['header']['MISP-version'] = $version;
+        $commit = $this->checkMIPSCommit();
         if ($commit) {
             $request['header']['commit'] = $commit;
         }
+        $request['header']['User-Agent'] = 'MISP ' . $version . (empty($commit) ? '' : ' - #' . $commit);
         return $request;
     }
 
+    /**
+     * Returns MISP version from VERSION.json file as array with major, minor and hotfix keys.
+     *
+     * @return array
+     * @throws JsonException
+     */
+    public function checkMISPVersion()
+    {
+        static $versionArray;
+        if ($versionArray === null) {
+            $file = new File(ROOT . DS . 'VERSION.json', true);
+            $versionArray = $this->jsonDecode($file->read());
+            $file->close();
+        }
+        return $versionArray;
+    }
+
+    /**
+     * Returns MISP commit hash.
+     *
+     * @return false|string
+     */
+    protected function checkMIPSCommit()
+    {
+        static $commit;
+        if ($commit === null) {
+            $commit = shell_exec('git log --pretty="%H" -n1 HEAD');
+            if ($commit) {
+                $commit = trim($commit);
+            } else {
+                $commit = false;
+            }
+        }
+        return $commit;
+    }
+
     // take filters in the {"OR" => [foo], "NOT" => [bar]} format along with conditions and set the conditions
     public function generic_add_filter($conditions, &$filter, $keys)
     {
@@ -2736,6 +2789,7 @@ class AppModel extends Model
             $temp = explode('&&', $filter);
             $filter = array();
             foreach ($temp as $f) {
+                $f = strval($f);
                 if ($f[0] === '!') {
                     $filter['NOT'][] = substr($f, 1);
                 } else {
@@ -2747,6 +2801,7 @@ class AppModel extends Model
         if (!isset($filter['OR']) && !isset($filter['NOT']) && !isset($filter['AND'])) {
             $temp = array();
             foreach ($filter as $param) {
+                $param = strval($param);
                 if (!empty($param)) {
                     if ($param[0] === '!') {
                         $temp['NOT'][] = substr($param, 1);
@@ -3012,4 +3067,27 @@ class AppModel extends Model
 
         return $this->attachmentTool;
     }
+
+    /**
+     * @return AttachmentScan
+     */
+    protected function loadAttachmentScan()
+    {
+        if ($this->AttachmentScan === null) {
+            $this->AttachmentScan = ClassRegistry::init('AttachmentScan');
+        }
+
+        return $this->AttachmentScan;
+    }
+
+    /**
+     * @return Log
+     */
+    protected function loadLog()
+    {
+        if (!isset($this->Log)) {
+            $this->Log = ClassRegistry::init('Log');
+        }
+        return $this->Log;
+    }
 }

app/Model/AttachmentScan.php

@@ -0,0 +1,528 @@
+<?php
+App::uses('AppModel', 'Model');
+
+class AttachmentScan extends AppModel
+{
+    const TYPE_ATTRIBUTE = 'Attribute',
+        TYPE_SHADOW_ATTRIBUTE = 'ShadowAttribute';
+
+    // base64 encoded eicar.exe
+    const EICAR = 'WDVPIVAlQEFQWzRcUFpYNTQoUF4pN0NDKTd9JEVJQ0FSLVNUQU5EQVJELUFOVElWSVJVUy1URVNULUZJTEUhJEgrSCo=';
+
+    /** @var AttachmentTool */
+    private $attachmentTool;
+
+    /** @var Module */
+    private $moduleModel;
+
+    /** @var mixed|null  */
+    private $attachmentScanModuleName;
+
+    /**
+     * List of supported object templates
+     * @var string[]
+     */
+    private $signatureTemplates = [
+        '4dbb56ef-4763-4c97-8696-a2bfc305cf8e', // av-signature
+        '984c5c39-be7f-4e1e-b034-d3213bac51cb', // sb-signature
+    ];
+
+    /**
+     * List of supported ways how to send data to module. From the most reliable to worst.
+     * @var string[]
+     */
+    private $possibleTypes = [
+        'attachment',
+        'sha3-512',
+        'sha3-384',
+        'sha3-256',
+        'sha3-224',
+        'sha512',
+        'sha512/224',
+        'sha512/256',
+        'sha384',
+        'sha256',
+        'sha224',
+        'sha1',
+        'md5',
+    ];
+
+    public function __construct($id = false, $table = null, $ds = null)
+    {
+        parent::__construct($id, $table, $ds);
+        $this->attachmentScanModuleName = Configure::read('MISP.attachment_scan_module');
+
+        // This can be useful, if you use third party service (like VirusTotal) and you don't want leak uploaded
+        // files payload. Then just file hash will be send.
+        if (Configure::read('MISP.attachment_scan_hash_only')) {
+            array_shift($this->possibleTypes); // remove 'attachment' type
+        }
+    }
+
+    /**
+     * Checks configuration and connection to module wth AV engine and returns an array of scanning software.
+     *
+     * @return array
+     * @throws Exception
+     */
+    public function diagnostic()
+    {
+        if (!$this->isEnabled()) {
+            throw new Exception("Malware scanning module is not configured.");
+        }
+
+        if ($this->attachmentTool()->attachmentDirIsS3()) {
+            throw new Exception("S3 attachment storage is not supported now for malware scanning.");
+        }
+
+        $moduleInfo = $this->loadModuleInfo($this->attachmentScanModuleName);
+
+        if (in_array('attachment', $moduleInfo['types'])) {
+            $fakeAttribute = [
+                'uuid' => CakeText::uuid(),
+                'event_id' => 1,
+                'type' => 'attachment',
+                'value' => 'eicar.com',
+                'data' => self::EICAR,
+            ];
+        } else {
+            $hashAlgo = $moduleInfo['types'][0];
+            $hash = hash($hashAlgo, base64_decode(self::EICAR));
+            $fakeAttribute = [
+                'uuid' => CakeText::uuid(),
+                'event_id' => 1,
+                'type' => $hashAlgo,
+                'value' => $hash,
+            ];
+        }
+        $results = $this->sendToModule($fakeAttribute, $moduleInfo['config']);
+        if (empty($results)) {
+            throw new Exception("Eicar test file was not detected.");
+        }
+
+        return array_column($results, 'software');
+    }
+
+    /**
+     * @return bool
+     */
+    public function isEnabled()
+    {
+        return !empty($this->attachmentScanModuleName);
+    }
+
+    /**
+     * @param string $type
+     * @param int $attributeId Attribute or Shadow Attribute ID
+     * @param bool $infected
+     * @param string|null $malwareName
+     * @return bool
+     * @throws Exception
+     */
+    public function insertScan($type, $attributeId, $infected, $malwareName = null)
+    {
+        $this->checkType($type);
+        $this->create();
+        $result = $this->save(array(
+            'type' => $type,
+            'attribute_id' => $attributeId,
+            'infected' => $infected,
+            'malware_name' => $malwareName,
+            'timestamp' => time(),
+        ));
+        if (!$result) {
+            throw new Exception("Could not save scan result for attribute $attributeId: " . json_encode($this->validationErrors));
+        }
+        return true;
+    }
+
+    /**
+     * @param string $type
+     * @param int $attributeId Attribute or Shadow Attribute ID
+     * @return array|null
+     */
+    public function getLatestScan($type, $attributeId)
+    {
+        $this->checkType($type);
+        return $this->find('first', array(
+            'conditions' => array(
+                'type' => $type,
+                'attribute_id' => $attributeId,
+            ),
+            'fields' => ['infected', 'malware_name'],
+            'order' => 'timestamp DESC', // newest first
+        ));
+    }
+
+    /**
+     * Checks if file is infected according to latest scan. Return values:
+     *  - null - file was never checked
+     *  - false - file is not infected according to latest scan
+     *  - string - file is infected, string contains malware name
+     *
+     * @param string $type
+     * @param int $attributeId Attribute or Shadow Attribute ID
+     * @return bool|null|string
+     */
+    public function isInfected($type, $attributeId)
+    {
+        $latest = $this->getLatestScan($type, $attributeId);
+        if (empty($latest)) {
+            return null;
+        }
+        if ($latest['AttachmentScan']['infected']) {
+            return $latest['AttachmentScan']['malware_name'];
+        } else {
+            return false;
+        }
+    }
+
+    /**
+     * @param string $type
+     * @param int $attributeId Attribute or ShadowAttribute ID
+     * @param int|null $jobId
+     * @return bool|string
+     * @throws Exception
+     */
+    public function scan($type, $attributeId = null, $jobId = null)
+    {
+        /** @var Job $job */
+        $job = ClassRegistry::init('Job');
+        if ($jobId && !$job->exists($jobId)) {
+            $jobId = null;
+        }
+
+        if (!$this->isEnabled()) {
+            throw new Exception("Malware scanning module is not configured.");
+        }
+
+        if ($this->attachmentTool()->attachmentDirIsS3()) {
+            throw new Exception("S3 attachment storage is not supported now for malware scanning.");
+        }
+
+        $fields = ['id', 'uuid', 'type', 'value', 'event_id'];
+        if ($type === 'all') {
+            $attributes = ClassRegistry::init('Attribute')->find('all', array(
+                'recursive' => -1,
+                'conditions' => ['type' => 'attachment'],
+                'fields' => $fields,
+            ));
+            $shadowAttributes = ClassRegistry::init('ShadowAttribute')->find('all', array(
+                'recursive' => -1,
+                'conditions' => ['type' => 'attachment'],
+                'fields' => $fields,
+            ));
+            $attributes = array_merge($attributes, $shadowAttributes);
+        } else if ($type === self::TYPE_ATTRIBUTE) {
+            $attributes = ClassRegistry::init('Attribute')->find('all', array(
+                'recursive' => -1,
+                'conditions' => ['type' => 'attachment', 'id' => $attributeId],
+                'fields' => $fields,
+            ));
+        } else if ($type === self::TYPE_SHADOW_ATTRIBUTE) {
+            $attributes = ClassRegistry::init('ShadowAttribute')->find('all', array(
+                'recursive' => -1,
+                'conditions' => ['type' => 'attachment', 'id' => $attributeId],
+                'fields' => $fields,
+            ));
+        } else {
+            throw new InvalidArgumentException("Input must be 'all', 'Attribute' or 'ShadowAttribute', '$type' provided.");
+        }
+
+        if (empty($attributes) && $type !== 'all') {
+            $message = "$type not found";
+            $job->saveStatus($jobId, false, $message);
+            return $message;
+        }
+
+        try {
+            $moduleInfo = $this->loadModuleInfo($this->attachmentScanModuleName);
+        } catch (Exception $e) {
+            $job->saveStatus($jobId, false, 'Could not connect to attachment scan module.');
+            $this->logException('Could not connect to attachment scan module.', $e);
+            return false;
+        }
+
+        $scanned = 0;
+        $fails = 0;
+        $virusFound = 0;
+        foreach ($attributes as $attribute) {
+            $type = isset($attribute['Attribute']) ? self::TYPE_ATTRIBUTE : self::TYPE_SHADOW_ATTRIBUTE;
+            try {
+                $infected = $this->scanAttachment($type, $attribute[$type], $moduleInfo);
+                if ($infected === true) {
+                    $virusFound++;
+                }
+                $scanned++;
+            } catch (NotFoundException $e) {
+                // skip
+            } catch (Exception $e) {
+                $this->logException("Could not scan attachment for $type {$attribute['Attribute']['id']}", $e);
+                $fails++;
+            }
+
+            $message = "$scanned files scanned, $virusFound malware files found.";
+            $job->saveProgress($jobId, $message, ($scanned + $fails) / count($attributes) * 100);
+        }
+
+        if ($scanned === 0 && $fails > 0) {
+            $job->saveStatus($jobId, false);
+            return false;
+        } else {
+            $message = "$scanned files scanned, $virusFound malware files found.";
+            if ($fails) {
+                $message .= " $fails files failed to scan (see error log for more details).";
+            }
+
+            $job->saveStatus($jobId, true, "Job done, $message");
+            return $message;
+        }
+    }
+
+    /**
+     * @param string $type
+     * @param array $attribute Attribute or Shadow Attribute
+     * @throws Exception
+     */
+    public function backgroundScan($type, array $attribute)
+    {
+        $this->checkType($type);
+
+        $canScan = $attribute['type'] === 'attachment' &&
+            $this->isEnabled() &&
+            Configure::read('MISP.background_jobs') &&
+            !$this->attachmentTool()->attachmentDirIsS3();
+
+        if ($canScan) {
+            $job = ClassRegistry::init('Job');
+            $job->create();
+            $job->save(array(
+                'worker' => 'default',
+                'job_type' => 'virus_scan',
+                'job_input' => ($type === self::TYPE_ATTRIBUTE ? 'Attribute: ' : 'Shadow attribute: ') . $attribute['id'],
+                'status' => 0,
+                'retries' => 0,
+                'org' => 'SYSTEM',
+                'message' => 'Scanning...',
+            ));
+
+            $processId = CakeResque::enqueue(
+                'default',
+                'AdminShell',
+                array('scanAttachment', $type, $attribute['id'], $job->id),
+                true
+            );
+            $job->saveField('process_id', $processId);
+        }
+    }
+
+    /**
+     * @param string $type
+     * @param array $attribute
+     * @param array $moduleInfo
+     * @return bool|null Return true if attachment is infected.
+     * @throws Exception
+     */
+    private function scanAttachment($type, array $attribute, array $moduleInfo)
+    {
+        if (!isset($attribute['type'])) {
+            throw new InvalidArgumentException("Invalid attribute provided.");
+        }
+
+        if ($attribute['type'] !== 'attachment') {
+            throw new InvalidArgumentException("Just attachment attributes can be scanned, attribute with type '{$attribute['type']}' provided.");
+        }
+
+        if ($type === self::TYPE_ATTRIBUTE) {
+            $file = $this->attachmentTool()->getFile($attribute['event_id'], $attribute['id']);
+        } else {
+            $file = $this->attachmentTool()->getShadowFile($attribute['event_id'], $attribute['id']);
+        }
+
+        if (in_array('attachment', $moduleInfo['types'])) {
+            /* if ($file->size() > 50 * 1024 * 1024) {
+             $this->log("File '$file->path' is bigger than 50 MB, will be not scanned.", LOG_NOTICE);
+             return false;
+         }*/
+
+            $fileContent = $file->read();
+            if ($fileContent === false) {
+                throw new Exception("Could not read content of  file '$file->path'.");
+            }
+            $attribute['data'] = base64_encode($fileContent);
+        } else {
+            // Instead of sending whole file to module, just generate file hash and send that hash as fake attribute.
+            $hashAlgo = $moduleInfo['types'][0];
+            $hash = hash_file($hashAlgo, $file->pwd());
+            if (!$hash) {
+                throw new Exception("Could not generate $hashAlgo hash for file '$file->path'.");
+            }
+
+            $attribute = [
+                'uuid' => CakeText::uuid(),
+                'event_id' => $attribute['event_id'],
+                'type' => $hashAlgo,
+                'value' => $hash,
+            ];
+        }
+
+        $results = $this->sendToModule($attribute, $moduleInfo['config']);
+
+        if (!empty($results)) {
+            $signatures = [];
+            foreach ($results as $result) {
+                $signatures = array_merge($signatures, $result['signatures']);
+            }
+            $this->insertScan($type, $attribute['id'], true, implode(', ', $signatures));
+            return true;
+
+        } else {
+            $this->insertScan($type, $attribute['id'], false);
+            return false;
+        }
+    }
+
+    /**
+     * @param array $attribute
+     * @param array $moduleConfig
+     * @return array
+     * @throws Exception
+     */
+    private function sendToModule(array $attribute, array $moduleConfig)
+    {
+        // How long we will wait for scan result
+        $timeout = Configure::read('MISP.attachment_scan_timeout') ?: 30;
+        $data = [
+            'module' => $this->attachmentScanModuleName,
+            'attribute' => $attribute,
+            'event_id' => $attribute['event_id'],
+            'config' => $moduleConfig,
+            'timeout' => $timeout, // module internal timeout
+        ];
+
+        $results = $this->moduleModel()->sendRequest('/query', $timeout + 1, $data, 'Enrichment');
+        if (isset($results['error'])) {
+            throw new Exception("{$this->attachmentScanModuleName} module returns error: " . $results['error']);
+        }
+        if (!isset($results['results'])) {
+            throw new Exception("Invalid data received from {$this->attachmentScanModuleName} module.");
+        }
+        return $this->extractInfoFromModuleResult($results['results']);
+    }
+
+    /**
+     * Extracts data from scan results.
+     * @param array $results
+     * @return array
+     * @throws Exception
+     */
+    private function extractInfoFromModuleResult(array $results)
+    {
+        if (!isset($results['Object']) || !is_array($results['Object'])) {
+            return [];
+        }
+
+        $output = [];
+        foreach ($results['Object'] as $object) {
+            if (!isset($object['template_uuid'])) {
+                continue;
+            }
+            if (in_array($object['template_uuid'], $this->signatureTemplates)) {
+                $software = null;
+                $signatures = array();
+                foreach ($object['Attribute'] as $attribute) {
+                    if (!isset($attribute['object_relation']) || !isset($attribute['value'])) {
+                        continue;
+                    }
+                    if ($attribute['object_relation'] === 'signature') {
+                        $signatures[] = $attribute['value'];
+                    } else if ($attribute['object_relation'] === 'software') {
+                        $software = $attribute['value'];
+                    }
+                }
+                if (!empty($signatures) && $software) {
+                    $output[] = ['signatures' => $signatures, 'software' => $software];
+                }
+            }
+        }
+        return $output;
+    }
+
+    /**
+     * @param string $moduleName
+     * @return array
+     * @throws Exception
+     */
+    private function loadModuleInfo($moduleName)
+    {
+        $modules = $this->moduleModel()->getModules('Enrichment', true);
+
+        $module = null;
+        foreach ($modules as $temp) {
+            if (strtolower($temp['name']) === strtolower($moduleName)) {
+                $module = $temp;
+                break;
+            }
+        }
+
+        if (!$module) {
+            throw new Exception("Module $moduleName not found.");
+        }
+
+        if (!in_array('expansion', $module['meta']['module-type'])) {
+            throw new Exception("Module $moduleName must be expansion type.");
+        }
+
+        $types = array_intersect($this->possibleTypes, $module['mispattributes']['input']);
+        if (empty($types)) {
+            throw new Exception("Module $moduleName doesn't support at least one required type: " . implode(", ", $this->possibleTypes) . ".");
+        }
+
+        if (!isset($module['mispattributes']['format']) || $module['mispattributes']['format'] !== 'misp_standard') {
+            throw new Exception("Module $moduleName doesn't support misp_standard output format.");
+        }
+
+        $config = [];
+        if (isset($module['meta']['config'])) {
+            foreach ($module['meta']['config'] as $conf) {
+                $config[$conf] = Configure::read("Plugin.Enrichment_{$moduleName}_$conf");
+            }
+        }
+        return ['config' => $config, 'types' => $types];
+    }
+
+    /**
+     * @return AttachmentTool
+     */
+    private function attachmentTool()
+    {
+        if (!$this->attachmentTool) {
+            $this->attachmentTool = new AttachmentTool();
+        }
+
+        return $this->attachmentTool;
+    }
+
+    /**
+     * @return Module
+     */
+    private function moduleModel()
+    {
+        if (!$this->moduleModel) {
+            $this->moduleModel = ClassRegistry::init('Module');
+        }
+
+        return $this->moduleModel;
+    }
+
+    /**
+     * @param string $type
+     * @raise InvalidArgumentException
+     */
+    private function checkType($type)
+    {
+        if (!in_array($type, [self::TYPE_ATTRIBUTE, self::TYPE_SHADOW_ATTRIBUTE])) {
+            throw new InvalidArgumentException("Type must be 'Attribute' or 'ShadowAttribute', '$type' provided.");
+        }
+    }
+}

app/Model/Attribute.php

@@ -7,9 +7,11 @@ App::uses('FinancialTool', 'Tools');
 App::uses('RandomTool', 'Tools');
 App::uses('AttachmentTool', 'Tools');
 App::uses('TmpFileTool', 'Tools');
+App::uses('ComplexTypeTool', 'Tools');
 
 /**
  * @property Event $Event
+ * @property AttributeTag $AttributeTag
  */
 class Attribute extends AppModel
 {
@@ -96,16 +98,16 @@ class Attribute extends AppModel
             'Payload delivery' => array(
                     'desc' => __('Information about how the malware is delivered'),
                     'formdesc' => __('Information about the way the malware payload is initially delivered, for example information about the email or web-site, vulnerability used, originating IP etc. Malware sample itself should be attached here.'),
-                    'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash','filename|impfuzzy', 'filename|pehash', 'mac-address', 'mac-eui-64', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'hostname', 'domain', 'email', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'email-body', 'url', 'user-agent', 'AS', 'pattern-in-file', 'pattern-in-traffic', 'stix2-pattern', 'yara', 'sigma', 'mime-type', 'attachment', 'malware-sample', 'link', 'malware-type', 'comment', 'text', 'hex', 'vulnerability', 'weakness', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'other', 'hostname|port', 'email-dst-display-name', 'email-src-display-name', 'email-header', 'email-reply-to', 'email-x-mailer', 'email-mime-boundary', 'email-thread-index', 'email-message-id', 'mobile-application-id', 'chrome-extension-id', 'whois-registrant-email', 'anonymised')
+                    'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash','filename|impfuzzy', 'filename|pehash', 'mac-address', 'mac-eui-64', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'hostname', 'domain', 'email', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'email-body', 'url', 'user-agent', 'AS', 'pattern-in-file', 'pattern-in-traffic', 'filename-pattern', 'stix2-pattern', 'yara', 'sigma', 'mime-type', 'attachment', 'malware-sample', 'link', 'malware-type', 'comment', 'text', 'hex', 'vulnerability', 'cpe', 'weakness', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'other', 'hostname|port', 'email-dst-display-name', 'email-src-display-name', 'email-header', 'email-reply-to', 'email-x-mailer', 'email-mime-boundary', 'email-thread-index', 'email-message-id', 'mobile-application-id', 'chrome-extension-id', 'whois-registrant-email', 'anonymised')
                     ),
             'Artifacts dropped' => array(
                     'desc' => __('Any artifact (files, registry keys etc.) dropped by the malware or other modifications to the system'),
-                    'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'vhash', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy','filename|pehash', 'regkey', 'regkey|value', 'pattern-in-file', 'pattern-in-memory','pdb', 'stix2-pattern', 'yara', 'sigma', 'attachment', 'malware-sample', 'named pipe', 'mutex', 'windows-scheduled-task', 'windows-service-name', 'windows-service-displayname', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'cookie', 'gene', 'kusto-query', 'mime-type', 'anonymised', 'pgp-public-key', 'pgp-private-key')
+                    'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy','filename|pehash', 'regkey', 'regkey|value', 'pattern-in-file', 'pattern-in-memory', 'filename-pattern', 'pdb', 'stix2-pattern', 'yara', 'sigma', 'attachment', 'malware-sample', 'named pipe', 'mutex', 'windows-scheduled-task', 'windows-service-name', 'windows-service-displayname', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'cookie', 'gene', 'kusto-query', 'mime-type', 'anonymised', 'pgp-public-key', 'pgp-private-key')
                     ),
             'Payload installation' => array(
                     'desc' => __('Info on where the malware gets installed in the system'),
                     'formdesc' => __('Location where the payload was placed in the system and the way it was installed. For example, a filename|md5 type attribute can be added here like this: c:\\windows\\system32\\malicious.exe|41d8cd98f00b204e9800998ecf8427e.'),
-                    'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy', 'filename|pehash', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'stix2-pattern', 'yara', 'sigma', 'vulnerability', 'weakness', 'attachment', 'malware-sample', 'malware-type', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'mobile-application-id', 'chrome-extension-id', 'other', 'mime-type', 'anonymised')
+                    'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy', 'filename|pehash', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'filename-pattern', 'stix2-pattern', 'yara', 'sigma', 'vulnerability', 'cpe','weakness', 'attachment', 'malware-sample', 'malware-type', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'mobile-application-id', 'chrome-extension-id', 'other', 'mime-type', 'anonymised')
                     ),
             'Persistence mechanism' => array(
                     'desc' => __('Mechanisms used by the malware to start at boot'),
@@ -114,7 +116,7 @@ class Attribute extends AppModel
                     ),
             'Network activity' => array(
                     'desc' => __('Information about network traffic generated by the malware'),
-                    'types' => array('ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'port', 'hostname', 'domain', 'domain|ip', 'mac-address', 'mac-eui-64', 'email', 'email-dst', 'email-src', 'eppn', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'pattern-in-file', 'stix2-pattern', 'pattern-in-traffic', 'attachment', 'comment', 'text', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'other', 'hex', 'cookie', 'hostname|port', 'bro', 'zeek', 'anonymised', 'community-id', 'email-subject')
+                    'types' => array('ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'port', 'hostname', 'domain', 'domain|ip', 'mac-address', 'mac-eui-64', 'email', 'email-dst', 'email-src', 'eppn', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'pattern-in-file', 'filename-pattern','stix2-pattern', 'pattern-in-traffic', 'attachment', 'comment', 'text', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'other', 'hex', 'cookie', 'hostname|port', 'bro', 'zeek', 'anonymised', 'community-id', 'email-subject')
                     ),
             'Payload type' => array(
                     'desc' => __('Information about the final payload(s)'),
@@ -128,7 +130,7 @@ class Attribute extends AppModel
             'External analysis' => array(
                     'desc' => __('Any other result from additional analysis of the malware like tools output'),
                     'formdesc' => __('Any other result from additional analysis of the malware like tools output Examples: pdf-parser output, automated sandbox analysis, reverse engineering report.'),
-                    'types' => array('md5', 'sha1', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'mac-address', 'mac-eui-64', 'hostname', 'domain', 'domain|ip', 'url', 'user-agent', 'regkey', 'regkey|value', 'AS', 'snort', 'bro', 'zeek', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'vulnerability', 'weakness', 'attachment', 'malware-sample', 'link', 'comment', 'text', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'github-repository', 'other', 'cortex', 'anonymised', 'community-id')
+                    'types' => array('md5', 'sha1', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'mac-address', 'mac-eui-64', 'hostname', 'domain', 'domain|ip', 'url', 'user-agent', 'regkey', 'regkey|value', 'AS', 'snort', 'bro', 'zeek', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'filename-pattern','vulnerability', 'cpe', 'weakness', 'attachment', 'malware-sample', 'link', 'comment', 'text', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'github-repository', 'other', 'cortex', 'anonymised', 'community-id')
                     ),
             'Financial fraud' => array(
                     'desc' => __('Financial Fraud indicators'),
@@ -196,6 +198,7 @@ class Attribute extends AppModel
             'pattern-in-file' => array('desc' => __('Pattern in file that identifies the malware'), 'default_category' => 'Payload installation', 'to_ids' => 1),
             'pattern-in-traffic' => array('desc' => __('Pattern in network traffic that identifies the malware'), 'default_category' => 'Network activity', 'to_ids' => 1),
             'pattern-in-memory' => array('desc' => __('Pattern in memory dump that identifies the malware'), 'default_category' => 'Payload installation', 'to_ids' => 1),
+            'pattern-filename' => array('desc' => __('A pattern in the name of a file'), 'default_category' => 'Payload installation', 'to_ids' => 1),
             'pgp-public-key' => array('desc' => __('A PGP public key'), 'default_category' => 'Person', 'to_ids' => 0),
             'pgp-private-key' => array('desc' => __('A PGP private key'), 'default_category' => 'Person', 'to_ids' => 0),
             'yara' => array('desc' => __('Yara signature'), 'default_category' => 'Payload installation', 'to_ids' => 1),
@@ -207,6 +210,7 @@ class Attribute extends AppModel
             'identity-card-number' => array('desc' => __('Identity card number'), 'default_category' => 'Person', 'to_ids' => 0),
             'cookie' => array('desc' => __('HTTP cookie as often stored on the user web client. This can include authentication cookie or session cookie.'), 'default_category' => 'Network activity', 'to_ids' => 0),
             'vulnerability' => array('desc' => __('A reference to the vulnerability used in the exploit'), 'default_category' => 'External analysis', 'to_ids' => 0),
+            'cpe' => array('desc' => __('Common Platform Enumeration - structured naming scheme for information technology systems, software, and packages.'), 'default_category' => 'External analysis', 'to_ids' => 0),
             'weakness' => array('desc'=> __('A reference to the weakness used in the exploit'), 'default_category' => 'External analysis', 'to_ids' => 0),
             'attachment' => array('desc' => __('Attachment with external information'), 'formdesc' => __("Please upload files using the <em>Upload Attachment</em> button."), 'default_category' => 'External analysis', 'to_ids' => 0),
             'malware-sample' => array('desc' => __('Attachment containing encrypted malware sample'), 'formdesc' => __("Please upload files using the <em>Upload Attachment</em> button."), 'default_category' => 'Payload delivery', 'to_ids' => 1),
@@ -243,6 +247,7 @@ class Attribute extends AppModel
             'vhash' => array('desc' => __('A VirusTotal checksum'), 'formdesc' => __("You are encouraged to use filename|vhash instead. A checksum from VirusTotal, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'ssdeep' => array('desc' => __('A checksum in ssdeep format'), 'formdesc' => __("You are encouraged to use filename|ssdeep instead. A checksum in the SSDeep format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'imphash' => array('desc' => __('Import hash - a hash created based on the imports in the sample.'), 'formdesc' => __("You are encouraged to use filename|imphash instead. A hash created based on the imports in the sample, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
+            'telfhash' => array('desc' => __('telfhash is symbol hash for ELF files, just like imphash is imports hash for PE files.'), 'formdesc' => __("You are encouraged to use a file object with telfash"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'pehash' => array('desc' => __('PEhash - a hash calculated based of certain pieces of a PE executable file'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'impfuzzy' => array('desc' => __('A fuzzy hash of import table of Portable Executable format'), 'formdesc' => __("You are encouraged to use filename|impfuzzy instead. A fuzzy hash created based on the imports in the sample, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'sha224' => array('desc' => __('A checksum in sha-224 format'), 'formdesc' => __("You are encouraged to use filename|sha224 instead. A checksum in sha224 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
@@ -292,7 +297,6 @@ class Attribute extends AppModel
             'size-in-bytes' => array('desc' => __('Size expressed in bytes'), 'default_category' => 'Other', 'to_ids' => 0),
             'counter' => array('desc' => __('An integer counter, generally to be used in objects'), 'default_category' => 'Other', 'to_ids' => 0),
             'datetime' => array('desc' => __('Datetime in the ISO 8601 format'), 'default_category' => 'Other', 'to_ids' => 0),
-            'cpe' => array('desc' => __('Common platform enumeration'), 'default_category' => 'Other', 'to_ids' => 0),
             'port' => array('desc' => __('Port number'), 'default_category' => 'Network activity', 'to_ids' => 0),
             'ip-dst|port' => array('desc' => __('IP destination and port number separated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1),
             'ip-src|port' => array('desc' => __('IP source and port number separated by a |'), 'default_category' => 'Network activity', 'to_ids' => 1),
@@ -378,7 +382,8 @@ class Attribute extends AppModel
 
     public $primaryOnlyCorrelatingTypes = array(
         'ip-src|port',
-        'ip-dst|port'
+        'ip-dst|port',
+        'hostname|port',
     );
 
     public $captureFields = array(
@@ -456,6 +461,7 @@ class Attribute extends AppModel
             'authentihash' => 'Payload delivery',
             'vhash' => 'Payload delivery',
             'imphash' => 'Payload delivery',
+            'telfhash' => 'Payload delivery',
             'impfuzzy'=> 'Payload delivery',
             'pehash' => 'Payload delivery',
             'cdhash' => 'Payload delivery',
@@ -495,7 +501,7 @@ class Attribute extends AppModel
     // whilst filenames and hashes are file related attribute types
     // This helps generate quick filtering for the event view, but we may reuse this and enhance it in the future for other uses (such as the API?)
     public $typeGroupings = array(
-        'file' => array('attachment', 'pattern-in-file', 'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|pehash', 'malware-sample', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'x509-fingerprint-md5'),
+        'file' => array('attachment', 'pattern-in-file', 'filename-pattern', 'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'telfhash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|pehash', 'malware-sample', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'x509-fingerprint-md5'),
         'network' => array('ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port', 'mac-address', 'mac-eui-64', 'hostname', 'hostname|port', 'domain', 'domain|ip', 'email-dst', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'bro', 'zeek',  'pattern-in-traffic', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256','ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'community-id'),
         'financial' => array('btc', 'xmr', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn', 'phone-number')
     );
@@ -847,8 +853,7 @@ class Attribute extends AppModel
             return false;
         }
         App::uses('ComplexTypeTool', 'Tools');
-        $this->complexTypeTool = new ComplexTypeTool();
-        $this->data['Attribute']['value'] = $this->complexTypeTool->refangValue($this->data['Attribute']['value'], $this->data['Attribute']['type']);
+        $this->data['Attribute']['value'] = ComplexTypeTool::refangValue($this->data['Attribute']['value'], $this->data['Attribute']['type']);
 
         if (!empty($this->data['Attribute']['object_id']) && empty($this->data['Attribute']['object_relation'])) {
             $this->validationErrors['type'] = ['Object attribute sent, but no object_relation set.'];
@@ -918,10 +923,6 @@ class Attribute extends AppModel
             $this->data['Attribute']['sharing_group_id'] = 0;
         }
         // return true, otherwise the object cannot be saved
-
-        if ($this->data['Attribute']['type'] == 'float' && $this->data['Attribute']['value'] == 0) {
-            $this->data['Attribute']['value'] = '0.0';
-        }
         return true;
     }
 
@@ -929,8 +930,7 @@ class Attribute extends AppModel
     {
         $compositeTypes = $this->getCompositeTypes();
         if (in_array($this->data['Attribute']['type'], $compositeTypes)) {
-            $pieces = explode('|', $fields['value']);
-            if (2 != count($pieces)) {
+            if (substr_count($fields['value'], '|') !== 1) {
                 return false;
             }
         }
@@ -1037,6 +1037,7 @@ class Attribute extends AppModel
         'authentihash' => 64,
         'md5' => 32,
         'imphash' => 32,
+        'telfhash' => 70,
         'sha1' => 40,
         'git-commit-id' => 40,
         'x509-fingerprint-md5' => 32,
@@ -1065,6 +1066,7 @@ class Attribute extends AppModel
         switch ($type) {
             case 'md5':
             case 'imphash':
+            case 'telfhash':
             case 'sha1':
             case 'sha224':
             case 'sha256':
@@ -1084,13 +1086,12 @@ class Attribute extends AppModel
             case 'x509-fingerprint-sha256':
             case 'x509-fingerprint-sha1':
             case 'git-commit-id':
-                $length = $this->__hexHashLengths[$type];
-                if (preg_match("#^[0-9a-f]{" . $length . "}$#", $value)) {
-                    $returnValue = true;
+                if ($this->isHashValid($type, $value)) {
+                    return true;
                 } else {
-                    $returnValue = __('Checksum has an invalid length or format (expected: %s hexadecimal characters). Please double check the value or select type "other".', $length);
+                    $length = $this->__hexHashLengths[$type];
+                    return __('Checksum has an invalid length or format (expected: %s hexadecimal characters). Please double check the value or select type "other".', $length);
                 }
-                break;
             case 'tlsh':
                 if (preg_match("#^[0-9a-f]{35,}$#", $value)) {
                     $returnValue = true;
@@ -1099,27 +1100,24 @@ class Attribute extends AppModel
                 }
                 break;
             case 'pehash':
-                if (preg_match("#^[0-9a-f]{40}$#", $value)) {
+                if ($this->isHashValid('pehash', $value)) {
                     $returnValue = true;
                 } else {
                     $returnValue = __('The input doesn\'t match the expected sha1 format (expected: 40 hexadecimal characters). Keep in mind that MISP currently only supports SHA1 for PEhashes, if you would like to get the support extended to other hash types, make sure to create a github ticket about it at https://github.com/MISP/MISP!');
                 }
                 break;
             case 'ssdeep':
-                if (substr_count($value, ':') == 2) {
+                if (substr_count($value, ':') === 2) {
                     $parts = explode(':', $value);
-                    if (is_numeric($parts[0])) {
-                        $returnValue = true;
+                    if ($this->isPositiveInteger($parts[0])) {
+                        return true;
                     }
                 }
-                if (!$returnValue) {
-                    $returnValue = __('Invalid SSDeep hash. The format has to be blocksize:hash:hash');
-                }
-                break;
+                return __('Invalid SSDeep hash. The format has to be blocksize:hash:hash');
             case 'impfuzzy':
-                if (substr_count($value, ':') == 2) {
+                if (substr_count($value, ':') === 2) {
                     $parts = explode(':', $value);
-                    if (is_numeric($parts[0])) {
+                    if ($this->isPositiveInteger($parts[0])) {
                         $returnValue = true;
                     }
                 }
@@ -1138,7 +1136,7 @@ class Attribute extends AppModel
                 if (preg_match("#(OPTIONS|GET|HEAD|POST|PUT|DELETE|TRACE|CONNECT|PROPFIND|PROPPATCH|MKCOL|COPY|MOVE|LOCK|UNLOCK|VERSION-CONTROL|REPORT|CHECKOUT|CHECKIN|UNCHECKOUT|MKWORKSPACE|UPDATE|LABEL|MERGE|BASELINE-CONTROL|MKACTIVITY|ORDERPATCH|ACL|PATCH|SEARCH)#", $value)) {
                     $returnValue = true;
                 } else {
-                    $returnValue = 'Unknown HTTP method.';
+                    $returnValue = __('Unknown HTTP method.');
                 }
                 break;
             case 'filename|pehash':
@@ -1179,7 +1177,7 @@ class Attribute extends AppModel
                     $value = $composite[1];
                     if (substr_count($value, ':') == 2) {
                         $parts = explode(':', $value);
-                        if (is_numeric($parts[0])) {
+                        if ($this->isPositiveInteger($parts[0])) {
                             $returnValue = true;
                         }
                     }
@@ -1196,7 +1194,7 @@ class Attribute extends AppModel
                 }
                 break;
             case 'filename|vhash':
-                if (preg_match("#^.+\|[a-zA-Z0-9&]+$#", $value)) {
+                if (preg_match('#^.+\|[a-zA-Z0-9&!=\"]+$#', $value)) {
                     $returnValue = true;
                 } else {
                     $returnValue = __('Checksum has an invalid length or format (expected: filename|string characters). Please double check the value or select type "other".');
@@ -1206,17 +1204,17 @@ class Attribute extends AppModel
             case 'ip-dst':
                 if (strpos($value, '/') !== false) {
                     $parts = explode("/", $value);
-                    if (count($parts) !== 2 || intval($parts[1]) != $parts[1] || $parts[1] < 0) {
+                    if (count($parts) !== 2 || !$this->isPositiveInteger($parts[1])) {
                         return __('Invalid CIDR notation value found.');
                     }
 
                     if (filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) {
                         if ($parts[1] > 32) {
-                            return __('Invalid CIDR notation value found.');
+                            return __('Invalid CIDR notation value found, for IPv4 must be lower or equal 32.');
                         }
                     } else if (filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) {
                         if ($parts[1] > 128) {
-                            return __('Invalid CIDR notation value found.');
+                            return __('Invalid CIDR notation value found, for IPv6 must be lower or equal 128.');
                         }
                     } else {
                         return __('IP address has an invalid format.');
@@ -1228,7 +1226,7 @@ class Attribute extends AppModel
 
             case 'port':
                 if (!$this->isPortValid($value)) {
-                    $returnValue = __('Port numbers have to be positive integers between 1 and 65535.');
+                    $returnValue = __('Port numbers have to be integers between 1 and 65535.');
                 } else {
                     $returnValue = true;
                 }
@@ -1236,17 +1234,20 @@ class Attribute extends AppModel
             case 'ip-dst|port':
             case 'ip-src|port':
                 $parts = explode('|', $value);
-                if (filter_var($parts[0], FILTER_VALIDATE_IP) && $this->isPortValid($parts[1])) {
-                    $returnValue = true;
+                if (!filter_var($parts[0], FILTER_VALIDATE_IP)) {
+                    return __('IP address has an invalid format.');
                 }
-                break;
+                if (!$this->isPortValid($parts[1])) {
+                    return __('Port numbers have to be integers between 1 and 65535.');
+                }
+                return true;
             case 'mac-address':
-                if (preg_match('/^([a-fA-F0-9]{2}[:|\-| |\.]?){6}$/', $value) == 1) {
+                if (preg_match('/^([a-fA-F0-9]{2}[:]?){6}$/', $value)) {
                     $returnValue = true;
                 }
                 break;
             case 'mac-eui-64':
-                if (preg_match('/^([a-fA-F0-9]{2}[:|\-| |\.]?){8}$/', $value) == 1) {
+                if (preg_match('/^([a-fA-F0-9]{2}[:]?){8}$/', $value)) {
                     $returnValue = true;
                 }
                 break;
@@ -1255,15 +1256,18 @@ class Attribute extends AppModel
                 if ($this->isDomainValid($value)) {
                     $returnValue = true;
                 } else {
-                    $returnValue = ucfirst($type) . __(' name has an invalid format. Please double check the value or select type "other".');
+                    $returnValue = __('%s has an invalid format. Please double check the value or select type "other".', ucfirst($type));
                 }
                 break;
             case 'hostname|port':
                 $parts = explode('|', $value);
-                if ($this->isDomainValid($parts[0]) && $this->isPortValid($parts[1])) {
-                    $returnValue = true;
+                if (!$this->isDomainValid($parts[0])) {
+                    return __('Hostname has an invalid format.');
                 }
-                break;
+                if (!$this->isPortValid($parts[1])) {
+                    return __('Port numbers have to be integers between 1 and 65535.');
+                }
+                return true;
             case 'domain|ip':
                 if (preg_match("#^[A-Z0-9.\-_]+\.[A-Z0-9\-]{2,}\|.*$#i", $value)) {
                     $parts = explode('|', $value);
@@ -1292,7 +1296,6 @@ class Attribute extends AppModel
                 }
                 break;
             case 'vulnerability':
-                $value = str_replace('–', '-', $value);
                 if (preg_match("#^(CVE-)[0-9]{4}(-)[0-9]{4,}$#", $value)) {
                     $returnValue = true;
                 } else {
@@ -1300,7 +1303,6 @@ class Attribute extends AppModel
                 }
                 break;
             case 'weakness':
-                $value = str_replace('–', '-', $value);
                 if (preg_match("#^(CWE-)[0-9]{1,}$#", $value)) {
                     $returnValue = true;
                 } else {
@@ -1321,7 +1323,6 @@ class Attribute extends AppModel
                 }
                 break;
             case 'mutex':
-            case 'AS':
             case 'snort':
             case 'bro':
             case 'zeek':
@@ -1330,6 +1331,7 @@ class Attribute extends AppModel
             case 'pattern-in-file':
             case 'pattern-in-traffic':
             case 'pattern-in-memory':
+            case 'filename-pattern':
             case 'pgp-public-key':
             case 'pgp-private-key':
             case 'yara':
@@ -1342,6 +1344,16 @@ class Attribute extends AppModel
             case 'cookie':
             case 'attachment':
             case 'malware-sample':
+            case 'comment':
+            case 'text':
+            case 'other':
+            case 'cpe':
+            case 'email-attachment':
+            case 'email-body':
+            case 'email-header':
+            case 'first-name':
+            case 'middle-name':
+            case 'last-name':
                 $returnValue = true;
                 break;
             case 'link':
@@ -1350,21 +1362,8 @@ class Attribute extends AppModel
                     $returnValue = true;
                 }
                 break;
-            case 'comment':
-            case 'text':
-            case 'other':
-            case 'email-attachment':
-            case 'email-body':
-            case 'first-name':
-            case 'middle-name':
-            case 'last-name':
-                $returnValue = true;
-                break;
             case 'hex':
-                if (preg_match("/^[0-9a-f]*$/i", $value)) {
-                    $returnValue = true;
-                }
-                break;
+                return ctype_xdigit($value);
             case 'target-user':
             case 'campaign-name':
             case 'campaign-id':
@@ -1384,8 +1383,8 @@ class Attribute extends AppModel
             case 'filename':
             case 'pdb':
             case 'windows-scheduled-task':
-      case 'whois-registrant-name':
-      case 'whois-registrant-org':
+            case 'whois-registrant-name':
+            case 'whois-registrant-org':
             case 'whois-registrar':
             case 'whois-creation-date':
             case 'date-of-birth':
@@ -1418,18 +1417,13 @@ class Attribute extends AppModel
             case 'github-username':
             case 'github-repository':
             case 'github-organisation':
-            case 'cpe':
             case 'twitter-id':
             case 'chrome-extension-id':
             case 'mobile-application-id':
-                // no newline
-                if (!preg_match("#\n#", $value)) {
-                    $returnValue = true;
+                if (strpos($value, "\n") !== false) {
+                    return __('Value must not contain new line character.');
                 }
-                break;
-            case 'email-header':
-                $returnValue = true;
-                break;
+                return true;
             case 'datetime':
                 try {
                     new DateTime($value);
@@ -1440,12 +1434,10 @@ class Attribute extends AppModel
                 break;
             case 'size-in-bytes':
             case 'counter':
-                if (!is_numeric($value) || $value < 0) {
-                    $returnValue = __('The value has to be a number greater or equal 0.');
-                } else {
-                    $returnValue = true;
+                if ($this->isPositiveInteger($value)) {
+                    return true;
                 }
-                break;
+                return __('The value has to be a whole number greater or equal 0.');
             case 'targeted-threat-index':
                 if (!is_numeric($value) || $value < 0 || $value > 10) {
                     $returnValue = __('The value has to be a number between 0 and 10.');
@@ -1458,8 +1450,12 @@ class Attribute extends AppModel
             case 'btc':
             case 'dash':
             case 'xmr':
+                if (preg_match('/^[a-zA-Z0-9]+$/', $value)) {
+                    $returnValue = true;
+                }
+                break;
             case 'vhash':
-                if (preg_match('/^[a-zA-Z0-9&]+$/', $value)) {
+                if (preg_match('/^[a-zA-Z0-9&!="]+$/', $value)) {
                     $returnValue = true;
                 }
                 break;
@@ -1479,16 +1475,17 @@ class Attribute extends AppModel
                 $returnValue = (json_last_error() == JSON_ERROR_NONE);
                 break;
             case 'float':
-                $value = floatval($value);
-                if (is_float($value)) {
-                    $returnValue = true;
-                }
-                break;
+                return is_numeric($value);
             case 'boolean':
                 if ($value == 1 || $value == 0) {
                     $returnValue = true;
                 }
                 break;
+            case 'AS':
+                if ($this->isPositiveInteger($value) && $value <= 4294967295) {
+                    return true;
+                }
+                return __('AS number have to be integers between 1 and 4294967295');
         }
         return $returnValue;
     }
@@ -1518,6 +1515,7 @@ class Attribute extends AppModel
             case 'authentihash':
             case 'vhash':
             case 'imphash':
+            case 'telfhash':
             case 'tlsh':
             case 'anonymised':
             case 'cdhash':
@@ -1577,8 +1575,12 @@ class Attribute extends AppModel
                 $value = $pieces[0] . '|' . strtolower($pieces[1]);
                 break;
             case 'http-method':
-                $value = strtoupper($value);
-                break;
+            case 'hex':
+                return strtoupper($value);
+            case 'vulnerability':
+            case 'weakness':
+                $value = str_replace('–', '-', $value);
+                return strtoupper($value);
             case 'cc-number':
             case 'bin':
                 $value = preg_replace('/[^0-9]+/', '', $value);
@@ -1597,12 +1599,8 @@ class Attribute extends AppModel
                 $value = preg_replace('/\(0\)/', '', $value);
                 $value = preg_replace('/[^\+0-9]+/', '', $value);
                 break;
-            case 'url':
-                $value = preg_replace('/^hxxp/i', 'http', $value);
-                $value = preg_replace('/\[\.\]/', '.', $value);
-                break;
-      case 'x509-fingerprint-md5':
-      case 'x509-fingerprint-sha256':
+            case 'x509-fingerprint-md5':
+            case 'x509-fingerprint-sha256':
             case 'x509-fingerprint-sha1':
                 $value = str_replace(':', '', $value);
                 $value = strtolower($value);
@@ -1647,19 +1645,14 @@ class Attribute extends AppModel
                         $parts[0] = inet_ntop(inet_pton($parts[0]));
                     }
                     return $parts[0] . '|' . $parts[1];
-                break;
             case 'mac-address':
             case 'mac-eui-64':
-                $value = str_replace(array('.', ':', '-', ' '), '', $value);
+                $value = str_replace(array('.', ':', '-', ' '), '', strtolower($value));
                 $value = wordwrap($value, 2, ':', true);
                 break;
             case 'hostname|port':
                 $value = strtolower($value);
-                str_replace(':', '|', $value);
-                break;
-            case 'hex':
-                $value = strtoupper($value);
-                break;
+                return str_replace(':', '|', $value);
             case 'boolean':
                 if ('true' == trim(strtolower($value))) {
                     $value = 1;
@@ -1676,6 +1669,17 @@ class Attribute extends AppModel
                     // silently skip. Rejection will be done in runValidation()
                 }
                 break;
+            case 'AS':
+                if (strtoupper(substr($value, 0, 2)) === 'AS') {
+                    $value = substr($value, 2); // remove 'AS'
+                }
+                if (strpos($value, '.') !== false) { // maybe value is in asdot notation
+                    $parts = explode('.', $value);
+                    if ($this->isPositiveInteger($parts[0]) && $this->isPositiveInteger($parts[1])) {
+                        return $parts[0] * 65536 + $parts[1];
+                    }
+                }
+                break;
         }
         return $value;
     }
@@ -1816,7 +1820,11 @@ class Attribute extends AppModel
 
     public function saveAttachment($attribute, $path_suffix='')
     {
-        return $this->loadAttachmentTool()->save($attribute['event_id'], $attribute['id'], $attribute['data'], $path_suffix);
+        $result = $this->loadAttachmentTool()->save($attribute['event_id'], $attribute['id'], $attribute['data'], $path_suffix);
+        if ($result) {
+            $this->loadAttachmentScan()->backgroundScan(AttachmentScan::TYPE_ATTRIBUTE, $attribute);
+        }
+        return $result;
     }
 
     /**
@@ -3134,7 +3142,6 @@ class Attribute extends AppModel
                 'value' => $value,
         );
         if ($element['complex']) {
-            App::uses('ComplexTypeTool', 'Tools');
             $complexTypeTool = new ComplexTypeTool();
             $result = $complexTypeTool->checkComplexRouter($value, ucfirst($element['type']));
             if (isset($result['multi'])) {
@@ -3242,13 +3249,20 @@ class Attribute extends AppModel
         return $attribute;
     }
 
-    public function fetchAttributesSimple($user, $options = array())
+    /**
+     * Fetches attributes that $user can see.
+     *
+     * @param array $user
+     * @param array $options
+     * @return array
+     */
+    public function fetchAttributesSimple(array $user, array $options = array())
     {
         $params = array(
             'conditions' => $this->buildConditions($user),
             'fields' => array(),
             'recursive' => -1,
-            'contain' => array()
+            'contain' => ['Event', 'Object'], // by default include Event and Object, because it is required for conditions
         );
         if (isset($options['conditions'])) {
             $params['conditions']['AND'][] = $options['conditions'];
@@ -3259,14 +3273,13 @@ class Attribute extends AppModel
         if (isset($options['contain'])) {
             $params['contain'] = $options['contain'];
         }
-        $results = $this->find('all', array(
+        return $this->find('all', array(
             'conditions' => $params['conditions'],
             'recursive' => -1,
             'fields' => $params['fields'],
             'contain' => $params['contain'],
             'sort' => false
         ));
-        return $results;
     }
 
     // Method that fetches all attributes for the various exports
@@ -3466,9 +3479,8 @@ class Attribute extends AppModel
             return $results;
         }
 
-        if (($options['enforceWarninglist'] || $options['includeWarninglistHits']) && !isset($this->warninglists)) {
+        if (($options['enforceWarninglist'] || $options['includeWarninglistHits']) && !isset($this->Warninglist)) {
             $this->Warninglist = ClassRegistry::init('Warninglist');
-            $this->warninglists = $this->Warninglist->fetchForEventView();
         }
         if (empty($params['limit'])) {
             $loopLimit = 50000;
@@ -3477,7 +3489,6 @@ class Attribute extends AppModel
             $params['page'] = 0;
         } else {
             $loop = false;
-            $pagesToFetch = 1;
         }
         $attributes = array();
         if (!empty($options['includeEventTags'])) {
@@ -3538,14 +3549,15 @@ class Attribute extends AppModel
             $results = array_values($results);
             $proposals_block_attributes = Configure::read('MISP.proposals_block_attributes');
             foreach ($results as $key => $attribute) {
+                if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttribute($attribute['Attribute'])) {
+                    unset($results[$key]); // Remove attribute that match any enabled warninglists
+                    continue;
+                }
                 if (!empty($options['includeEventTags'])) {
                     $results = $this->__attachEventTagsToAttributes($eventTags, $results, $key, $options);
                 }
-                if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttributes($this->warninglists, $attribute['Attribute'])) {
-                    continue;
-                }
                 if ($options['includeWarninglistHits']) {
-                    $results[$key]['Attribute'] = $this->Warninglist->simpleCheckForWarning($results[$key]['Attribute'], $this->warninglists, true);
+                    $results[$key]['Attribute'] = $this->Warninglist->checkForWarning($results[$key]['Attribute']);
                 }
                 if (!empty($options['includeAttributeUuid']) || !empty($options['includeEventUuid'])) {
                     $results[$key]['Attribute']['event_uuid'] = $results[$key]['Event']['uuid'];
@@ -3699,12 +3711,18 @@ class Attribute extends AppModel
         return $validTypes;
     }
 
-    public function validateAttribute($attribute, $context = true)
+    /**
+     * @param array $attribute
+     * @param bool $context
+     * @return array|true
+     */
+    public function validateAttribute(array $attribute, $context = true)
     {
         $this->set($attribute);
         if (!$context) {
             unset($this->validate['event_id']);
             unset($this->validate['value']['uniqueValue']);
+            unset($this->validate['uuid']['unique']);
         }
         if ($this->validates()) {
             return true;
@@ -3928,7 +3946,7 @@ class Attribute extends AppModel
                 'value1 LIKE' => '%/%'
             ),
             'fields' => array('value1'),
-            'group' => 'value1', // return just unique value
+            'group' => array('value1', 'id'), // return just unique value
             'order' => false
         ));
     }
@@ -4118,10 +4136,7 @@ class Attribute extends AppModel
         }
         if (!empty($attribute['enforceWarninglist']) || !empty($params['enforceWarninglist'])) {
             $this->Warninglist = ClassRegistry::init('Warninglist');
-            if (empty($this->warninglists)) {
-                $this->warninglists = $this->Warninglist->fetchForEventView();
-            }
-            if (!$this->Warninglist->filterWarninglistAttributes($warninglists, $attributes[$k])) {
+            if (!$this->Warninglist->filterWarninglistAttribute($attribute)) {
                 $this->validationErrors['warninglist'] = 'Attribute could not be saved as it trips over a warninglist and enforceWarninglist is enforced.';
                 $validationErrors = $this->validationErrors['warninglist'];
                 $log->create();
@@ -4132,7 +4147,7 @@ class Attribute extends AppModel
                         'email' => $user['email'],
                         'action' => 'add',
                         'user_id' => $user['id'],
-                        'title' => 'Attribute dropped due to validation for Event ' . $eventId . ' failed: ' . $attribute_short,
+                        'title' => 'Attribute dropped due to validation for Event ' . $eventId . ' failed',
                         'change' => 'Validation errors: ' . json_encode($this->validationErrors) . ' Full Attribute: ' . json_encode($attribute),
                 ));
                 return $attribute;
@@ -4638,46 +4653,49 @@ class Attribute extends AppModel
             $params['page'] = 1;
         }
         if (empty($exportTool->mock_query_only)) {
-            $this->__iteratedFetch($user, $params, $loop, $tmpfile, $exportTool, $exportToolParams, $elementCounter);
+            $elementCounter = $this->__iteratedFetch($user, $params, $loop, $tmpfile, $exportTool, $exportToolParams);
         }
         $tmpfile->write($exportTool->footer($exportToolParams));
         return $tmpfile->finish();
     }
 
-    private function __iteratedFetch($user, &$params, &$loop, TmpFileTool $tmpfile, $exportTool, $exportToolParams, &$elementCounter = 0)
+    /**
+     * @param array $user
+     * @param array $params
+     * @param bool $loop If true, data are fetched in loop to keep memory usage low
+     * @param TmpFileTool $tmpfile
+     * @param $exportTool
+     * @param array $exportToolParams
+     * @return int
+     * @throws Exception
+     */
+    private function __iteratedFetch(array $user, array $params, $loop, TmpFileTool $tmpfile, $exportTool, array $exportToolParams)
     {
         $this->Allowedlist = ClassRegistry::init('Allowedlist');
+        $separator = $exportTool->separator($exportToolParams);
+        $elementCounter = 0;
         $continue = true;
-        while ($continue) {
+        do {
             $results = $this->fetchAttributes($user, $params, $continue);
+            if (empty($results)) {
+                break; // nothing found, skip rest
+            }
             if ($params['includeSightingdb']) {
                 $this->Sightingdb = ClassRegistry::init('Sightingdb');
                 $results = $this->Sightingdb->attachToAttributes($results, $user);
             }
-            $params['page'] += 1;
             $results = $this->Allowedlist->removeAllowedlistedFromArray($results, true);
-            $i = 0;
-            $temp = '';
+            $elementCounter += count($results);
             foreach ($results as $attribute) {
-                $elementCounter++;
                 $handlerResult = $exportTool->handler($attribute, $exportToolParams);
-                $temp .= $handlerResult;
                 if ($handlerResult !== '') {
-                    if ($i != count($results) -1) {
-                        $temp .= $exportTool->separator($exportToolParams);
-                    }
+                    $tmpfile->writeWithSeparator($handlerResult, $separator);
                 }
-                $i++;
             }
-            if (!$loop) {
-                $continue = false;
-            }
-            if ($continue) {
-                $temp .= $exportTool->separator($exportToolParams);
-            }
-            $tmpfile->write($temp);
-        }
-        return true;
+            $params['page'] += 1;
+        } while ($loop && $continue);
+
+        return $elementCounter;
     }
 
     public function set_filter_uuid(&$params, $conditions, $options)
@@ -4743,6 +4761,44 @@ class Attribute extends AppModel
      */
     private function isPortValid($value)
     {
-        return is_numeric($value) && $value >= 1 && $value <= 65535;
+        return $this->isPositiveInteger($value) && $value >= 1 && $value <= 65535;
+    }
+
+    /**
+     * @param string $type
+     * @param string $value
+     * @return bool
+     */
+    private function isHashValid($type, $value)
+    {
+        if (!isset($this->__hexHashLengths[$type])) {
+            throw new InvalidArgumentException("Invalid hash type '$type'.");
+        }
+        $length = $this->__hexHashLengths[$type];
+        return strlen($value) === $length && ctype_xdigit($value);
+    }
+
+    /**
+     * Returns true if input value is positive integer or zero.
+     * @param int|string $value
+     * @return bool
+     */
+    private function isPositiveInteger($value)
+    {
+        return (is_int($value) && $value >= 0) || ctype_digit($value);
+    }
+
+    public function typeToCategoryMapping()
+    {
+        $typeCategoryMapping = array();
+        foreach ($this->categoryDefinitions as $k => $cat) {
+            foreach ($cat['types'] as $type) {
+                $typeCategoryMapping[$type][$k] = $k;
+            }
+        }
+        foreach ($typeCategoryMapping as $k => $v) {
+            $typeCategoryMapping[$k] = array_values($v);
+        }
+        return $typeCategoryMapping;
     }
 }

app/Model/AttributeTag.php

@@ -2,7 +2,7 @@
 App::uses('AppModel', 'Model');
 
 /**
- * @property Attribute $Attribute
+ * @property Tag $Tag
  */
 class AttributeTag extends AppModel
 {
@@ -180,30 +180,11 @@ class AttributeTag extends AppModel
         }
     }
 
-    /**
-     * Count number of not deleted attributes that contains given tag for given user. Tag must contains 'AttributeTag'.
-     *
-     * @param array $tag
-     * @param array $user
-     * @return int
-     */
-    public function countForTag(array $tag, array $user)
+    public function countForTag($tag_id, $user)
     {
-        $attributeIds = [];
-        foreach ($tag['AttributeTag'] as $attributeTag) {
-            $attributeIds[] = $attributeTag['attribute_id'];
-        }
-
-        if (empty($attributeIds)) {
-            return 0;
-        }
-
-        $conditions = $this->Attribute->buildConditions($user);
-        $conditions['Attribute.id'] = $attributeIds;
-        $conditions['Attribute.deleted'] = 0;
-        return $this->Attribute->find('count', array(
-            'recursive' => 0,
-            'conditions' => $conditions,
+        return $this->find('count', array(
+            'recursive' => -1,
+            'conditions' => array('AttributeTag.tag_id' => $tag_id)
         ));
     }
 

app/Model/DecayingModelsFormulas/Base.php

@@ -106,7 +106,7 @@ abstract class DecayingModelBase
         $flag_contain_matching_taxonomy = false;
         if (!empty($taxonomy_effective_ratios)) {
             foreach ($tags as $k => $tag) {
-                $taxonomy = explode('=', $tag['Tag']['name'])[0];
+                $taxonomy = $this->__extractTagBasename($tag['Tag']['name'])['base'];
                 if (isset($taxonomy_effective_ratios[$taxonomy])) {
                     $flag_contain_matching_taxonomy = true;
                     $base_score += $taxonomy_effective_ratios[$taxonomy] * $tag['Tag']['numerical_value'];

app/Model/Dns.php

@@ -1,41 +0,0 @@
-<?php
-
-App::uses('AppModel', 'Model');
-
-/*
- * Domain Name System related
- */
-class Dns extends AppModel
-{
-    public $useTable = false;
-
-    // Checks for a valid internet name
-    // Returns true if the name is an existing domain name, false otherwise
-    public function testipaddress($nametotest)
-    {
-        if (intval($nametotest) > 0) {
-            return true;
-        } else {
-            $ipaddress = gethostbyname($nametotest);
-            if ($ipaddress == $nametotest) {
-                return false;
-            } else {
-                return true;
-            }
-        }
-    }
-
-    // Name to IP list
-    // get all IP addresses of a certain domain name via DNS.
-    public function nametoipl($name = '')
-    {
-        if ('true' == Configure::read('MISP.dns')) {
-            if (!$ips = gethostbynamel($name)) {
-                $ips = array();
-            }
-        } else {
-            $ips = array();
-        }
-        return $ips;
-    }
-}

app/Model/EventBlocklist.php

@@ -1,5 +1,6 @@
 <?php
 App::uses('AppModel', 'Model');
+
 class EventBlocklist extends AppModel
 {
     public $useTable = 'event_blocklists';
@@ -38,13 +39,25 @@ class EventBlocklist extends AppModel
         if (!isset($schema['event_info'])) {
             $this->updateDatabase('addEventBlocklistsContext');
         }
-        $date = date('Y-m-d H:i:s');
         if (empty($this->data['EventBlocklist']['id'])) {
-            $this->data['EventBlocklist']['date_created'] = $date;
+            $this->data['EventBlocklist']['date_created'] = date('Y-m-d H:i:s');
         }
         if (empty($this->data['EventBlocklist']['comment'])) {
             $this->data['EventBlocklist']['comment'] = '';
         }
         return true;
     }
+
+    /**
+     * @param string $eventUuid
+     * @return bool
+     */
+    public function isBlocked($eventUuid)
+    {
+        $result = $this->find('first', [
+            'conditions' => ['event_uuid' => $eventUuid],
+            'fields' => ['id']
+        ]);
+        return !empty($result);
+    }
 }

app/Model/EventReport.php

@@ -0,0 +1,899 @@
+<?php
+App::uses('AppModel', 'Model');
+
+/**
+ * @property Event $Event
+ */
+class EventReport extends AppModel
+{
+    public $actsAs = array(
+        'Containable',
+        'SysLogLogable.SysLogLogable' => array(
+            'userModel' => 'User',
+            'userKey' => 'user_id',
+            'change' => 'full'
+        ),
+        'Regexp' => array('fields' => array('value')),
+    );
+
+    public $validate = array(
+        'event_id' => array(
+            'numeric' => array(
+                'rule' => array('numeric')
+            )
+        ),
+        'uuid' => array(
+            'uuid' => array(
+                'rule' => 'uuid',
+                'message' => 'Please provide a valid RFC 4122 UUID'
+            ),
+            'unique' => array(
+                'rule' => 'isUnique',
+                'message' => 'The UUID provided is not unique',
+                'required' => 'create'
+            )
+        ),
+        'distribution' => array(
+            'rule' => array('inList', array('0', '1', '2', '3', '4', '5')),
+            'message' => 'Options: Your organisation only, This community only, Connected communities, All communities, Sharing group, Inherit event',
+            'required' => true
+        )
+    );
+
+    public $captureFields = array('uuid', 'name', 'content', 'distribution', 'sharing_group_id', 'timestamp', 'deleted', 'event_id');
+    public $defaultContain = array(
+        'SharingGroup' => array('fields' => array('id', 'name', 'uuid')),
+        'Event' => array(
+            'fields' =>  array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.info', 'Event.user_id', 'Event.date'),
+            'Orgc' => array('fields' => array('Orgc.id', 'Orgc.name')),
+            'Org' => array('fields' => array('Org.id', 'Org.name'))
+        )
+    );
+
+    public $belongsTo = array(
+        'Event' => array(
+            'className' => 'Event',
+            'foreignKey' => 'event_id'
+        ),
+        'SharingGroup' => array(
+            'className' => 'SharingGroup',
+            'foreignKey' => 'sharing_group_id'
+        ),
+    );
+
+    public function beforeValidate($options = array())
+    {
+        parent::beforeValidate();
+        // generate UUID if it doesn't exist
+        if (empty($this->data['EventReport']['uuid'])) {
+            $this->data['EventReport']['uuid'] = CakeText::uuid();
+        } else {
+            $this->data['EventReport']['uuid'] = strtolower($this->data['EventReport']['uuid']);
+        }
+        // generate timestamp if it doesn't exist
+        if (empty($this->data['EventReport']['timestamp'])) {
+            $date = new DateTime();
+            $this->data['EventReport']['timestamp'] = $date->getTimestamp();
+        }
+        if ($this->data['EventReport']['distribution'] != 4) {
+            $this->data['EventReport']['sharing_group_id'] = 0;
+        }
+        // Set defaults for when some of the mandatory fields don't have defaults
+        // These fields all have sane defaults either based on another field, or due to server settings
+        if (!isset($this->data['EventReport']['distribution'])) {
+            $this->data['EventReport']['distribution'] = Configure::read('MISP.default_attribute_distribution');
+            if ($this->data['EventReport']['distribution'] == 'event') {
+                $this->data['EventReport']['distribution'] = 5;
+            }
+        }
+        return true;
+    }
+    /**
+     * captureReport Gets a report then save it
+     *
+     * @param  array $user
+     * @param  array $report
+     * @param  int|string $eventId
+     * @return array Any errors preventing the capture
+     */
+    public function captureReport(array $user, array $report, $eventId)
+    {
+        $this->Log = ClassRegistry::init('Log');
+        if (!isset($report['EventReport'])) {
+            $report = ['EventReport' => $report];
+        }
+        $report['EventReport']['event_id'] = $eventId;
+        $report = $this->captureSG($user, $report);
+        $this->create();
+        $errors = $this->saveAndReturnErrors($report, ['fieldList' => $this->captureFields]);
+        if (!empty($errors)) {
+            $this->Log->createLogEntry($user, 'add', 'EventReport', 0,
+                __('Event Report dropped due to validation for Event report %s failed: %s', $report['EventReport']['uuid'], ' failed: ' . $report['EventReport']['name']),
+                __('Validation errors: %s.%sFull report: %s', json_encode($errors), PHP_EOL, json_encode($report['EventReport']))
+            );
+        }
+        return $errors;
+    }
+    
+    /**
+     * addReport Add a report
+     *
+     * @param  array $user
+     * @param  array $report
+     * @param  int|string $eventId
+     * @return array Any errors preventing the addition
+     */
+    public function addReport(array $user, array $report, $eventId)
+    {
+        $errors = $this->captureReport($user, $report, $eventId);
+        if (empty($errors)) {
+            $this->Event->unpublishEvent($eventId);
+        }
+        return $errors;
+    }
+    
+    /**
+     * editReport Edit a report
+     *
+     * @param  array $user
+     * @param  array $report
+     * @param  int|string $eventId
+     * @param  bool  $fromPull
+     * @param  bool  $nothingToChange
+     * @return array Any errors preventing the edition
+     */
+    public function editReport(array $user, array $report, $eventId, $fromPull = false, &$nothingToChange = false)
+    {
+        $errors = array();
+        if (!isset($report['EventReport']['uuid'])) {
+            $errors[] = __('Event Report doesn\'t have an UUID');
+            return $errors;
+        }
+        $report['EventReport']['event_id'] = $eventId;
+        $existingReport = $this->find('first', array(
+            'conditions' => array('EventReport.uuid' => $report['EventReport']['uuid']),
+            'recursive' => -1,
+        ));
+        if (empty($existingReport)) {
+            if ($fromPull) {
+                return $this->captureReport($user, $report, $eventId);
+            } else {
+                $errors[] = __('Event Report not found.');
+                return $errors;
+            }
+        }
+
+        if ($fromPull) {
+            if (isset($report['EventReport']['timestamp'])) {
+                if ($report['EventReport']['timestamp'] <= $existingReport['EventReport']['timestamp']) {
+                    $nothingToChange = true;
+                    return array();
+                }
+            }
+        } else {
+            unset($report['EventReport']['timestamp']);
+        }
+        $errors = $this->saveAndReturnErrors($report, ['fieldList' => $this->captureFields], $errors);
+        if (empty($errors)) {
+            $this->Event->unpublishEvent($eventId);
+        }
+        return $errors;
+    }
+
+    /**
+     * deleteReport ACL-aware method to delete the report.
+     *
+     * @param  array $user
+     * @param  int|string $id
+     * @param  bool $hard
+     * @return array Any errors preventing the deletion
+     */
+    public function deleteReport(array $user, $report, $hard=false)
+    {
+        $report = $this->fetchIfAuthorized($user, $report, 'delete', $throwErrors=true, $full=false);
+        $errors = [];
+        if ($hard) {
+            $deleted = $this->delete($report['EventReport']['id'], true);
+            if (!$deleted) {
+                $errors[] = __('Failed to delete report');
+            }
+        } else {
+            $report['EventReport']['deleted'] = true;
+            $errors = $this->saveAndReturnErrors($report, ['fieldList' => ['deleted']]);
+        }
+        if (empty($errors)) {
+            $this->Event->unpublishEvent($report['EventReport']['event_id']);
+        }
+        return $errors;
+    }
+    
+    /**
+     * restoreReport ACL-aware method to restore a report.
+     *
+     * @param  array $user
+     * @param  int|string $id
+     * @return array Any errors preventing the restoration
+     */
+    public function restoreReport(array $user, $id)
+    {
+        $report = $this->fetchIfAuthorized($user, $id, 'edit', $throwErrors=true, $full=false);
+        $report['EventReport']['deleted'] = false;
+        $errors = $this->saveAndReturnErrors($report, ['fieldList' => ['deleted']]);
+        if (empty($errors)) {
+            $this->Event->unpublishEvent($report['EventReport']['event_id']);
+        }
+        return $errors;
+    }
+
+    private function captureSG(array $user, array $report)
+    {
+        $this->Event = ClassRegistry::init('Event');
+        if (isset($report['EventReport']['distribution']) && $report['EventReport']['distribution'] == 4) {
+            $report['EventReport'] = $this->Event->__captureSGForElement($report['EventReport'], $user);
+        }
+        return $report;
+    }
+    
+    /**
+     * buildACLConditions Generate ACL conditions for viewing the report
+     *
+     * @param  array $user
+     * @return array
+     */
+    public function buildACLConditions(array $user)
+    {
+        $this->Event = ClassRegistry::init('Event');
+        $conditions = array();
+        if (!$user['Role']['perm_site_admin']) {
+            $sgids = $this->Event->cacheSgids($user, true);
+            $eventConditions = $this->Event->createEventConditions($user);
+            $conditions = array(
+                'AND' => array(
+                    $eventConditions['AND'],
+                    array(
+                        'OR' => array(
+                            'Event.org_id' => $user['org_id'],
+                            'EventReport.distribution' => array('1', '2', '3', '5'),
+                            'AND '=> array(
+                                'EventReport.distribution' => 4,
+                                'EventReport.sharing_group_id' => $sgids,
+                            )
+                        )
+                    )
+                )
+            );
+        }
+        return $conditions;
+    }
+
+    /**
+     * fetchById Simple ACL-aware method to fetch a report by Id or UUID
+     *
+     * @param  array $user
+     * @param  int|string $reportId
+     * @param  bool  $throwErrors
+     * @param  bool  $full
+     * @return array
+     */
+    public function simpleFetchById(array $user, $reportId, $throwErrors=true, $full=false)
+    {
+        if (is_numeric($reportId)) {
+            $options = array('conditions' => array("EventReport.id" => $reportId));
+        } elseif (Validation::uuid($reportId)) {
+            $options = array('conditions' => array("EventReport.uuid" => $reportId));
+        } else {
+            if ($throwErrors) {
+                throw new NotFoundException(__('Invalid report'));
+            }
+            return array();
+        }
+
+        $report = $this->fetchReports($user, $options, $full);
+        if (!empty($report)) {
+            return $report[0];
+        }
+        if ($throwErrors) {
+            throw new NotFoundException(__('Invalid report'));
+        }
+        return array();
+    }
+    
+    /**
+     * fetchReports ACL-aware method. Basically find with ACL
+     *
+     * @param  array $user
+     * @param  array $options
+     * @param  bool  $full
+     * @return array
+     */
+    public function fetchReports(array $user, array $options = array(), $full=false)
+    {
+        $params = array(
+            'conditions' => $this->buildACLConditions($user),
+            'contain' => $this->defaultContain,
+            'recursive' => -1
+        );
+        if ($full) {
+            $params['recursive'] = 1;
+        }
+        if (isset($options['fields'])) {
+            $params['fields'] = $options['fields'];
+        }
+        if (isset($options['conditions'])) {
+            $params['conditions']['AND'][] = $options['conditions'];
+        }
+        if (isset($options['group'])) {
+            $params['group'] = empty($options['group']) ? $options['group'] : false;
+        }
+        $reports = $this->find('all', $params);
+        return $reports;
+    }
+
+    /**
+     * fetchIfAuthorized Fetches a report and checks if the user has the authorization to perform the requested operation
+     *
+     * @param  array $user
+     * @param  int|string|array $report
+     * @param  mixed $authorizations the requested actions to be performed on the report
+     * @param  bool  $throwErrors Should the function throws excpetion if users is not allowed to perform the action
+     * @param  bool  $full
+     * @return array The report or an error message
+     */
+    public function fetchIfAuthorized(array $user, $report, $authorizations, $throwErrors=true, $full=false)
+    {
+        $authorizations = is_array($authorizations) ? $authorizations : array($authorizations);
+        $possibleAuthorizations = array('view', 'edit', 'delete');
+        if (!empty(array_diff($authorizations, $possibleAuthorizations))) {
+            throw new NotFoundException(__('Invalid authorization requested'));
+        }
+        if (isset($report['uuid'])) {
+            $report['EventReport'] = $report;
+        }
+        if (!isset($report['EventReport']['uuid'])) {
+            $report = $this->simpleFetchById($user, $report, $throwErrors=$throwErrors, $full=$full);
+            if (empty($report)) {
+                $message = __('Invalid report');
+                return array('authorized' => false, 'error' => $message);
+            }
+        }
+        if ($user['Role']['perm_site_admin']) {
+            return $report;
+        }
+
+        if (in_array('view', $authorizations) && count($authorizations) == 1) {
+            return $report;
+        } else {
+            if (in_array('edit', $authorizations) || in_array('delete', $authorizations)) {
+                $checkResult = $this->canEditReport($user, $report);
+                if ($checkResult !== true) {
+                    if ($throwErrors) {
+                        throw new UnauthorizedException($checkResult);
+                    }
+                    return array('authorized' => false, 'error' => $checkResult);
+                }
+            }
+            return $report;
+        }
+    }
+
+    public function canEditReport(array $user, array $report)
+    {
+        if ($user['Role']['perm_site_admin']) {
+            return true;
+        }
+        if (empty($report['Event'])) {
+            return __('Could not find associated event');
+        }
+        if ($report['Event']['orgc_id'] != $user['org_id']) {
+            return __('Only the creator organisation of the event can modify the report');
+        }
+        return true;
+    }
+    
+    public function reArrangeReport(array $report)
+    {
+        $rearrangeObjects = array('Event', 'SharingGroup');
+        if (isset($report['EventReport'])) {
+            foreach ($rearrangeObjects as $ro) {
+                if (isset($report[$ro]) && !is_null($report[$ro]['id'])) {
+                    $report['EventReport'][$ro] = $report[$ro];
+                }
+                unset($report[$ro]);
+            }
+        }
+        return $report;
+    }
+
+    /**
+     * getProxyMISPElements Extract MISP Elements from an event and make them accessible by their UUID
+     *
+     * @param array $user
+     * @param int|string $eventid
+     * @return array
+     * @throws Exception
+     */
+    public function getProxyMISPElements(array $user, $eventid)
+    {
+        $options = [
+            'noSightings' => true,
+            'sgReferenceOnly' => true,
+            'noEventReports' => true,
+            'noShadowAttributes' => true,
+        ];
+
+        $event = $this->Event->fetchEvent($user, array_merge(['eventid' => $eventid], $options));
+        if (empty($event)) {
+            throw new NotFoundException(__('Invalid Event'));
+        }
+        $event = $event[0];
+
+        if (!empty($event['Event']['extends_uuid'])) {
+            $extendedParentEvent = $this->Event->fetchEvent($user, array_merge([
+                'event_uuid' => $event['Event']['extends_uuid'],
+                'extended' => true,
+            ], $options));
+            if (!empty($extendedParentEvent)) {
+                $event = $extendedParentEvent[0];
+            }
+        }
+
+        $allTagNames = [];
+        foreach ($event['EventTag'] as $eventTag) {
+            // include just tags that belongs to requested event or its parent, not to other child
+            if ($eventTag['event_id'] == $eventid || $eventTag['event_id'] == $event['Event']['id']) {
+                $allTagNames[$eventTag['Tag']['name']] = $eventTag['Tag'];
+            }
+        }
+
+        $attributes = [];
+        foreach ($event['Attribute'] as $attribute) {
+            unset($attribute['ShadowAttribute']);
+            foreach ($attribute['AttributeTag'] as $at) {
+                $allTagNames[$at['Tag']['name']] = $at['Tag'];
+            }
+            $this->Event->Attribute->removeGalaxyClusterTags($attribute);
+            $attributes[$attribute['uuid']] = $attribute;
+        }
+
+        $objects = [];
+        $templateConditions = [];
+        foreach ($event['Object'] as $k => $object) {
+            foreach ($object['Attribute'] as &$objectAttribute) {
+                unset($objectAttribute['ShadowAttribute']);
+                $objectAttribute['object_uuid'] = $object['uuid'];
+                $attributes[$objectAttribute['uuid']] = $objectAttribute;
+
+                foreach ($objectAttribute['AttributeTag'] as $at) {
+                    $allTagNames[$at['Tag']['name']] = $at['Tag'];
+                }
+                $this->Event->Attribute->removeGalaxyClusterTags($objectAttribute);
+            }
+            $objects[$object['uuid']] = $object;
+
+            $uniqueCondition = "{$object['template_uuid']}.{$object['template_version']}";
+            if (!isset($templateConditions[$uniqueCondition])) {
+                $templateConditions[$uniqueCondition]['AND'] = [
+                    'ObjectTemplate.uuid' => $object['template_uuid'],
+                    'ObjectTemplate.version' => $object['template_version']
+                ];
+            }
+        }
+        if (!empty($templateConditions)) {
+            // Fetch object templates for event objects
+            $this->ObjectTemplate = ClassRegistry::init('ObjectTemplate');
+            $templates = $this->ObjectTemplate->find('all', array(
+                'conditions' => ['OR' => array_values($templateConditions)],
+                'recursive' => -1,
+                'contain' => array(
+                    'ObjectTemplateElement' => [
+                        'order' => ['ui-priority' => 'DESC'],
+                        'fields' => ['object_relation', 'type', 'ui-priority']
+                    ]
+                )
+            ));
+            $objectTemplates = [];
+            foreach ($templates as $template) {
+                $objectTemplates["{$template['ObjectTemplate']['uuid']}.{$template['ObjectTemplate']['version']}"] = $template;
+            }
+        } else {
+            $objectTemplates = [];
+        }
+        $this->Galaxy = ClassRegistry::init('Galaxy');
+        $allowedGalaxies = $this->Galaxy->getAllowedMatrixGalaxies();
+        $allowedGalaxies = Hash::combine($allowedGalaxies, '{n}.Galaxy.uuid', '{n}.Galaxy');
+        return [
+            'attribute' => $attributes,
+            'object' => $objects,
+            'objectTemplates' => $objectTemplates,
+            'galaxymatrix' => $allowedGalaxies,
+            'tagname' => $allTagNames
+        ];
+    }
+
+    private function saveAndReturnErrors($data, $saveOptions = [], $errors = [])
+    {
+        $saveSuccess = $this->save($data, $saveOptions);
+        if (!$saveSuccess) {
+            foreach ($this->validationErrors as $validationError) {
+                $errors[] = $validationError[0];
+            }
+        }
+        return $errors;
+    }
+
+    public function applySuggestions($user, $report, $contentWithSuggestions, $suggestionsMapping) {
+        $errors = [];
+        $replacedContent = $contentWithSuggestions;
+        $success = 0;
+        foreach ($suggestionsMapping as $value => $suggestedAttribute) {
+            $suggestedAttribute['value'] = $value;
+            $savedAttribute = $this->createAttributeFromSuggestion($user, $report, $suggestedAttribute);
+            if (empty($savedAttribute['errors'])) {
+                $success++;
+                $replacedContent = $this->applySuggestionsInText($replacedContent, $savedAttribute['attribute'], $value);
+            } else {
+                $replacedContent = $this->revertToOriginalInText($replacedContent, $value);
+                $errors[] = $savedAttribute['errors'];
+            }
+        }
+        if ($success > 0 || count($suggestionsMapping) == 0) {
+            $report['EventReport']['content'] = $replacedContent;
+            $editErrors = $this->editReport($user, $report, $report['EventReport']['event_id']);
+            if (!empty($editErrors)) {
+                $errors[] = $editErrors;
+            }
+        }
+        return $errors;
+    }
+    
+    public function applySuggestionsInText($contentWithSuggestions, $attribute, $value)
+    {
+        $textToBeReplaced = sprintf('@[suggestion](%s)', $value);
+        $textToInject = sprintf('@[attribute](%s)', $attribute['Attribute']['uuid']);
+        $replacedContent = str_replace($textToBeReplaced, $textToInject, $contentWithSuggestions);
+        return $replacedContent;
+    }
+
+    public function revertToOriginalInText($contentWithSuggestions, $value)
+    {
+        $textToBeReplaced = sprintf('@[suggestion](%s)', $value);
+        $textToInject = $value;
+        $replacedContent = str_replace($textToBeReplaced, $textToInject, $contentWithSuggestions);
+        return $replacedContent;
+    }
+
+    private function createAttributeFromSuggestion($user, $report, $suggestedAttribute)
+    {
+        $errors = [];
+        $attribute = [
+            'event_id' => $report['EventReport']['event_id'],
+            'distribution' => 5,
+            'category' => $suggestedAttribute['category'],
+            'type' => $suggestedAttribute['type'],
+            'value' => $suggestedAttribute['value'],
+            'to_ids' => $suggestedAttribute['to_ids'],
+        ];
+        $validationErrors = array();
+        $this->Event->Attribute->captureAttribute($attribute, $report['EventReport']['event_id'], $user, false, false, false, $validationErrors);
+        $savedAttribute = false;
+        if (!empty($validationErrors)) {
+            $errors = $validationErrors;
+        } else {
+            $savedAttribute = $this->Event->Attribute->find('first', array(
+                'recursive' => -1,
+                'conditions' => array('Attribute.id' => $this->Event->Attribute->id),
+            ));
+        }
+        return [
+            'errors' => $errors,
+            'attribute' => $savedAttribute
+        ];
+    }
+    
+    /**
+     * transformFreeTextIntoReplacement
+     *
+     * @param  array $user
+     * @param  array $report
+     * @param  array $complexTypeToolResult Uses the complex type tool output to support import regex replacements.
+     *                                      Another solution would be to run the regex replacement on each token of the report which is too heavy
+     * @return array
+     */
+    public function transformFreeTextIntoReplacement(array $user, array $report, array $complexTypeToolResult)
+    {
+        $complexTypeToolResultWithImportRegex = $this->injectImportRegexOnComplexTypeToolResult($complexTypeToolResult);
+        $valueToValueWithRegex = Hash::combine($complexTypeToolResultWithImportRegex, '{n}.valueWithImportRegex', '{n}.value');
+        $proxyElements = $this->getProxyMISPElements($user, $report['EventReport']['event_id']);
+        $originalContent = $report['EventReport']['content'];
+        $content = $originalContent;
+        $replacedValues = [];
+        foreach ($proxyElements['attribute'] as $uuid => $attribute) {
+            $count = 0;
+            $textToInject = sprintf('@[attribute](%s)', $uuid);
+            $content = str_replace($attribute['value'], $textToInject, $content, $count);
+            if ($count > 0 || strpos($originalContent, $attribute['value'])) { // Check if the value has been replaced by the first match
+                if (!isset($replacedValues[$attribute['value']])) {
+                    $replacedValues[$attribute['value']] = [
+                        'attributeUUIDs' => [$uuid],
+                        'valueInReport' => $attribute['value'],
+                    ];
+                } else {
+                    $replacedValues[$attribute['value']]['attributeUUIDs'][] = $uuid;
+                }
+                $count = 0;
+            }
+            if (isset($valueToValueWithRegex[$attribute['value']]) && $valueToValueWithRegex[$attribute['value']] != $attribute['value']) {
+                $content = str_replace($valueToValueWithRegex[$attribute['value']], $textToInject, $content, $count);
+                if ($count > 0 || strpos($originalContent, $valueToValueWithRegex[$attribute['value']])) {
+                    if (!isset($replacedValues[$attribute['value']])) {
+                        $replacedValues[$attribute['value']] = [
+                            'attributeUUIDs' => [$uuid],
+                            'valueInReport' => $valueToValueWithRegex[$attribute['value']],
+                        ];
+                    } else {
+                        $replacedValues[$attribute['value']]['attributeUUIDs'][] = $uuid;
+                    }
+                }
+            }
+        }
+        return [
+            'contentWithReplacements' => $content,
+            'replacedValues' => $replacedValues
+        ];
+    }
+
+    public function transformFreeTextIntoSuggestion($content, $complexTypeToolResult)
+    {
+        $replacedContent = $content;
+        $suggestionsMapping = [];
+        $typeToCategoryMapping = $this->Event->Attribute->typeToCategoryMapping();
+        foreach ($complexTypeToolResult as $i => $complexTypeToolEntry) {
+            $textToBeReplaced = $complexTypeToolEntry['value'];
+            $textToInject = sprintf('@[suggestion](%s)', $textToBeReplaced);
+            $suggestionsMapping[$textToBeReplaced] = [
+                'category' => $typeToCategoryMapping[$complexTypeToolEntry['default_type']][0],
+                'type' => $complexTypeToolEntry['default_type'],
+                'value' => $textToBeReplaced,
+                'to_ids' => $complexTypeToolEntry['to_ids'],
+            ];
+            $replacedContent = str_replace($textToBeReplaced, $textToInject, $replacedContent);
+        }
+        return [
+            'contentWithSuggestions' => $replacedContent,
+            'suggestionsMapping' => $suggestionsMapping
+        ];
+    }
+
+    public function injectImportRegexOnComplexTypeToolResult($complexTypeToolResult) {
+        foreach ($complexTypeToolResult as $i => $complexTypeToolEntry) {
+            $transformedValue = $this->runRegexp($complexTypeToolEntry['default_type'], $complexTypeToolEntry['value']);
+            if ($transformedValue !== false) {
+                $complexTypeToolResult[$i]['valueWithImportRegex'] = $transformedValue;
+            }
+        }
+        return $complexTypeToolResult;
+    }
+
+    public function getComplexTypeToolResultFromReport($content)
+    {
+        App::uses('ComplexTypeTool', 'Tools');
+        $complexTypeTool = new ComplexTypeTool();
+        $this->Warninglist = ClassRegistry::init('Warninglist');
+        $complexTypeTool->setTLDs($this->Warninglist->fetchTLDLists());
+        $complexTypeToolResult = $complexTypeTool->checkComplexRouter($content, 'freetext');
+        return $complexTypeToolResult;
+    }
+
+    public function getComplexTypeToolResultWithReplacements($user, $report)
+    {
+        $complexTypeToolResult = $this->getComplexTypeToolResultFromReport($report['EventReport']['content']);
+        $replacementResult = $this->transformFreeTextIntoReplacement($user, $report, $complexTypeToolResult);
+        $complexTypeToolResult = $this->getComplexTypeToolResultFromReport($replacementResult['contentWithReplacements']);
+        return [
+            'complexTypeToolResult' => $complexTypeToolResult,
+            'replacementResult' => $replacementResult,
+        ];
+    }
+    
+    /**
+     * extractWithReplacements Extract context information from report with special care for ATT&CK
+     *
+     * @param  array $user
+     * @param  array $report
+     * @return array
+     */
+    public function extractWithReplacements(array $user, array $report, array $options = [])
+    {
+        $baseOptions = [
+            'replace' => false,
+            'tags' => true,
+            'synonyms' => true,
+            'synonyms_min_characters' => 4,
+            'prune_deprecated' => true,
+            'attack' => true,
+        ];
+        $options = array_merge($baseOptions, $options);
+        $originalContent = $report['EventReport']['content'];
+        $this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
+        $mitreAttackGalaxyId = $this->GalaxyCluster->Galaxy->getMitreAttackGalaxyId();
+        $clusterContain = ['Tag'];
+        $replacedContext = [];
+
+        if ($options['prune_deprecated']) {
+            $clusterContain['Galaxy'] = ['conditions' => ['Galaxy.namespace !=' => 'deprecated']];
+        }
+        if ($options['synonyms']) {
+            $clusterContain['GalaxyElement'] = ['conditions' => ['GalaxyElement.key' => 'synonyms']];
+        }
+        $clusterConditions = [];
+        if ($options['attack']) {
+            $clusterConditions = ['GalaxyCluster.galaxy_id !=' => $mitreAttackGalaxyId];
+        }
+        $clusters = $this->GalaxyCluster->find('all', [
+            'conditions' => $clusterConditions,
+            'contain' => $clusterContain
+        ]);
+
+        if ($options['tags']) {
+            $this->Tag = ClassRegistry::init('Tag');
+            $tags = $this->Tag->fetchUsableTags($user);
+            foreach ($tags as $i => $tag) {
+                $tagName = $tag['Tag']['name'];
+                $found = $this->isValidReplacementTag($originalContent, $tagName);
+                if ($found) {
+                    $replacedContext[$tagName][$tagName] = $tag['Tag'];
+                } else {
+                    $tagNameUpper = strtoupper($tagName);
+                    $found = $this->isValidReplacementTag($originalContent, $tagNameUpper);
+                    if ($found) {
+                        $replacedContext[$tagNameUpper][$tagName] = $tag['Tag'];
+                    }
+                }
+            }
+        }
+
+        foreach ($clusters as $i => $cluster) {
+            $cluster['GalaxyCluster']['colour'] = '#0088cc';
+            $tagName = $cluster['GalaxyCluster']['tag_name'];
+            $found = $this->isValidReplacementTag($originalContent, $tagName);
+            if ($found) {
+                $replacedContext[$tagName][$tagName] = $cluster['GalaxyCluster'];
+            }
+            $toSearch = ' ' . $cluster['GalaxyCluster']['value'] . ' ';
+            $found = strpos($originalContent, $toSearch) !== false;
+            if ($found) {
+                $replacedContext[$cluster['GalaxyCluster']['value']][$tagName] = $cluster['GalaxyCluster'];
+            }
+            if ($options['synonyms']) {
+                foreach ($cluster['GalaxyElement'] as $j => $element) {
+                    if (strlen($element['value']) >= $options['synonyms_min_characters']) {
+                        $toSearch = ' ' . $element['value'] . ' ';
+                        $found = strpos($originalContent, $toSearch) !== false;
+                        if ($found) {
+                            $replacedContext[$element['value']][$tagName] = $cluster['GalaxyCluster'];
+                        }
+                    }
+                }
+            }
+        }
+
+        if ($options['attack']) {
+            unset($clusterContain['Galaxy']);
+            $attackClusters = $this->GalaxyCluster->find('all', [
+                'conditions' => ['GalaxyCluster.galaxy_id' => $mitreAttackGalaxyId],
+                'contain' => $clusterContain
+            ]);
+            foreach ($attackClusters as $i => $cluster) {
+                $cluster['GalaxyCluster']['colour'] = '#0088cc';
+                $tagName = $cluster['GalaxyCluster']['tag_name'];
+                $toSearch = ' ' . $cluster['GalaxyCluster']['value'] . ' ';
+                $found = strpos($originalContent, $toSearch) !== false;
+                if ($found) {
+                    $replacedContext[$cluster['GalaxyCluster']['value']][$tagName] = $cluster['GalaxyCluster'];
+                } else {
+                    $clusterParts = explode(' - ', $cluster['GalaxyCluster']['value'], 2);
+                    $toSearch = ' ' . $clusterParts[0] . ' ';
+                    $found = strpos($originalContent, $toSearch) !== false;
+                    if ($found) {
+                        $replacedContext[$clusterParts[0]][$tagName] = $cluster['GalaxyCluster'];
+                    } else {
+                        $toSearch = ' ' . $clusterParts[1] . ' ';
+                        $found = strpos($originalContent, $toSearch) !== false;
+                        if ($found) {
+                            $replacedContext[$clusterParts[1]][$tagName] = $cluster['GalaxyCluster'];
+                        }
+                    }
+                }
+            }
+        }
+        $toReturn = [
+            'replacedContext' => $replacedContext
+        ];
+        if ($options['replace']) {
+            $content = $originalContent;
+            foreach ($replacedContext as $rawText => $replacements) {
+                // Replace with first one until a better strategy is found
+                reset($replacements);
+                $replacement = key($replacements);
+                $textToInject = sprintf('@[tag](%s)', $replacement);
+                $content = str_replace($rawText, $textToInject, $content);
+            }
+            $toReturn['contentWithReplacements'] = $content;
+        }
+        return $toReturn;
+    }
+
+    public function downloadMarkdownFromURL($event_id, $url)
+    {
+        $this->Module = ClassRegistry::init('Module');
+        $module = $this->isFetchURLModuleEnabled();
+        if (!is_array($module)) {
+            return false;
+        }
+        $modulePayload = [
+            'module' => $module['name'],
+            'event_id' => $event_id,
+            'url' => $url
+        ];
+        $module = $this->isFetchURLModuleEnabled();
+        if (!empty($module)) {
+            $result = $this->Module->queryModuleServer($modulePayload, false);
+            if (empty($result['results'][0]['values'][0])) {
+                return '';
+            }
+            return $result['results'][0]['values'][0];
+        }
+        return false;
+    }
+
+    public function isFetchURLModuleEnabled() {
+        $this->Module = ClassRegistry::init('Module');
+        $module = $this->Module->getEnabledModule('html_to_markdown', 'expansion');
+        return !empty($module) ? $module : false;
+    }
+
+    /**
+     * findValidReplacementTag Search if tagName is in content and is not wrapped in a tag reference
+     *
+     * @param  string $content
+     * @param  string $tagName
+     * @return bool
+     */
+    private function isValidReplacementTag($content, $tagName)
+    {
+        $lastIndex = 0;
+        $allIndices = [];
+        $toSearch = strpos($tagName, ':') === false ? ' ' . $tagName . ' ' : $tagName;
+        while (($lastIndex = strpos($content, $toSearch, $lastIndex)) !== false) {
+            $allIndices[] = $lastIndex;
+            $lastIndex = $lastIndex + strlen($toSearch);
+        }
+        if (empty($allIndices)) {
+            return false;
+        } else {
+            $wrapper = '@[tag](';
+            foreach ($allIndices as $i => $index) {
+                $stringBeforeTag = substr($content, $index - strlen($wrapper), strlen($wrapper));
+                if ($stringBeforeTag != $wrapper) {
+                    return true;
+                }
+            }
+            return false;
+        }
+    }
+
+    public function attachTagsAfterReplacements($user, $replacedContext, $eventId)
+    {
+        $this->EventTag = ClassRegistry::init('EventTag');
+        foreach ($replacedContext as $rawText => $tagNames) {
+            // Replace with first one until a better strategy is found
+            reset($tagNames);
+            $tagName = key($tagNames);
+            $tagId = $this->EventTag->Tag->lookupTagIdFromName($tagName);
+            if ($tagId === -1) {
+                $tagId = $this->EventTag->Tag->captureTag(['name' => $tagName], $user);
+            }
+            $this->EventTag->attachTagToEvent($eventId, $tagId);
+        }
+    }
+}

app/Model/EventTag.php

@@ -1,9 +1,6 @@
 <?php
 App::uses('AppModel', 'Model');
 
-/**
- * @property Event $Event
- */
 class EventTag extends AppModel
 {
     public $actsAs = array('Containable');
@@ -160,29 +157,11 @@ class EventTag extends AppModel
         return $tags;
     }
 
-    /**
-     * Count number of event that contains given tag for given user. Tag must contains 'EventTag'.
-     *
-     * @param array $tag
-     * @param array $user
-     * @return int
-     */
-    public function countForTag(array $tag, array $user)
+    public function countForTag($tag_id, $user)
     {
-        $eventIds = [];
-        foreach ($tag['EventTag'] as $eventTag) {
-            $eventIds[] = $eventTag['event_id'];
-        }
-
-        if (empty($eventIds)) {
-            return 0;
-        }
-
-        $conditions = $this->Event->createEventConditions($user);
-        $conditions['Event.id'] = $eventIds;
-        return $this->Event->find('count', array(
+        return $this->find('count', array(
             'recursive' => -1,
-            'conditions' => $conditions,
+            'conditions' => array('EventTag.tag_id' => $tag_id)
         ));
     }
 

app/Model/Feed.php

@@ -163,7 +163,7 @@ class Feed extends AppModel
     /**
      * @param array $feed
      * @param HttpSocket|null $HttpSocket Null can be for local feed
-     * @return Generator|array
+     * @return Generator
      * @throws Exception
      */
     public function getCache(array $feed, HttpSocket $HttpSocket = null)
@@ -180,10 +180,7 @@ class Feed extends AppModel
         $tmpFile->write(trim($data));
         unset($data);
 
-        foreach ($tmpFile->lines() as $line) {
-            yield explode(',', rtrim($line));
-        }
-        return;
+        return $tmpFile->csv();
     }
 
     /**
@@ -250,9 +247,7 @@ class Feed extends AppModel
             $data = $this->feedGetUri($feed, $feedUrl, $HttpSocket, true);
 
             if (!$isLocal) {
-                $redis = $this->setupRedisWithException();
-                $redis->del('misp:feed_cache:' . $feed['Feed']['id']);
-                file_put_contents($feedCache, $data);
+                file_put_contents($feedCache, $data); // save to cache
             }
         }
 
@@ -333,126 +328,155 @@ class Feed extends AppModel
     /**
      * Attach correlations from cached servers or feeds.
      *
-     * @param array $objects
+     * @param array $attributes
      * @param array $user
      * @param array $event
-     * @param bool $overrideLimit Override hardcoded limit for 10 000 attribute correlations.
+     * @param bool $overrideLimit Override hardcoded limit for 10 000 correlations.
      * @param string $scope `Feed` or `Server`
      * @return array
      */
-    public function attachFeedCorrelations($objects, $user, &$event, $overrideLimit = false, $scope = 'Feed')
+    public function attachFeedCorrelations(array $attributes, array $user, array &$event, $overrideLimit = false, $scope = 'Feed')
     {
+        if (empty($attributes)) {
+            return $attributes;
+        }
+
         try {
             $redis = $this->setupRedisWithException();
         } catch (Exception $e) {
-            return $objects;
+            return $attributes;
         }
 
         $cachePrefix = 'misp:' . strtolower($scope) . '_cache:';
 
-        // Redis cache for $scope is empty.
+        // Skip if redis cache for $scope is empty.
         if ($redis->sCard($cachePrefix . 'combined') === 0) {
-            return $objects;
+            return $attributes;
         }
 
+        if (!isset($this->Attribute)) {
+            $this->Attribute = ClassRegistry::init('Attribute');
+        }
+        $compositeTypes = $this->Attribute->getCompositeTypes();
+
         $pipe = $redis->multi(Redis::PIPELINE);
-        $hashTable = array();
+        $hashTable = [];
+        $redisResultToAttributePosition = [];
 
-        $this->Event = ClassRegistry::init('Event');
-        $compositeTypes = $this->Event->Attribute->getCompositeTypes();
-
-        foreach ($objects as $k => $object) {
-            if (in_array($object['type'], $compositeTypes)) {
-                $value = explode('|', $object['value']);
-                $hashTable[$k] = md5($value[0]);
-            } else {
-                $hashTable[$k] = md5($object['value']);
+        foreach ($attributes as $k => $attribute) {
+            if (in_array($attribute['type'], $this->Attribute->nonCorrelatingTypes)) {
+                continue; // attribute type is not correlateable
+            }
+            if (!empty($attribute['disable_correlation'])) {
+                continue; // attribute correlation is disabled
+            }
+
+            if (in_array($attribute['type'], $compositeTypes)) {
+                list($value1, $value2) = explode('|', $attribute['value']);
+                $parts = [$value1];
+
+                if (!in_array($attribute['type'], $this->Attribute->primaryOnlyCorrelatingTypes)) {
+                    $parts[] = $value2;
+                }
+            } else {
+                $parts = [$attribute['value']];
+
+                // Some feeds contains URL without protocol, so if attribute is URL and value contains protocol,
+                // we will check also value without protocol.
+                if ($attribute['type'] === 'url' || $attribute['type'] === 'uri') {
+                    $protocolPos = strpos($attribute['value'], '://');
+                    if ($protocolPos !== false) {
+                        $parts[] = substr($attribute['value'], $protocolPos + 3);
+                    }
+                }
+            }
+
+            foreach ($parts as $part) {
+                $md5 = md5($part);
+                $hashTable[] = $md5;
+                $redis->sismember($cachePrefix . 'combined', $md5);
+                $redisResultToAttributePosition[] = $k;
             }
-            $redis->sismember($cachePrefix . 'combined', $hashTable[$k]);
         }
+
+        if (empty($redisResultToAttributePosition)) {
+            // No attribute that can be correlated
+            $pipe->discard();
+            return $attributes;
+        }
+
         $results = $pipe->exec();
-        if (!$overrideLimit && count($objects) > 10000) {
-            foreach ($results as $k => $result) {
-                if ($result && empty($objects[$k]['disable_correlation'])) {
-                    if (isset($event['FeedCount'])) {
-                        $event['FeedCount']++;
-                    } else {
-                        $event['FeedCount'] = 1;
+
+        $hitIds = [];
+        foreach ($results as $k => $result) {
+            if ($result) {
+                $hitIds[] = $k;
+            }
+        }
+
+        if (empty($hitIds)) {
+            return $attributes; // nothing matches, skip
+        }
+
+        $hitCount = count($hitIds);
+        if (!$overrideLimit && $hitCount > 10000) {
+            $event['FeedCount'] = $hitCount;
+            foreach ($hitIds as $k) {
+                $attributes[$redisResultToAttributePosition[$k]]['FeedHit'] = true;
+            }
+            return $attributes;
+        }
+
+        $sources = $this->getCachedFeedsOrServers($user, $scope);
+        foreach ($sources as $source) {
+            $sourceId = $source[$scope]['id'];
+
+            $pipe = $redis->multi(Redis::PIPELINE);
+            foreach ($hitIds as $k) {
+                $redis->sismember($cachePrefix . $sourceId, $hashTable[$k]);
+            }
+            $sourceHits = $pipe->exec();
+            $sourceHasHit = false;
+            foreach ($sourceHits as $k => $hit) {
+                if ($hit) {
+                    if (!isset($event[$scope][$sourceId])) {
+                        $event[$scope][$sourceId] = $source[$scope];
                     }
-                    $objects[$k]['FeedHit'] = true;
+                    $attributePosition = $redisResultToAttributePosition[$hitIds[$k]];
+                    $attributes[$attributePosition][$scope][] = $source[$scope];
+                    $sourceHasHit = true;
                 }
             }
-        } else {
-            if ($scope === 'Feed') {
-                $params = array(
-                    'recursive' => -1,
-                    'fields' => array('id', 'name', 'url', 'provider', 'source_format')
-                );
-                if (!$user['Role']['perm_site_admin']) {
-                    $params['conditions'] = array('Feed.lookup_visible' => 1);
-                }
-                $sources = $this->find('all', $params);
-            } else {
-                $params = array(
-                    'recursive' => -1,
-                    'fields' => array('id', 'name', 'url', 'caching_enabled')
-                );
-                if (!$user['Role']['perm_site_admin']) {
-                    $params['conditions'] = array('Server.caching_enabled' => 1);
-                }
-                $this->Server = ClassRegistry::init('Server');
-                $sources = $this->Server->find('all', $params);
-            }
-
-            $hitIds = array();
-            foreach ($results as $k => $result) {
-                if ($result && empty($objects[$k]['disable_correlation'])) {
-                    $hitIds[] = $k;
-                }
-            }
-            foreach ($sources as $source) {
-                $sourceScopeId = $source[$scope]['id'];
-
+            // Append also exact MISP feed event UUID
+            // TODO: This can be optimised in future to do that in one pass
+            if ($sourceHasHit && ($scope === 'Server' || $source[$scope]['source_format'] === 'misp')) {
                 $pipe = $redis->multi(Redis::PIPELINE);
-                foreach ($hitIds as $k) {
-                    $redis->sismember($cachePrefix . $sourceScopeId, $hashTable[$k]);
-                }
-                $sourceHits = $pipe->exec();
-                foreach ($sourceHits as $k4 => $hit) {
-                    if ($hit) {
-                        if (!isset($event[$scope][$sourceScopeId]['id'])) {
-                            if (!isset($event[$scope][$sourceScopeId])) {
-                                $event[$scope][$sourceScopeId] = array();
-                            }
-                            $event[$scope][$sourceScopeId] = array_merge($event[$scope][$sourceScopeId], $source[$scope]);
-                        }
-                        $objects[$hitIds[$k4]][$scope][] = $source[$scope];
+                $eventUuidHitPosition = [];
+                foreach ($hitIds as $sourceHitPos => $k) {
+                    if ($sourceHits[$sourceHitPos]) {
+                        $redis->smembers($cachePrefix . 'event_uuid_lookup:' . $hashTable[$k]);
+                        $eventUuidHitPosition[] = $redisResultToAttributePosition[$k];
                     }
                 }
-                if ($scope === 'Server' || $source[$scope]['source_format'] == 'misp') {
-                    $pipe = $redis->multi(Redis::PIPELINE);
-                    $eventUuidHitPosition = array();
-                    foreach ($objects as $k => $object) {
-                        if (isset($object[$scope])) {
-                            foreach ($object[$scope] as $currentFeed) {
-                                if ($source[$scope]['id'] == $currentFeed['id']) {
-                                    $eventUuidHitPosition[] = $k;
-                                    $redis->smembers($cachePrefix . 'event_uuid_lookup:' . $hashTable[$k]);
-                                }
-                            }
-                        }
+                $mispFeedHits = $pipe->exec();
+                foreach ($mispFeedHits as $sourceHitPos => $feedUuidMatches) {
+                    if (empty($feedUuidMatches)) {
+                        continue;
                     }
-                    $mispFeedHits = $pipe->exec();
-                    foreach ($mispFeedHits as $sourcehitPos => $f) {
-                        foreach ($f as $url) {
-                            list($feedId, $eventUuid) = explode('/', $url);
-                            if (empty($event[$scope][$feedId]['event_uuids']) || !in_array($eventUuid, $event[$scope][$feedId]['event_uuids'])) {
-                                $event[$scope][$feedId]['event_uuids'][] = $eventUuid;
-                            }
-                            foreach ($objects[$eventUuidHitPosition[$sourcehitPos]][$scope] as $tempKey => $tempFeed) {
-                                if ($tempFeed['id'] == $feedId) {
-                                    $objects[$eventUuidHitPosition[$sourcehitPos]][$scope][$tempKey]['event_uuids'][] = $eventUuid;
-                                }
+                    foreach ($feedUuidMatches as $url) {
+                        list($feedId, $eventUuid) = explode('/', $url);
+                        if ($feedId != $sourceId) {
+                            continue; // just process current source, skip others
+                        }
+
+                        if (empty($event[$scope][$feedId]['event_uuids']) || !in_array($eventUuid, $event[$scope][$feedId]['event_uuids'])) {
+                            $event[$scope][$feedId]['event_uuids'][] = $eventUuid;
+                        }
+                        $attributePosition = $eventUuidHitPosition[$sourceHitPos];
+                        foreach ($attributes[$attributePosition][$scope] as $tempKey => $tempFeed) {
+                            if ($tempFeed['id'] == $feedId) {
+                                $attributes[$attributePosition][$scope][$tempKey]['event_uuids'][] = $eventUuid;
+                                break;
                             }
                         }
                     }
@@ -460,11 +484,58 @@ class Feed extends AppModel
             }
         }
 
-        if (!empty($event[$scope])) {
+        if (isset($event[$scope])) {
             $event[$scope] = array_values($event[$scope]);
         }
 
-        return $objects;
+        return $attributes;
+    }
+
+    /**
+     * Return just feeds or servers that has some data in Redis cache.
+     * @param array $user
+     * @param string $scope 'Feed' or 'Server'
+     * @return array
+     */
+    private function getCachedFeedsOrServers(array $user, $scope)
+    {
+        if ($scope === 'Feed') {
+            $params = array(
+                'recursive' => -1,
+                'fields' => array('id', 'name', 'url', 'provider', 'source_format')
+            );
+            if (!$user['Role']['perm_site_admin']) {
+                $params['conditions'] = array('Feed.lookup_visible' => 1);
+            }
+            $sources = $this->find('all', $params);
+        } else {
+            $params = array(
+                'recursive' => -1,
+                'fields' => array('id', 'name', 'url')
+            );
+            if (!$user['Role']['perm_site_admin']) {
+                $params['conditions'] = array('Server.caching_enabled' => 1);
+            }
+            $this->Server = ClassRegistry::init('Server');
+            $sources = $this->Server->find('all', $params);
+        }
+
+        try {
+            $redis = $this->setupRedisWithException();
+            $pipe = $redis->multi(Redis::PIPELINE);
+            $cachePrefix = 'misp:' . strtolower($scope) . '_cache:';
+            foreach ($sources as $source) {
+                $pipe->exists($cachePrefix . $source[$scope]['id']);
+            }
+            $results = $pipe->exec();
+            foreach ($sources as $k => $source) {
+                if (!$results[$k]) {
+                    unset($sources[$k]);
+                }
+            }
+        } catch (Exception $e) {}
+
+        return $sources;
     }
 
     public function downloadFromFeed($actions, $feed, HttpSocket $HttpSocket = null, $user, $jobId = false)
@@ -518,7 +589,6 @@ class Feed extends AppModel
     {
         $version = $this->checkMISPVersion();
         $version = implode('.', $version);
-        $commit = trim(shell_exec('git log --pretty="%H" -n1 HEAD'));
 
         $result = array(
             'header' => array(
@@ -533,6 +603,7 @@ class Feed extends AppModel
             $result['header']['Accept-Encoding'] = 'gzip';
         }
 
+        $commit = $this->checkMIPSCommit();
         if ($commit) {
             $result['header']['commit'] = $commit;
         }
@@ -1104,52 +1175,73 @@ class Feed extends AppModel
         return $results;
     }
 
-    public function attachFeedCacheTimestamps($data)
+    /**
+     * @param array $feeds
+     * @return array
+     */
+    public function attachFeedCacheTimestamps(array $feeds)
     {
-        $redis = $this->setupRedis();
-        if ($redis === false) {
-            return $data;
+        try {
+            $redis = $this->setupRedisWithException();
+        } catch (Exception $e) {
+            return $feeds;
         }
-        foreach ($data as $k => $v) {
-            $data[$k]['Feed']['cache_timestamp'] = $redis->get('misp:feed_cache_timestamp:' . $data[$k]['Feed']['id']);
+
+        $pipe = $redis->multi(Redis::PIPELINE);
+        foreach ($feeds as $feed) {
+            $pipe->get('misp:feed_cache_timestamp:' . $feed['Feed']['id']);
         }
-        return $data;
+        $result = $redis->exec();
+        foreach ($feeds as $k => $feed) {
+            $feeds[$k]['Feed']['cache_timestamp'] = $result[$k];
+        }
+        return $feeds;
     }
 
     private function __cacheFeed($feed, $redis, $jobId = false)
     {
         $HttpSocket = $this->isFeedLocal($feed) ? null : $this->__setupHttpSocket($feed);
-        if ($feed['Feed']['source_format'] == 'misp') {
+        if ($feed['Feed']['source_format'] === 'misp') {
             return $this->__cacheMISPFeed($feed, $redis, $HttpSocket, $jobId);
         } else {
             return $this->__cacheFreetextFeed($feed, $redis, $HttpSocket, $jobId);
         }
     }
 
+    /**
+     * @param array $feed
+     * @param Redis $redis
+     * @param HttpSocket|null $HttpSocket
+     * @param int|false $jobId
+     * @return bool
+     */
     private function __cacheFreetextFeed(array $feed, $redis, HttpSocket $HttpSocket = null, $jobId = false)
     {
         $feedId = $feed['Feed']['id'];
 
+        $this->jobProgress($jobId, __("Feed %s: Fetching.", $feedId));
+
         try {
             $values = $this->getFreetextFeed($feed, $HttpSocket, $feed['Feed']['source_format'], 'all');
         } catch (Exception $e) {
             $this->logException("Could not get freetext feed $feedId", $e);
-            $this->jobProgress($jobId, 'Could not fetch freetext feed. See error log for more details.');
+            $this->jobProgress($jobId, __('Could not fetch freetext feed. See error log for more details.'));
             return false;
         }
 
         $pipe = $redis->multi(Redis::PIPELINE);
+        $redis->del('misp:feed_cache:' . $feedId);
         foreach ($values as $k => $value) {
             $md5Value = md5($value['value']);
             $redis->sAdd('misp:feed_cache:' . $feedId, $md5Value);
             $redis->sAdd('misp:feed_cache:combined', $md5Value);
-            if ($k % 1000 == 0) {
+            if ($k % 1000 === 0) {
                 $this->jobProgress($jobId, "Feed $feedId: $k/" . count($values) . " values cached.");
                 $pipe->exec();
                 $pipe = $redis->multi(Redis::PIPELINE);
             }
         }
-        $redis->set('misp:feed_cache_timestamp:' . $feedId, time());
+        $pipe->set('misp:feed_cache_timestamp:' . $feedId, time());
         $pipe->exec();
         return true;
     }
@@ -1182,16 +1274,18 @@ class Feed extends AppModel
                     if (!in_array($attribute['type'], $this->Attribute->nonCorrelatingTypes)) {
                         if (in_array($attribute['type'], $this->Attribute->getCompositeTypes())) {
                             $value = explode('|', $attribute['value']);
-                            $redis->sAdd('misp:feed_cache:' . $feedId, md5($value[0]));
-                            $redis->sAdd('misp:feed_cache:' . $feedId, md5($value[1]));
-                            $redis->sAdd('misp:feed_cache:combined', md5($value[0]));
-                            $redis->sAdd('misp:feed_cache:combined', md5($value[1]));
-                            $redis->sAdd('misp:feed_cache:event_uuid_lookup:' . md5($value[0]), $feedId . '/' . $event['Event']['uuid']);
-                            $redis->sAdd('misp:feed_cache:event_uuid_lookup:' . md5($value[1]), $feedId . '/' . $event['Event']['uuid']);
+                            if (in_array($attribute['type'], $this->Attribute->primaryOnlyCorrelatingTypes)) {
+                                unset($value[1]);
+                            }
                         } else {
-                            $redis->sAdd('misp:feed_cache:' . $feedId, md5($attribute['value']));
-                            $redis->sAdd('misp:feed_cache:combined', md5($attribute['value']));
-                            $redis->sAdd('misp:feed_cache:event_uuid_lookup:' . md5($attribute['value']), $feedId . '/' . $event['Event']['uuid']);
+                            $value = [$attribute['value']];
+                        }
+
+                        foreach ($value as $v) {
+                            $md5 = md5($v);
+                            $redis->sAdd('misp:feed_cache:' . $feedId, $md5);
+                            $redis->sAdd('misp:feed_cache:combined', $md5);
+                            $redis->sAdd('misp:feed_cache:event_uuid_lookup:' . $md5, $feedId . '/' . $event['Event']['uuid']);
                         }
                     }
                 }
@@ -1218,10 +1312,12 @@ class Feed extends AppModel
         }
 
         $pipe = $redis->multi(Redis::PIPELINE);
+        $redis->del('misp:feed_cache:' . $feedId);
         foreach ($cache as $v) {
-            $redis->sAdd('misp:feed_cache:' . $feedId, $v[0]);
-            $redis->sAdd('misp:feed_cache:combined', $v[0]);
-            $redis->sAdd('misp:feed_cache:event_uuid_lookup:' . $v[0], $feedId . '/' . $v[1]);
+            list($hash, $eventUuid) = $v;
+            $redis->sAdd('misp:feed_cache:' . $feedId, $hash);
+            $redis->sAdd('misp:feed_cache:combined', $hash);
+            $redis->sAdd('misp:feed_cache:event_uuid_lookup:' . $hash, "$feedId/$eventUuid");
         }
         $pipe->exec();
         $this->jobProgress($jobId, "Feed $feedId: cached via quick cache.");
@@ -1233,7 +1329,7 @@ class Feed extends AppModel
         $result = true;
         if (!$this->__cacheMISPFeedCache($feed, $redis, $HttpSocket, $jobId)) {
             $result = $this->__cacheMISPFeedTraditional($feed, $redis, $HttpSocket, $jobId);
-        };
+        }
         if ($result) {
             $redis->set('misp:feed_cache_timestamp:' . $feed['Feed']['id'], time());
         }
@@ -1743,8 +1839,10 @@ class Feed extends AppModel
     private function jobProgress($jobId = null, $message = null, $progress = null)
     {
         if ($jobId) {
-            $job = ClassRegistry::init('Job');
-            $job->saveProgress($jobId, $message, $progress);
+            if (!isset($this->Job)) {
+                $this->Job = ClassRegistry::init('Job');
+            }
+            $this->Job->saveProgress($jobId, $message, $progress);
         }
     }
 
@@ -1804,13 +1902,25 @@ class Feed extends AppModel
     private function unzipFirstFile(File $zipFile)
     {
         if (!class_exists('ZipArchive')) {
-            throw new Exception('ZIP archive decompressing is not supported. ZIP support is missing in PHP.');
+            throw new Exception('ZIP archive decompressing is not supported. ZIP extension is missing in PHP.');
         }
 
         $zip = new ZipArchive();
         $result = $zip->open($zipFile->pwd());
         if ($result !== true) {
-            throw new Exception("Remote server returns ZIP file, that cannot be open (error $result)");
+            $errorCodes = [
+                ZipArchive::ER_EXISTS => 'file already exists',
+                ZipArchive::ER_INCONS => 'zip archive inconsistent',
+                ZipArchive::ER_INVAL => 'invalid argument',
+                ZipArchive::ER_MEMORY => 'malloc failure',
+                ZipArchive::ER_NOENT => 'no such file',
+                ZipArchive::ER_NOZIP => 'not a zip archive',
+                ZipArchive::ER_OPEN => 'can\'t open file',
+                ZipArchive::ER_READ => 'read error',
+                ZipArchive::ER_SEEK => 'seek error',
+            ];
+            $message = isset($errorCodes[$result]) ? $errorCodes[$result] : 'error ' . $result;
+            throw new Exception("Remote server returns ZIP file, that cannot be open ($message)");
         }
 
         if ($zip->numFiles !== 1) {
@@ -1827,7 +1937,7 @@ class Feed extends AppModel
         $destinationFile = $this->tempFileName();
         $result = copy("zip://{$zipFile->pwd()}#$filename", $destinationFile);
         if ($result === false) {
-            throw new Exception("Remote server returns ZIP file, that contains '$filename' file, that cannot be extracted.");
+            throw new Exception("Remote server returns ZIP file, that contains '$filename' file, but this file cannot be extracted.");
         }
 
         $unzipped = new File($destinationFile);

app/Model/MispObject.php

@@ -545,16 +545,15 @@ class MispObject extends AppModel
             }
         }
         $results = $this->find('all', $params);
-        if ($options['enforceWarninglist']) {
+        if ($options['enforceWarninglist'] && !isset($this->Warninglist)) {
             $this->Warninglist = ClassRegistry::init('Warninglist');
-            $warninglists = $this->Warninglist->fetchForEventView();
         }
         $results = array_values($results);
         $proposals_block_attributes = Configure::read('MISP.proposals_block_attributes');
         if (empty($options['metadata'])) {
             foreach ($results as $key => $object) {
                 foreach ($object['Attribute'] as $key2 => $attribute) {
-                    if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttributes($warninglists, $attribute['Attribute'], $this->Warninglist)) {
+                    if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttribute($attribute['Attribute'])) {
                         unset($results[$key][$key2]);
                         continue;
                     }
@@ -644,8 +643,10 @@ class MispObject extends AppModel
         return $template;
     }
 
-    /*
+    /**
      * Clean the attribute list up from artifacts introduced by the object form
+     * @param array $attributes
+     * @return string|array
      */
     public function attributeCleanup($attributes)
     {
@@ -795,21 +796,6 @@ class MispObject extends AppModel
                                     if (isset($newAttribute[$f]) && $newAttribute[$f] != $originalAttribute[$f]) {
                                         $different = true;
                                     }
-                                    // Set seen of object at attribute level
-                                    if (isset($forcedSeenOnElements['first_seen'])) {
-                                        $newAttribute['first_seen'] = $forcedSeenOnElements['first_seen'];
-                                        if ($newAttribute['object_relation'] == 'first-seen') {
-                                            // $newAttribute['value'] = $forcedSeenOnElements['first_seen'];
-                                        }
-                                        $different = true;
-                                    }
-                                    if (isset($forcedSeenOnElements['last_seen'])) {
-                                        $newAttribute['last_seen'] = $forcedSeenOnElements['last_seen'];
-                                        if ($newAttribute['object_relation'] == 'last-seen') {
-                                            // $newAttribute['value'] = $forcedSeenOnElements['last_seen'];
-                                        }
-                                        $different = true;
-                                    }
                                 }
                                 if ($different) {
                                     $newAttribute['id'] = $originalAttribute['id'];
@@ -831,13 +817,13 @@ class MispObject extends AppModel
                     $newAttribute['object_id'] = $object['Object']['id'];
                     // Set seen of object at attribute level
                     if (isset($forcedSeenOnElements['first_seen'])) {
-                        $newAttribute['first_seen'] = $forcedSeenOnElements['first_seen'];
+                        $newAttribute['first_seen'] = empty($newAttribute['first_seen']) ? $forcedSeenOnElements['first_seen'] : $newAttribute['first_seen'];
                         if ($newAttribute['object_relation'] == 'first-seen') {
                             $newAttribute['value'] = $forcedSeenOnElements['first_seen'];
                         }
                     }
                     if (isset($forcedSeenOnElements['last_seen'])) {
-                        $newAttribute['last_seen'] = $forcedSeenOnElements['last_seen'];
+                        $newAttribute['last_seen'] = empty($newAttribute['last_seen']) ? $forcedSeenOnElements['last_seen'] : $newAttribute['last_seen'];
                         if ($newAttribute['object_relation'] == 'last-seen') {
                             $newAttribute['value'] = $forcedSeenOnElements['last_seen'];
                         }

app/Model/Module.php

@@ -82,28 +82,32 @@ class Module extends AppModel
         return true;
     }
 
-
-    public function getModules($type = false, $moduleFamily = 'Enrichment', &$exception = false)
+    /**
+     * @param string $moduleFamily
+     * @param bool $throwException
+     * @return array[]|string
+     * @throws JsonException
+     */
+    public function getModules($moduleFamily = 'Enrichment', $throwException = false)
     {
-        $modules = $this->queryModuleServer('/modules', false, false, $moduleFamily, $exception);
-        if (!$modules) {
+        try {
+            // Wait just one second to not block loading pages when modules are not reachable
+            return $this->sendRequest('/modules', 1, null, $moduleFamily);
+        } catch (Exception $e) {
+            if ($throwException) {
+                throw $e;
+            }
             return 'Module service not reachable.';
         }
-        if (!empty($modules)) {
-            $result = array('modules' => $modules);
-            return $result;
-        } else {
-            return 'The module service reports that it found no modules.';
-        }
     }
 
     public function getEnabledModules($user, $type = false, $moduleFamily = 'Enrichment')
     {
-        $modules = $this->getModules($type, $moduleFamily);
+        $modules = $this->getModules($moduleFamily);
         if (is_array($modules)) {
-            foreach ($modules['modules'] as $k => $module) {
+            foreach ($modules as $k => $module) {
                 if (!Configure::read('Plugin.' . $moduleFamily . '_' . $module['name'] . '_enabled') || ($type && !in_array(strtolower($type), $module['meta']['module-type']))) {
-                    unset($modules['modules'][$k]);
+                    unset($modules[$k]);
                     continue;
                 }
                 if (
@@ -111,51 +115,52 @@ class Module extends AppModel
                     Configure::read('Plugin.' . $moduleFamily . '_' . $module['name'] . '_restrict') &&
                     Configure::read('Plugin.' . $moduleFamily . '_' . $module['name'] . '_restrict') != $user['org_id']
                 ) {
-                    unset($modules['modules'][$k]);
+                    unset($modules[$k]);
                 }
             }
         } else {
             return 'The modules system reports that it found no suitable modules.';
         }
-        if (!isset($modules) || empty($modules)) {
-            $modules = array();
+        if (empty($modules)) {
+            return [];
         }
-        if (isset($modules['modules']) && !empty($modules['modules'])) {
-            $modules['modules'] = array_values($modules['modules']);
-        }
-        if (!is_array($modules)) {
-            return array();
-        }
-        foreach ($modules['modules'] as $temp) {
+        $output = ['modules' => array_values($modules)];
+        foreach ($modules as $temp) {
             if (isset($temp['meta']['module-type']) && in_array('import', $temp['meta']['module-type'])) {
-                $modules['Import'] = $temp['name'];
+                $output['Import'] = $temp['name'];
             } elseif (isset($temp['meta']['module-type']) && in_array('export', $temp['meta']['module-type'])) {
-                $modules['Export'] = $temp['name'];
+                $output['Export'] = $temp['name'];
             } else {
                 foreach ($temp['mispattributes']['input'] as $input) {
                     if (!isset($temp['meta']['module-type']) || (in_array('expansion', $temp['meta']['module-type']) || in_array('cortex', $temp['meta']['module-type']))) {
-                        $modules['types'][$input][] = $temp['name'];
+                        $output['types'][$input][] = $temp['name'];
                     }
                     if (isset($temp['meta']['module-type']) && in_array('hover', $temp['meta']['module-type'])) {
-                        $modules['hover_type'][$input][] = $temp['name'];
+                        $output['hover_type'][$input][] = $temp['name'];
                     }
                 }
             }
         }
-        return $modules;
+        return $output;
     }
 
+    /**
+     * @param string $name
+     * @param string $type
+     * @return array|string
+     */
     public function getEnabledModule($name, $type)
     {
+        if (!isset($this->__typeToFamily[$type])) {
+            throw new InvalidArgumentException("Invalid type '$type'.");
+        }
         $moduleFamily = $this->__typeToFamily[$type];
-        $url = $this->__getModuleServer($moduleFamily);
-        $modules = $this->getModules($type, $moduleFamily);
-        $module = false;
+        $modules = $this->getModules($moduleFamily);
         if (!Configure::read('Plugin.' . $moduleFamily . '_' . $name . '_enabled')) {
             return 'The requested module is not enabled.';
         }
         if (is_array($modules)) {
-            foreach ($modules['modules'] as $module) {
+            foreach ($modules as $module) {
                 if ($module['name'] == $name) {
                     if ($type && in_array(strtolower($type), $module['meta']['module-type'])) {
                         return $module;
@@ -175,80 +180,132 @@ class Module extends AppModel
         if (!Configure::read('Plugin.' . $moduleFamily . '_services_enable')) {
             return false;
         }
-        $this->Server = ClassRegistry::init('Server');
-        $url = Configure::read('Plugin.' . $moduleFamily . '_services_url') ? Configure::read('Plugin.' . $moduleFamily . '_services_url') : $this->Server->serverSettings['Plugin'][$moduleFamily . '_services_url']['value'];
-        $port = Configure::read('Plugin.' . $moduleFamily . '_services_port') ? Configure::read('Plugin.' . $moduleFamily . '_services_port') : $this->Server->serverSettings['Plugin'][$moduleFamily . '_services_port']['value'];
-        return $url . ':' . $port;
+
+        $url = Configure::read('Plugin.' . $moduleFamily . '_services_url');
+        $port = Configure::read('Plugin.' . $moduleFamily . '_services_port');
+
+        if (empty($url) || empty($port)) {
+            // Load default values
+            $this->Server = ClassRegistry::init('Server');
+            if (empty($url)) {
+                $url = $this->Server->serverSettings['Plugin'][$moduleFamily . '_services_url']['value'];
+            }
+            if (empty($port)) {
+                $port = $this->Server->serverSettings['Plugin'][$moduleFamily . '_services_port']['value'];
+            }
+        }
+
+        return "$url:$port";
     }
 
-    public function queryModuleServer($uri, $post = false, $hover = false, $moduleFamily = 'Enrichment', &$exception = false)
+    /**
+     * Send request to `/query` module endpoint.
+     *
+     * @param array $postData
+     * @param bool $hover
+     * @param string $moduleFamily
+     * @param bool $throwException
+     * @return array|false
+     * @throws JsonException
+     */
+    public function queryModuleServer(array $postData, $hover = false, $moduleFamily = 'Enrichment', $throwException = false)
+    {
+        if ($hover) {
+            $timeout = Configure::read('Plugin.' . $moduleFamily . '_hover_timeout') ?: 5;
+        } else {
+            $timeout = Configure::read('Plugin.' . $moduleFamily . '_timeout') ?: 10;
+        }
+        try {
+            return $this->sendRequest('/query', $timeout, $postData, $moduleFamily);
+        } catch (Exception $e) {
+            if ($throwException) {
+                throw $e;
+            }
+            $this->logException('Failed to query module ' . $moduleFamily, $e);
+            return false;
+        }
+    }
+
+    /**
+     * Low-level way how to send request to module.
+     *
+     * @param string $uri
+     * @param int $timeout
+     * @param array|null $postData
+     * @param string $moduleFamily
+     * @return array
+     * @throws JsonException
+     */
+    public function sendRequest($uri, $timeout, $postData = null, $moduleFamily = 'Enrichment')
     {
         $url = $this->__getModuleServer($moduleFamily);
         if (!$url) {
-            return false;
+            throw new Exception("Module type $moduleFamily is not enabled.");
         }
         App::uses('HttpSocket', 'Network/Http');
-        if ($hover) {
-            $settings = array(
-                'timeout' => Configure::read('Plugin.' . $moduleFamily . '_hover_timeout') ? Configure::read('Plugin.' . $moduleFamily . '_hover_timeout') : 5
-            );
-        } else {
-            $settings = array(
-                'timeout' => Configure::read('Plugin.' . $moduleFamily . '_timeout') ? Configure::read('Plugin.' . $moduleFamily . '_timeout') : 10
-            );
-        }
         $sslSettings = array('ssl_verify_peer', 'ssl_verify_host', 'ssl_allow_self_signed', 'ssl_verify_peer', 'ssl_cafile');
         foreach ($sslSettings as $sslSetting) {
             if (Configure::check('Plugin.' . $moduleFamily . '_' . $sslSetting) && Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting) !== '') {
                 $settings[$sslSetting] = Configure::read('Plugin.' . $moduleFamily . '_' . $sslSetting);
             }
         }
-        // let's set a low timeout for the introspection so that we don't block the loading of pages due to a misconfigured modules
-        if ($uri == '/modules') {
-            $settings['timeout'] = 1;
-        }
-        $httpSocket = new HttpSocket($settings);
+        $httpSocket = new HttpSocket(['timeout' => $timeout]);
         $request = array(
-                'header' => array(
-                        'Content-Type' => 'application/json',
-                )
+            'header' => array(
+                'Content-Type' => 'application/json',
+            )
         );
         if ($moduleFamily == 'Cortex') {
             if (!empty(Configure::read('Plugin.' . $moduleFamily . '_authkey'))) {
                 $request['header']['Authorization'] = 'Bearer ' . Configure::read('Plugin.' . $moduleFamily . '_authkey');
             }
         }
-        try {
-            if ($post) {
-                $response = $httpSocket->post($url . $uri, $post, $request);
-            } else {
-                if ($moduleFamily == 'Cortex') {
-                    unset($request['header']['Content-Type']);
-                }
-                $response = $httpSocket->get($url . $uri, false, $request);
+        if ($postData) {
+            if (!is_array($postData)) {
+                throw new InvalidArgumentException("Post data must be array, " . gettype($postData) . " given.");
             }
-            return json_decode($response->body, true);
-        } catch (Exception $e) {
-            $this->logException('Failed to query module ' . $moduleFamily, $e);
-            $exception = $e->getMessage();
-            return false;
+            $post = json_encode($postData);
+            $response = $httpSocket->post($url . $uri, $post, $request);
+        } else {
+            if ($moduleFamily == 'Cortex') {
+                unset($request['header']['Content-Type']);
+            }
+            $response = $httpSocket->get($url . $uri, false, $request);
         }
+        if (!$response->isOk()) {
+            if ($httpSocket->lastError()) {
+                throw new Exception("Failed to get response from $moduleFamily module: " . $httpSocket->lastError['str']);
+            }
+            throw new Exception("Failed to get response from $moduleFamily module: HTTP $response->reasonPhrase", (int)$response->code);
+        }
+        return $this->jsonDecode($response->body);
     }
 
+    /**
+     * @param string $moduleFamily
+     * @return array
+     */
     public function getModuleSettings($moduleFamily = 'Enrichment')
     {
-        $modules = $this->getModules(false, $moduleFamily);
+        $modules = $this->getModules($moduleFamily);
         $result = array();
-        if (!empty($modules['modules'])) {
-            foreach ($modules['modules'] as $module) {
+        if (is_array($modules)) {
+            foreach ($modules as $module) {
                 if (array_intersect($this->__validTypes[$moduleFamily], $module['meta']['module-type'])) {
-                    $result[$module['name']][0] = array('name' => 'enabled', 'type' => 'boolean');
-                    $result[$module['name']][1] = array('name' => 'restrict', 'type' => 'orgs');
+                    $moduleSettings = [
+                        array('name' => 'enabled', 'type' => 'boolean'),
+                        array('name' => 'restrict', 'type' => 'orgs')
+                    ];
                     if (isset($module['meta']['config'])) {
-                        foreach ($module['meta']['config'] as $conf) {
-                            $result[$module['name']][] = array('name' => $conf, 'type' => 'string');
+                        foreach ($module['meta']['config'] as $key => $value) {
+                            if (is_string($key)) {
+                                $moduleSettings[] = array('name' => $key, 'type' => 'string', 'description' => $value);
+                            } else {
+                                $moduleSettings[] = array('name' => $value, 'type' => 'string');
+                            }
                         }
                     }
+                    $result[$module['name']] = $moduleSettings;
                 }
             }
         }

app/Model/Organisation.php

@@ -68,8 +68,6 @@ class Organisation extends AppModel
         ),
     );
 
-    public $countries = array('Not specified', 'International', 'Afghanistan', 'Albania', 'Algeria', 'Andorra', 'Angola', 'Antigua & Deps', 'Argentina', 'Armenia', 'Australia', 'Austria', 'Azerbaijan', 'Bahamas', 'Bahrain', 'Bangladesh', 'Barbados', 'Belarus', 'Belgium', 'Belize', 'Benin', 'Bhutan', 'Bolivia', 'Bosnia Herzegovina', 'Botswana', 'Brazil', 'Brunei', 'Bulgaria', 'Burkina', 'Burundi', 'Cambodia', 'Cameroon', 'Canada', 'Cape Verde', 'Central African Rep', 'Chad', 'Chile', 'China', 'Colombia', 'Comoros', 'Congo', 'Congo {Democratic Rep}', 'Costa Rica', 'Croatia', 'Cuba', 'Cyprus', 'Czech Republic', 'Denmark', 'Djibouti', 'Dominica', 'Dominican Republic', 'East Timor', 'Ecuador', 'Egypt', 'El Salvador', 'Equatorial Guinea', 'Eritrea', 'Estonia', 'Ethiopia', 'Fiji', 'Finland', 'France', 'Gabon', 'Gambia', 'Georgia', 'Germany', 'Ghana', 'Greece', 'Grenada', 'Guatemala', 'Guinea', 'Guinea-Bissau', 'Guyana', 'Haiti', 'Honduras', 'Hungary', 'Iceland', 'India', 'Indonesia', 'Iran', 'Iraq', 'Ireland {Republic}', 'Israel', 'Italy', 'Ivory Coast', 'Jamaica', 'Japan', 'Jordan', 'Kazakhstan', 'Kenya', 'Kiribati', 'Korea North', 'Korea South', 'Kosovo', 'Kuwait', 'Kyrgyzstan', 'Laos', 'Latvia', 'Lebanon', 'Lesotho', 'Liberia', 'Libya', 'Liechtenstein', 'Lithuania', 'Luxembourg', 'Macedonia', 'Madagascar', 'Malawi', 'Malaysia', 'Maldives', 'Mali', 'Malta', 'Marshall Islands', 'Mauritania', 'Mauritius', 'Mexico', 'Micronesia', 'Moldova', 'Monaco', 'Mongolia', 'Montenegro', 'Morocco', 'Mozambique', 'Myanmar, {Burma}', 'Namibia', 'Nauru', 'Nepal', 'Netherlands', 'New Zealand', 'Nicaragua', 'Niger', 'Nigeria', 'Norway', 'Oman', 'Pakistan', 'Palau', 'Panama', 'Papua New Guinea', 'Paraguay', 'Peru', 'Philippines', 'Poland', 'Portugal', 'Qatar', 'Romania', 'Russian Federation', 'Rwanda', 'St Kitts & Nevis', 'St Lucia', 'Saint Vincent & the Grenadines', 'Samoa', 'San Marino', 'Sao Tome & Principe', 'Saudi Arabia', 'Senegal', 'Serbia', 'Seychelles', 'Sierra Leone', 'Singapore', 'Slovakia', 'Slovenia', 'Solomon Islands', 'Somalia', 'South Africa', 'South Sudan', 'Spain', 'Sri Lanka', 'Sudan', 'Suriname', 'Swaziland', 'Sweden', 'Switzerland', 'Syria', 'Taiwan', 'Tajikistan', 'Tanzania', 'Thailand', 'Togo', 'Tonga', 'Trinidad & Tobago', 'Tunisia', 'Turkey', 'Turkmenistan', 'Tuvalu', 'Uganda', 'Ukraine', 'United Arab Emirates', 'United Kingdom', 'United States of America', 'Uruguay', 'Uzbekistan', 'Vanuatu', 'Vatican City', 'Venezuela', 'Vietnam', 'Yemen', 'Zambia', 'Zimbabwe');
-
     public $organisationAssociations = array(
             'Correlation' => array('table' => 'correlations', 'fields' => array('org_id')),
             'Event' => array('table' => 'events', 'fields' => array('org_id', 'orgc_id')),
@@ -123,7 +121,7 @@ class Organisation extends AppModel
         }
         $this->data['Organisation']['date_modified'] = $date;
         if (!isset($this->data['Organisation']['nationality']) || empty($this->data['Organisation']['nationality'])) {
-            $this->data['Organisation']['nationality'] = 'Not specified';
+            $this->data['Organisation']['nationality'] = '';
         }
         return true;
     }
@@ -216,7 +214,7 @@ class Organisation extends AppModel
                 $changed = true;
             }
             if ($force) {
-                $fields = array('type', 'date_created', 'date_modified', 'nationality', 'sector', 'contacts', 'landingpage');
+                $fields = array('type', 'date_created', 'date_modified', 'nationality', 'sector', 'contacts');
                 foreach ($fields as $field) {
                     if (isset($org[$field])) {
                         if ($existingOrg['Organisation'][$field] != $org[$field]) {
@@ -473,4 +471,46 @@ class Organisation extends AppModel
         }
         return $suggestedOrg;
     }
+
+    private function getCountryGalaxyCluster()
+    {
+        static $list;
+        if (!$list) {
+            $file = new File(APP . '/files/misp-galaxy/clusters/country.json');
+            if ($file->exists()) {
+                $list = $this->jsonDecode($file->read())['values'];
+                $file->close();
+            } else {
+                $this->log("MISP Galaxy are not updated, countries will not be available.", LOG_WARNING);
+                $list = [];
+            }
+        }
+        return $list;
+    }
+
+    /**
+     * @param string $countryName
+     * @return string|null
+     */
+    public function getCountryCode($countryName)
+    {
+        foreach ($this->getCountryGalaxyCluster() as $country) {
+            if ($country['description'] === $countryName) {
+                return $country['meta']['ISO'];
+            }
+        }
+        return null;
+    }
+
+    /**
+     * @return string[]
+     */
+    public function getCountries()
+    {
+        $countries = ['International'];
+        foreach ($this->getCountryGalaxyCluster() as $country) {
+            $countries[] = $country['description'];
+        }
+        return $countries;
+    }
 }

app/Model/Server.php

@@ -4,6 +4,10 @@ App::uses('GpgTool', 'Tools');
 
 class Server extends AppModel
 {
+    const SETTING_CRITICAL = 0,
+        SETTING_RECOMMENDED = 1,
+        SETTING_OPTIONAL = 2;
+
     public $name = 'Server';
 
     public $actsAs = array('SysLogLogable.SysLogLogable' => array(
@@ -153,6 +157,7 @@ class Server extends AppModel
             ),
             'console_automation_tasks' => array(
                 'data' => array(
+                    'PullAll' => 'MISP/app/Console/cake Server pullAll [user_id] [full|update]',
                     'Pull' => 'MISP/app/Console/cake Server pull [user_id] [server_id] [full|update]',
                     'Push' => 'MISP/app/Console/cake Server push [user_id] [server_id]',
                     'Cache feeds for quick lookups' => 'MISP/app/Console/cake Server cacheFeed [user_id] [feed_id|all|csv|text|misp]',
@@ -542,6 +547,14 @@ class Server extends AppModel
                             'test' => 'testBool',
                             'type' => 'boolean',
                         ),
+                        'osuser' => array(
+                                'level' => 0,
+                                'description' => __('The Unix user MISP (php) is running as'),
+                                'value' => 'www-data',
+                                'errorMessage' => '',
+                                'test' => 'testForEmpty',
+                                'type' => 'string',
+                        ),
                         'email' => array(
                                 'level' => 0,
                                 'description' => __('The e-mail address that MISP should use for all notifications'),
@@ -1091,7 +1104,33 @@ class Server extends AppModel
                                'test' => 'testBool',
                                'type' => 'boolean',
                                'null' => true
-                       )
+                       ),
+                    'attachment_scan_module' => [
+                        'level' => self::SETTING_OPTIONAL,
+                        'description' => __('Name of enrichment module that will be used for attachment malware scanning. This module must return av-signature or sb-signature object.'),
+                        'value' => '',
+                        'errorMessage' => '',
+                        'type' => 'string',
+                        'null' => true,
+                    ],
+                    'attachment_scan_hash_only' => [
+                        'level' => self::SETTING_OPTIONAL,
+                        'description' => __('Send to attachment scan module just file hash. This can be useful if module sends attachment to remote service and you don\'t want to leak real data.'),
+                        'value' => false,
+                        'errorMessage' => '',
+                        'test' => 'testBool',
+                        'type' => 'boolean',
+                        'null' => true,
+                    ],
+                    'attachment_scan_timeout' => [
+                        'level' => self::SETTING_OPTIONAL,
+                        'description' => __('How long to wait for scan results in seconds.'),
+                        'value' => 30,
+                        'errorMessage' => '',
+                        'test' => 'testForPositiveInteger',
+                        'type' => 'numeric',
+                        'null' => true,
+                    ]
                 ),
                 'GnuPG' => array(
                         'branch' => 1,
@@ -1270,6 +1309,23 @@ class Server extends AppModel
                                 'editable' => false,
                                 'redacted' => true
                         ),
+                        'rest_client_enable_arbitrary_urls' => array(
+                            'level' => 0,
+                            'description' => __('Enable this setting if you wish for users to be able to query any arbitrary URL via the rest client. Keep in mind that queries are executed by the MISP server, so internal IPs in your MISP\'s network may be reachable.'),
+                            'value' => false,
+                            'errorMessage' => '',
+                            'test' => 'testBool',
+                            'type' => 'boolean',
+                            'null' => true
+                        ),
+                        'rest_client_baseurl' => array(
+                            'level' => 1,
+                            'description' => __('If left empty, the baseurl of your MISP is used. However, in some instances (such as port-forwarded VM installations) this will not work. You can override the baseurl with a url through which your MISP can reach itself (typically https://127.0.0.1 would work).'),
+                            'value' => false,
+                            'errorMessage' => '',
+                            'test' => null,
+                            'type' => 'string',
+                        ),
                         'syslog' => array(
                             'level' => 0,
                             'description' => __('Enable this setting to pass all audit log entries directly to syslog. Keep in mind, this is verbose and will include user, organisation, event data.'),
@@ -2242,6 +2298,14 @@ class Server extends AppModel
                                 'test' => 'testBool',
                                 'type' => 'boolean'
                         ),
+                        'Enrichment_hover_popover_only' => array(
+                            'level' => 0,
+                            'description' => __('When enabled, user have to click on magnifier icon to show enrichment'),
+                            'value' => false,
+                            'errorMessage' => '',
+                            'test' => 'testBool',
+                            'type' => 'boolean'
+                        ),
                         'Enrichment_hover_timeout' => array(
                                 'level' => 1,
                                 'description' => __('Set a timeout for the hover services'),
@@ -2464,8 +2528,7 @@ class Server extends AppModel
     {
         if (Configure::read('MISP.enableEventBlocklisting') !== false) {
             $this->EventBlocklist = ClassRegistry::init('EventBlocklist');
-            $r = $this->EventBlocklist->find('first', array('conditions' => array('event_uuid' => $event['Event']['uuid'])));
-            if (!empty($r)) {
+            if ($this->EventBlocklist->isBlocked($event['Event']['uuid'])) {
                 return true;
             }
         }
@@ -2528,6 +2591,18 @@ class Server extends AppModel
                     }
                 }
             }
+            if (isset($event['Event']['EventReport']) && !empty($event['Event']['EventReport'])) {
+                foreach ($event['Event']['EventReport'] as $key => $r) {
+                    switch ($r['distribution']) {
+                        case '1':
+                            $event['Event']['EventReport'][$key]['distribution'] = '0';
+                            break;
+                        case '2':
+                            $event['Event']['EventReport'][$key]['distribution'] = '1';
+                            break;
+                    }
+                }
+            }
         }
 
         // Distribution, set reporter of the event, being the admin that initiated the pull
@@ -2557,6 +2632,13 @@ class Server extends AppModel
                 }
             }
         }
+        if (!empty($event['Event']['EventReport'])) {
+            foreach ($event['Event']['EventReport'] as $report) {
+                if (empty($report['deleted'])) {
+                    return true;
+                }
+            }
+        }
         return false;
     }
 
@@ -2570,6 +2652,10 @@ class Server extends AppModel
             $result = $eventModel->_add($event, true, $user, $server['Server']['org_id'], $passAlong, true, $jobId);
             if ($result) {
                 $successes[] = $eventId;
+                if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) {
+                    $pubSubTool = $this->getPubSubTool();
+                    $pubSubTool->event_save(array('Event' => $eventId, 'Server' => $server['Server']['id']), 'add_from_connected_server');
+                }
             } else {
                 $fails[$eventId] = __('Failed (partially?) because of validation errors: ') . json_encode($eventModel->validationErrors, true);
             }
@@ -2580,6 +2666,10 @@ class Server extends AppModel
                 $result = $eventModel->_edit($event, $user, $existingEvent['Event']['id'], $jobId, $passAlong, $force);
                 if ($result === true) {
                     $successes[] = $eventId;
+                    if (Configure::read('Plugin.ZeroMQ_enable') && Configure::read('Plugin.ZeroMQ_event_notifications_enable')) {
+                        $pubSubTool = $this->getPubSubTool();
+                        $pubSubTool->event_save(array('Event' => $eventId, 'Server' => $server['Server']['id']), 'edit_from_connected_server');
+                    }
                 } elseif (isset($result['error'])) {
                     $fails[$eventId] = $result['error'];
                 } else {
@@ -2616,62 +2706,6 @@ class Server extends AppModel
         return true;
     }
 
-    private function __handlePulledProposals($proposals, $events, $job, $jobId, $eventModel, $user)
-    {
-        $pulledProposals = array();
-        if (!empty($proposals)) {
-            $shadowAttribute = ClassRegistry::init('ShadowAttribute');
-            $shadowAttribute->recursive = -1;
-            $uuidEvents = array_flip($events);
-            foreach ($proposals as $k => &$proposal) {
-                $proposal = $proposal['ShadowAttribute'];
-                $oldsa = $shadowAttribute->findOldProposal($proposal);
-                $proposal['event_id'] = $uuidEvents[$proposal['event_uuid']];
-                if (!$oldsa || $oldsa['timestamp'] < $proposal['timestamp']) {
-                    if ($oldsa) {
-                        $shadowAttribute->delete($oldsa['id']);
-                    }
-                    if (!isset($pulledProposals[$proposal['event_id']])) {
-                        $pulledProposals[$proposal['event_id']] = 0;
-                    }
-                    $pulledProposals[$proposal['event_id']]++;
-                    if (isset($proposal['old_id'])) {
-                        $oldAttribute = $eventModel->Attribute->find('first', array('recursive' => -1, 'conditions' => array('uuid' => $proposal['uuid'])));
-                        if ($oldAttribute) {
-                            $proposal['old_id'] = $oldAttribute['Attribute']['id'];
-                        } else {
-                            $proposal['old_id'] = 0;
-                        }
-                    }
-                    // check if this is a proposal from an old MISP instance
-                    if (!isset($proposal['Org']) && isset($proposal['org']) && !empty($proposal['org'])) {
-                        $proposal['Org'] = $proposal['org'];
-                        $proposal['EventOrg'] = $proposal['event_org'];
-                    } elseif (!isset($proposal['Org']) && !isset($proposal['EventOrg'])) {
-                        continue;
-                    }
-                    $proposal['org_id'] = $this->Organisation->captureOrg($proposal['Org'], $user);
-                    $proposal['event_org_id'] = $this->Organisation->captureOrg($proposal['EventOrg'], $user);
-                    unset($proposal['Org']);
-                    unset($proposal['EventOrg']);
-                    $shadowAttribute->create();
-                    if (!isset($proposal['deleted']) || !$proposal['deleted']) {
-                        if ($shadowAttribute->save($proposal)) {
-                            $shadowAttribute->sendProposalAlertEmail($proposal['event_id']);
-                        }
-                    }
-                }
-                if ($jobId) {
-                    if ($k % 50 == 0) {
-                        $job->id =  $jobId;
-                        $job->saveField('progress', 50 * (($k + 1) / count($proposals)) + 50);
-                    }
-                }
-            }
-        }
-        return $pulledProposals;
-    }
-
     public function pull($user, $id = null, $technique=false, $server, $jobId = false, $force = false)
     {
         if ($jobId) {
@@ -3125,11 +3159,16 @@ class Server extends AppModel
             if (empty($sgIds)) {
                 $sgIds = array(-1);
             }
+            $tableName = $this->Event->EventReport->table;
+            $eventReportQuery = sprintf('EXISTS (SELECT id, deleted FROM %s WHERE %s.event_id = Event.id and %s.deleted = 0)', $tableName, $tableName, $tableName);
             $findParams = array(
                     'conditions' => array(
                             $eventid_conditions_key => $eventid_conditions_value,
                             'Event.published' => 1,
-                            'Event.attribute_count >' => 0,
+                            'OR' => array(
+                                array('Event.attribute_count >' => 0),
+                                array($eventReportQuery),
+                            ),
                             'OR' => array(
                                 array(
                                     'AND' => array(
@@ -3442,7 +3481,7 @@ class Server extends AppModel
                         } else {
                             $setting['test'] = 'testForEmpty';
                             $setting['type'] = 'string';
-                            $setting['description'] = __('Set this required module specific setting.');
+                            $setting['description'] = isset($result['description']) ? $result['description'] : __('Set this required module specific setting.');
                             $setting['value'] = '';
                         }
                         $serverSettings['Plugin'][$moduleType . '_' . $module . '_' .  $result['name']] = $setting;
@@ -3612,6 +3651,14 @@ class Server extends AppModel
         return true;
     }
 
+    public function testForPositiveInteger($value)
+    {
+        if ((is_int($value) && $value >= 0) || ctype_digit($value)) {
+            return true;
+        }
+        return __('The value has to be a whole number greater or equal 0.');
+    }
+
     public function testForCookieTimeout($value)
     {
         $numeric = $this->testForNumeric($value);
@@ -3799,7 +3846,7 @@ class Server extends AppModel
         if ($this->testForEmpty($value) !== true) {
             return $this->testForEmpty($value);
         }
-        $regex = "%^(?<proto>https?)://(?<host>(?:(?:\w|-)+\.)+[a-z]{2,5})(?::(?<port>[0-9]+))?(?<base>/[a-z0-9_\-\.]+)?$%i";
+        $regex = "%^(?<proto>https?)://(?<host>(?:(?:\w|-)+\.)+[a-z]{2,})(?::(?<port>[0-9]+))?(?<base>/[a-z0-9_\-\.]+)?$%i";
 	if ( !preg_match($regex, $value, $matches)
                 || strtolower($matches['proto']) != strtolower($this->getProto())
                 || strtolower($matches['host']) != strtolower($this->getHost()) ) {
@@ -4471,96 +4518,99 @@ class Server extends AppModel
         return $validItems;
     }
 
-    public function runConnectionTest($id)
+    /**
+     * @param array $server
+     * @return array
+     * @throws JsonException
+     */
+    public function runConnectionTest(array $server)
     {
-        $server = $this->find('first', array('conditions' => array('Server.id' => $id)));
+        App::uses('SyncTool', 'Tools');
+        try {
+            $clientCertificate = SyncTool::getServerClientCertificateInfo($server);
+            if ($clientCertificate) {
+                $clientCertificate['valid_from'] = $clientCertificate['valid_from'] ? $clientCertificate['valid_from']->format('c') : __('Not defined');
+                $clientCertificate['valid_to'] = $clientCertificate['valid_to'] ? $clientCertificate['valid_to']->format('c') : __('Not defined');
+                $clientCertificate['public_key_size'] = $clientCertificate['public_key_size'] ?: __('Unknwon');
+                $clientCertificate['public_key_type'] = $clientCertificate['public_key_type'] ?: __('Unknwon');
+            }
+        } catch (Exception $e) {
+            $clientCertificate = ['error' => $e->getMessage()];
+        }
+
         $HttpSocket = $this->setupHttpSocket($server, null, 5);
         $request = $this->setupSyncRequest($server);
         $uri = $server['Server']['url'] . '/servers/getVersion';
+
         try {
             $response = $HttpSocket->get($uri, false, $request);
+            if ($response === false) {
+                throw new Exception("Connection failed for unknown reason.");
+            }
         } catch (Exception $e) {
-            $this->Log = ClassRegistry::init('Log');
-            $this->Log->create();
-            $this->Log->save(array(
-                    'org' => 'SYSTEM',
-                    'model' => 'Server',
-                    'model_id' => $id,
-                    'email' => 'SYSTEM',
-                    'action' => 'error',
-                    'user_id' => 0,
-                    'title' => 'Error: Connection test failed. Reason: ' . json_encode($e->getMessage()),
-            ));
-            return array('status' => 2);
+            $logTitle = 'Error: Connection test failed. Reason: ' .  $e->getMessage();
+            $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', $server['Server']['id'], $logTitle);
+            return array('status' => 2, 'client_certificate' => $clientCertificate);
         }
-        if ($response->isOk()) {
-            return array('status' => 1, 'message' => $response->body());
-        } else {
-            if ($response->code == '403') {
-                return array('status' => 4);
-            }
-            if ($response->code == '405') {
-                try {
-                    $responseText = $this->jsonDecode($response->body)['message'];
-                } catch (Exception $e) {
-                    return array('status' => 3);
-                }
+
+        if ($response->code == '403') {
+            return array('status' => 4, 'client_certificate' => $clientCertificate);
+        } else if ($response->code == '405') {
+            try {
+                $responseText = $this->jsonDecode($response->body)['message'];
                 if ($responseText === 'Your user account is expecting a password change, please log in via the web interface and change it before proceeding.') {
-                    return array('status' => 5);
+                    return array('status' => 5, 'client_certificate' => $clientCertificate);
                 } elseif ($responseText === 'You have not accepted the terms of use yet, please log in via the web interface and accept them.') {
-                    return array('status' => 6);
+                    return array('status' => 6, 'client_certificate' => $clientCertificate);
                 }
+            } catch (Exception $e) {
+                // pass
+            }
+        } else if ($response->isOk()) {
+            try {
+                $info = $this->jsonDecode($response->body());
+                if (!isset($info['version'])) {
+                    throw new Exception("Server returns JSON response, but doesn't contain required 'version' field.");
+                }
+                return array('status' => 1, 'info' => $info, 'client_certificate' => $clientCertificate);
+            } catch (Exception $e) {
+                // Even if server returns OK status, that doesn't mean that connection to another MISP instance works
             }
-            $this->Log = ClassRegistry::init('Log');
-            $this->Log->create();
-            $this->Log->save(array(
-                    'org' => 'SYSTEM',
-                    'model' => 'Server',
-                    'model_id' => $id,
-                    'email' => 'SYSTEM',
-                    'action' => 'error',
-                    'user_id' => 0,
-                    'title' => 'Error: Connection test failed. Returned data is in the change field.',
-                    'change' => sprintf(
-                        'response () => (%s), response-code () => (%s)',
-                        $response->body,
-                        $response->code
-                    )
-            ));
-            return array('status' => 3);
         }
+
+        $logTitle = 'Error: Connection test failed. Returned data is in the change field.';
+        $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', $server['Server']['id'], $logTitle, [
+            'response' => ['', $response->body],
+            'response-code' => ['', $response->code],
+        ]);
+        return array('status' => 3, 'client_certificate' => $clientCertificate);
     }
 
-    public function runPOSTtest($id)
+    /**
+     * @param array $server
+     * @return int
+     * @throws JsonException
+     */
+    public function runPOSTtest(array $server)
     {
-        $server = $this->find('first', array('conditions' => array('Server.id' => $id)));
-        if (empty($server)) {
-            throw new InvalidArgumentException(__('Invalid server.'));
+        $testFile = file_get_contents(APP . 'files/scripts/test_payload.txt');
+        if (!$testFile) {
+            throw new Exception("Could not load payload for POST test.");
         }
         $HttpSocket = $this->setupHttpSocket($server);
         $request = $this->setupSyncRequest($server);
-        $testFile = file_get_contents(APP . 'files/scripts/test_payload.txt');
         $uri = $server['Server']['url'] . '/servers/postTest';
-        $this->Log = ClassRegistry::init('Log');
+
         try {
             $response = $HttpSocket->post($uri, json_encode(array('testString' => $testFile)), $request);
             $rawBody = $response->body;
-            $response = json_decode($response, true);
+            $response = $this->jsonDecode($rawBody);
         } catch (Exception $e) {
-            $this->Log->create();
-            $this->Log->save(array(
-                    'org' => 'SYSTEM',
-                    'model' => 'Server',
-                    'model_id' => $id,
-                    'email' => 'SYSTEM',
-                    'action' => 'error',
-                    'user_id' => 0,
-                    'title' => 'Error: POST connection test failed. Reason: ' . json_encode($e->getMessage()),
-            ));
+            $title = 'Error: POST connection test failed. Reason: ' . $e->getMessage();
+            $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', $server['Server']['id'], $title);
             return 8;
         }
         if (!isset($response['body']['testString']) || $response['body']['testString'] !== $testFile) {
-            $responseString = '';
             if (!empty($repsonse['body']['testString'])) {
                 $responseString = $response['body']['testString'];
             } else if (!empty($rawBody)){
@@ -4568,32 +4618,17 @@ class Server extends AppModel
             } else {
                 $responseString = __('Response was empty.');
             }
-            $this->Log->create();
-            $this->Log->save(array(
-                    'org' => 'SYSTEM',
-                    'model' => 'Server',
-                    'model_id' => $id,
-                    'email' => 'SYSTEM',
-                    'action' => 'error',
-                    'user_id' => 0,
-                    'title' => 'Error: POST connection test failed due to the message body not containing the expected data. Response: ' . PHP_EOL . PHP_EOL . $responseString,
-            ));
+
+            $title = 'Error: POST connection test failed due to the message body not containing the expected data. Response: ' . PHP_EOL . PHP_EOL . $responseString;
+            $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', $server['Server']['id'], $title);
             return 9;
         }
         $headers = array('Accept', 'Content-type');
         foreach ($headers as $header) {
             if (!isset($response['headers'][$header]) || $response['headers'][$header] != 'application/json') {
                 $responseHeader = isset($response['headers'][$header]) ? $response['headers'][$header] : 'Header was not set.';
-                $this->Log->create();
-                $this->Log->save(array(
-                        'org' => 'SYSTEM',
-                        'model' => 'Server',
-                        'model_id' => $id,
-                        'email' => 'SYSTEM',
-                        'action' => 'error',
-                        'user_id' => 0,
-                        'title' => 'Error: POST connection test failed due to a header not matching the expected value. Expected: "application/json", received "' . $responseHeader,
-                ));
+                $title = 'Error: POST connection test failed due to a header not matching the expected value. Expected: "application/json", received "' . $responseHeader . '"';
+                $this->loadLog()->createLogEntry('SYSTEM', 'error', 'Server', $server['Server']['id'], $title);
                 return 10;
             }
         }
@@ -5373,9 +5408,8 @@ class Server extends AppModel
         $gpgStatus = 0;
         if (Configure::read('GnuPG.email') && Configure::read('GnuPG.homedir')) {
             $continue = true;
-            $gpgTool = new GpgTool();
             try {
-                $gpg = $gpgTool->initializeGpg();
+                $gpg = GpgTool::initializeGpg();
             } catch (Exception $e) {
                 $this->logException("Error during initializing GPG.", $e, LOG_NOTICE);
                 $gpgStatus = 2;
@@ -5428,13 +5462,12 @@ class Server extends AppModel
     public function moduleDiagnostics(&$diagnostic_errors, $type = 'Enrichment')
     {
         $this->Module = ClassRegistry::init('Module');
-        $types = array('Enrichment', 'Import', 'Export', 'Cortex');
         $diagnostic_errors++;
         if (Configure::read('Plugin.' . $type . '_services_enable')) {
-            $exception = false;
-            $result = $this->Module->getModules(false, $type, $exception);
-            if ($exception) {
-                return $exception;
+            try {
+                $result = $this->Module->getModules($type, true);
+            } catch (Exception $e) {
+                return $e->getMessage();
             }
             if (empty($result)) {
                 return 2;
@@ -5493,18 +5526,19 @@ class Server extends AppModel
 
     public function workerDiagnostics(&$workerIssueCount)
     {
+        $worker_array = array(
+            'cache' => array('ok' => false),
+            'default' => array('ok' => false),
+            'email' => array('ok' => false),
+            'prio' => array('ok' => false),
+            'update' => array('ok' => false),
+            'scheduler' => array('ok' => false)
+        );
         try {
             $this->ResqueStatus = new ResqueStatus\ResqueStatus(Resque::redis());
         } catch (Exception $e) {
             // redis connection failed
-            return array(
-                    'cache' => array('ok' => false),
-                    'default' => array('ok' => false),
-                    'email' => array('ok' => false),
-                    'prio' => array('ok' => false),
-                    'update' => array('ok' => false),
-                    'scheduler' => array('ok' => false)
-            );
+            return $worker_array;
         }
         $workers = $this->ResqueStatus->getWorkers();
         if (function_exists('posix_getpwuid')) {
@@ -5513,14 +5547,6 @@ class Server extends AppModel
         } else {
             $currentUser = trim(shell_exec('whoami'));
         }
-        $worker_array = array(
-                'cache' => array('ok' => true),
-                'default' => array('ok' => true),
-                'email' => array('ok' => true),
-                'prio' => array('ok' => true),
-                'update' => array('ok' => true),
-                'scheduler' => array('ok' => true)
-        );
         $procAccessible = file_exists('/proc');
         foreach ($workers as $pid => $worker) {
             $entry = ($worker['type'] == 'regular') ? $worker['queue'] : $worker['type'];
@@ -5538,7 +5564,13 @@ class Server extends AppModel
                 $ok = false;
                 $workerIssueCount++;
             }
-            $worker_array[$entry]['workers'][] = array('pid' => $pid, 'user' => $worker['user'], 'alive' => $alive, 'correct_user' => $correct_user, 'ok' => $ok);
+            $worker_array[$entry]['workers'][] = array(
+                'pid' => $pid,
+                'user' => $worker['user'],
+                'alive' => $alive,
+                'correct_user' => $correct_user,
+                'ok' => $ok
+            );
         }
         foreach ($worker_array as $k => $queue) {
             if (isset($worker_array[$k]['workers'])) {

app/Model/ShadowAttribute.php

@@ -1,12 +1,13 @@
 <?php
-
 App::uses('AppModel', 'Model');
 App::uses('Folder', 'Utility');
 App::uses('File', 'Utility');
 App::uses('AttachmentTool', 'Tools');
+App::uses('ComplexTypeTool', 'Tools');
 
 /**
  * @property Event $Event
+ * @property Attribute $Attribute
  */
 class ShadowAttribute extends AppModel
 {
@@ -58,7 +59,6 @@ class ShadowAttribute extends AppModel
     // explanations of certain fields to be used in various views
     public $fieldDescriptions = array(
             'signature' => array('desc' => 'Is this attribute eligible to automatically create an IDS signature (network IDS or host IDS) out of it ?'),
-            //'private' => array('desc' => 'Prevents upload of this single Attribute to other CyDefSIG servers', 'formdesc' => 'Prevents upload of <em>this single Attribute</em> to other CyDefSIG servers.<br/>Used only when the Event is NOT set as Private')
     );
 
     // if these then a category my have upload to be zipped
@@ -122,7 +122,7 @@ class ShadowAttribute extends AppModel
         ),
         'to_ids' => array(
             'boolean' => array(
-                'rule' => array('boolean'),
+                'rule' => 'boolean',
                 'required' => false,
             ),
         ),
@@ -133,9 +133,9 @@ class ShadowAttribute extends AppModel
             ),
         ),
         'proposal_to_delete' => array(
-                'boolean' => array(
-                        'rule' => array('boolean'),
-                ),
+            'boolean' => array(
+                'rule' => 'boolean',
+            ),
         ),
         'first_seen' => array(
             'rule' => array('datetimeOrNull'),
@@ -152,15 +152,8 @@ class ShadowAttribute extends AppModel
     public function __construct($id = false, $table = null, $ds = null)
     {
         parent::__construct($id, $table, $ds);
-        $this->virtualFields = Set::merge($this->virtualFields, array(
-            //'distribution' => 'IF (Attribute.private=true, "Your organization only", IF (Attribute.cluster=true, "This Community-only", "All communities"))',
-            //'distribution' => 'IF (ShadowAttribute.private=true AND ShadowAttribute.cluster=false, "Your organization only", IF (ShadowAttribute.private=true AND ShadowAttribute.cluster=true, "This server-only", IF (ShadowAttribute.private=false AND ShadowAttribute.cluster=true, "This Community-only", IF (ShadowAttribute.communitie=true, "Connected communities" , "All communities"))))',
-        ));
-        $this->fieldDescriptions = Set::merge($this->fieldDescriptions, array(
-            //'distribution' => array('desc' => 'This fields indicates the intended distribution of the attribute (same as when adding an event, see Add Event)'),
-        ));
-        $this->categoryDefinitions = $this->Event->Attribute->categoryDefinitions;
-        $this->typeDefinitions = $this->Event->Attribute->typeDefinitions;
+        $this->categoryDefinitions = $this->Attribute->categoryDefinitions;
+        $this->typeDefinitions = $this->Attribute->typeDefinitions;
     }
 
     // The Associations below have been created with all possible keys, those that are not needed can be removed
@@ -211,7 +204,7 @@ class ShadowAttribute extends AppModel
         if (isset($sa['ShadowAttribute'])) {
             $sa = $sa['ShadowAttribute'];
         }
-        if (in_array($sa['type'], $this->Event->Attribute->nonCorrelatingTypes)) {
+        if (in_array($sa['type'], $this->Attribute->nonCorrelatingTypes)) {
             return;
         }
         $this->ShadowAttributeCorrelation = ClassRegistry::init('ShadowAttributeCorrelation');
@@ -222,14 +215,14 @@ class ShadowAttribute extends AppModel
             $correlatingValues[] = $sa['value2'];
         }
         foreach ($correlatingValues as $k => $cV) {
-            $correlatingAttributes[$k] = $this->Event->Attribute->find('all', array(
+            $correlatingAttributes[$k] = $this->Attribute->find('all', array(
                     'conditions' => array(
                             'AND' => array(
                                     'OR' => array(
                                             'Attribute.value1' => $cV,
                                             'Attribute.value2' => $cV
                                     ),
-                                    'Attribute.type !=' => $this->Event->Attribute->nonCorrelatingTypes,
+                                    'Attribute.type !=' => $this->Attribute->nonCorrelatingTypes,
                                     'Attribute.deleted' => 0,
                                     'Attribute.event_id !=' => $sa['event_id']
                             ),
@@ -304,9 +297,6 @@ class ShadowAttribute extends AppModel
         parent::beforeValidate();
         // remove leading and trailing blanks
         //$this->trimStringFields(); // TODO
-        if (isset($this->data['ShadowAttribute']['value'])) {
-            $this->data['ShadowAttribute']['value'] = trim($this->data['ShadowAttribute']['value']);
-        }
 
         if (!isset($this->data['ShadowAttribute']['comment'])) {
             $this->data['ShadowAttribute']['comment'] = '';
@@ -316,6 +306,18 @@ class ShadowAttribute extends AppModel
             return false;
         }
 
+        // make some changes to the inserted value
+        if (isset($this->data['ShadowAttribute']['value'])) {
+            $value = trim($this->data['ShadowAttribute']['value']);
+            $value = ComplexTypeTool::refangValue($value, $this->data['ShadowAttribute']['type']);
+            $value = $this->Attribute->modifyBeforeValidation($this->data['ShadowAttribute']['type'], $value);
+            $this->data['ShadowAttribute']['value'] = $value;
+        }
+
+        if (!isset($this->data['ShadowAttribute']['org'])) {
+            $this->data['ShadowAttribute']['org'] = '';
+        }
+
         if (empty($this->data['ShadowAttribute']['timestamp'])) {
             $date = new DateTime();
             $this->data['ShadowAttribute']['timestamp'] = $date->getTimestamp();
@@ -325,9 +327,6 @@ class ShadowAttribute extends AppModel
             $this->data['ShadowAttribute']['proposal_to_delete'] = 0;
         }
 
-        // make some last changes to the inserted value
-        $this->data['ShadowAttribute']['value'] = $this->Event->Attribute->modifyBeforeValidation($this->data['ShadowAttribute']['type'], $this->data['ShadowAttribute']['value']);
-
         // generate UUID if it doesn't exist
         if (empty($this->data['ShadowAttribute']['uuid'])) {
             $this->data['ShadowAttribute']['uuid'] = CakeText::uuid();
@@ -336,7 +335,7 @@ class ShadowAttribute extends AppModel
         }
 
         if (!empty($this->data['ShadowAttribute']['type']) && empty($this->data['ShadowAttribute']['category'])) {
-            $this->data['ShadowAttribute']['category'] = $this->Event->Attribute->typeDefinitions[$this->data['ShadowAttribute']['type']]['default_category'];
+            $this->data['ShadowAttribute']['category'] = $this->Attribute->typeDefinitions[$this->data['ShadowAttribute']['type']]['default_category'];
         }
 
         // always return true, otherwise the object cannot be saved
@@ -362,38 +361,28 @@ class ShadowAttribute extends AppModel
 
     public function validCategory($fields)
     {
-        return $this->Event->Attribute->validCategory($fields);
+        return $this->Attribute->validCategory($fields);
     }
 
     public function validateAttributeValue($fields)
     {
         $value = $fields['value'];
-        return $this->Event->Attribute->runValidation($value, $this->data['ShadowAttribute']['type']);
+        return $this->Attribute->runValidation($value, $this->data['ShadowAttribute']['type']);
     }
 
     public function getCompositeTypes()
     {
-        // build the list of composite Attribute.type dynamically by checking if type contains a |
-        // default composite types
-        $compositeTypes = array('malware-sample');  // TODO hardcoded composite
-        // dynamically generated list
-        foreach (array_keys($this->typeDefinitions) as $type) {
-            $pieces = explode('|', $type);
-            if (2 == count($pieces)) {
-                $compositeTypes[] = $type;
-            }
-        }
-        return $compositeTypes;
+        return $this->Attribute->getCompositeTypes();
     }
 
     public function typeIsMalware($type)
     {
-        return in_array($type, $this->zippedDefinitions);
+        return $this->Attribute->typeIsAttachment($type);
     }
 
     public function typeIsAttachment($type)
     {
-        return in_array($type, $this->zippedDefinitions) || in_array($type, $this->uploadDefinitions);
+        return $this->Attribute->typeIsAttachment($type);
     }
 
     public function base64EncodeAttachment(array $attribute)
@@ -414,7 +403,11 @@ class ShadowAttribute extends AppModel
     public function saveBase64EncodedAttachment($attribute)
     {
         $data = base64_decode($attribute['data']);
-        return $this->loadAttachmentTool()->saveShadow($attribute['event_id'], $attribute['id'], $data);
+        $result = $this->loadAttachmentTool()->saveShadow($attribute['event_id'], $attribute['id'], $data);
+        if ($result) {
+            $this->loadAttachmentScan()->backgroundScan(AttachmentScan::TYPE_SHADOW_ATTRIBUTE, $attribute);
+        }
+        return $result;
     }
 
     /**
@@ -445,15 +438,7 @@ class ShadowAttribute extends AppModel
     // check whether the variable is null or datetime
     public function datetimeOrNull($fields)
     {
-        $k = array_keys($fields)[0];
-        $seen = $fields[$k];
-        try {
-            new DateTime($seen);
-            $returnValue = true;
-        } catch (Exception $e) {
-            $returnValue = false;
-        }
-        return $returnValue || is_null($seen);
+        return $this->Attribute->datetimeOrNull($fields);
     }
 
     public function setDeleted($id)
@@ -490,22 +475,29 @@ class ShadowAttribute extends AppModel
         }
     }
 
-    public function getEventContributors($id)
+    /**
+     * @param int $eventId
+     * @return array Key is organisation ID, value is an organisation name
+     */
+    public function getEventContributors($eventId)
     {
-        $orgs = $this->find('all', array('fields' => array(
-            'DISTINCT(ShadowAttribute.org_id)'),
-            'conditions' => array('event_id' => $id),
+        $orgs = $this->find('all', array(
+            'fields' => array('DISTINCT(ShadowAttribute.org_id)'),
+            'conditions' => array('event_id' => $eventId),
             'recursive' => -1,
             'order' => false
         ));
-        $org_ids = array();
-        $this->Organisation = ClassRegistry::init('Organisation');
-        foreach ($orgs as $org) {
-            $org_ids[] = $this->Organisation->find('first', array('recursive' => -1, 'fields' => array('id', 'name'), 'conditions' => array('Organisation.id' => $org['ShadowAttribute']['org_id'])));
+        if (empty($orgs)) {
+            return [];
         }
-        return $org_ids;
-    }
 
+        $this->Organisation = ClassRegistry::init('Organisation');
+        return $this->Organisation->find('list', array(
+            'recursive' => -1,
+            'fields' => array('id', 'name'),
+            'conditions' => array('Organisation.id' => Hash::extract($orgs, "{n}.ShadowAttribute.org_id")))
+        );
+    }
 
     /**
      * Sends an email to members of the organization that owns the event
@@ -710,7 +702,7 @@ class ShadowAttribute extends AppModel
                     'OR' => array(
                         'Event.org_id' => $user['org_id'],
                         ['AND' => [
-                            'Event.distribution' => array(1,2,3,5),
+                            'Event.distribution' => array(1,2,3),
                             $unpublishedPrivate ? ['Event.published' => 1] : [],
                         ]],
                         ['AND' => [

app/Model/Sighting.php

@@ -5,6 +5,7 @@ App::uses('TmpFileTool', 'Tools');
 
 /**
  * @property Attribute $Attribute
+ * @property Event $Event
  */
 class Sighting extends AppModel
 {
@@ -21,6 +22,7 @@ class Sighting extends AppModel
         'attribute_id' => 'numeric',
         'org_id' => 'numeric',
         'date_sighting' => 'numeric',
+        'uuid' => 'uuid',
         'type' => array(
             'rule' => array('inList', array(0, 1, 2)),
             'message' => 'Invalid type. Valid options are: 0 (Sighting), 1 (False-positive), 2 (Expiration).'
@@ -51,12 +53,13 @@ class Sighting extends AppModel
     public function beforeValidate($options = array())
     {
         parent::beforeValidate();
-        $date = date('Y-m-d H:i:s');
         if (empty($this->data['Sighting']['id']) && empty($this->data['Sighting']['date_sighting'])) {
-            $this->data['Sighting']['date_sighting'] = $date;
+            $this->data['Sighting']['date_sighting'] = date('Y-m-d H:i:s');
         }
         if (empty($this->data['Sighting']['uuid'])) {
             $this->data['Sighting']['uuid'] = CakeText::uuid();
+        } else {
+            $this->data['Sighting']['uuid'] = strtolower($this->data['Sighting']['uuid']);
         }
         return true;
     }
@@ -208,15 +211,13 @@ class Sighting extends AppModel
      * @param array $event
      * @param array $user
      * @param array|int|null $attribute Attribute model or attribute ID
-     * @param bool $extraConditions
+     * @param array|bool $extraConditions
+     * @param bool $forSync
      * @return array|int
      */
     public function attachToEvent(array $event, array $user, $attribute = null, $extraConditions = false, $forSync = false)
     {
-        $ownEvent = false;
-        if ($user['Role']['perm_site_admin'] || $event['Event']['org_id'] == $user['org_id']) {
-            $ownEvent = true;
-        }
+        $ownEvent = $user['Role']['perm_site_admin'] || $event['Event']['org_id'] == $user['org_id'];
 
         $contain = [];
         $conditions = array('Sighting.event_id' => $event['Event']['id']);
@@ -247,16 +248,22 @@ class Sighting extends AppModel
         // If the event has any sightings for the user's org, then the user is a sighting reporter for the event too.
         // This means that he /she has access to the sightings data contained within
         if (!$ownEvent && Configure::read('Plugin.Sightings_policy') == 1) {
-            $temp = $this->find('first', array('recursive' => -1, 'conditions' => array('Sighting.event_id' => $event['Event']['id'], 'Sighting.org_id' => $user['org_id'])));
+            $temp = $this->find('first', array(
+                'recursive' => -1,
+                'conditions' => array(
+                    'Sighting.event_id' => $event['Event']['id'],
+                    'Sighting.org_id' => $user['org_id'],
+                )
+            ));
             if (empty($temp)) {
                 return array();
             }
         }
 
         $sightings = $this->find('all', array(
-                'conditions' => $conditions,
-                'recursive' => -1,
-                'contain' => $contain,
+            'conditions' => $conditions,
+            'recursive' => -1,
+            'contain' => $contain,
         ));
         if (empty($sightings)) {
             return array();
@@ -279,95 +286,30 @@ class Sighting extends AppModel
             ) {
                 if ($sighting['Sighting']['org_id'] != $user['org_id']) {
                     if (empty($anonOrg)) {
-                        unset($sightings[$k]['Sighting']['org_id']);
-                        unset($sightings[$k]['Organisation']);
+                        unset($sighting['Sighting']['org_id']);
+                        unset($sighting['Organisation']);
                     } else {
-                        $sightings[$k]['Sighting']['org_id'] = $anonOrg['Organisation']['id'];
-                        $sightings[$k]['Organisation'] = $anonOrg['Organisation'];
+                        $sighting['Sighting']['org_id'] = $anonOrg['Organisation']['id'];
+                        $sighting['Organisation'] = $anonOrg['Organisation'];
                     }
                 }
             }
             // rearrange it to match the event format of fetchevent
-            if (isset($sightings[$k]['Organisation'])) {
-                $sightings[$k]['Sighting']['Organisation'] = $sightings[$k]['Organisation'];
+            if (isset($sighting['Organisation'])) {
+                $sighting['Sighting']['Organisation'] = $sighting['Organisation'];
             }
             // zeroq: add attribute UUID to sighting to make synchronization easier
             if (isset($sighting['Attribute']['uuid'])) {
-                $sightings[$k]['Sighting']['attribute_uuid'] = $sighting['Attribute']['uuid'];
+                $sighting['Sighting']['attribute_uuid'] = $sighting['Attribute']['uuid'];
             } else {
-                $sightings[$k]['Sighting']['attribute_uuid'] = $attribute['Attribute']['uuid'];
+                $sighting['Sighting']['attribute_uuid'] = $attribute['Attribute']['uuid'];
             }
 
-            $sightings[$k] = $sightings[$k]['Sighting'] ;
+            $sightings[$k] = $sighting['Sighting'] ;
         }
         return $sightings;
     }
 
-    /*
-     * Loop through all attributes of an event, including those in objects
-     * and pass each value to SightingDB. If there's a hit, append the data
-     * directly to the attributes
-     */
-    public function attachSightingDB($event, $user)
-    {
-        if (!empty(Configure::read('Plugin.Sightings_sighting_db_enable'))) {
-            $host = empty(Configure::read('Plugin.Sightings_sighting_db_host')) ? 'localhost' : Configure::read('Plugin.Sightings_sighting_db_host');
-            $port = empty(Configure::read('Plugin.Sightings_sighting_db_port')) ? 9999 : Configure::read('Plugin.Sightings_sighting_db_port');
-            App::uses('SyncTool', 'Tools');
-            $syncTool = new SyncTool();
-            $params = array(
-                'ssl_verify_peer' => false,
-                'ssl_verify_peer_name' => false,
-                'ssl_verify_host' => false
-            );
-            $HttpSocket = $syncTool->createHttpSocket($params);
-            if (!empty($event['Attribute'])) {
-                $event['Attribute'] = $this->__attachSightingDBToAttribute($event['Attribute'], $user, $host, $port, $HttpSocket);
-            }
-            if (!empty($event['Object'])) {
-                foreach ($event['Object'] as &$object) {
-                    if (!empty($object['Attribute'])) {
-                        $object['Attribute'] = $this->__attachSightingDBToAttribute($object['Attribute'], $user, $host, $port, $HttpSocket);
-                    }
-                }
-            }
-        }
-        return $event;
-    }
-
-    private function __attachSightingDBToAttribute($attributes, $user, $host, $port, $HttpSocket = false)
-    {
-        if (empty($HttpSocket)) {
-            App::uses('SyncTool', 'Tools');
-            $syncTool = new SyncTool();
-            $params = array(
-                'ssl_allow_self_signed' => true,
-                'ssl_verify_peer' => false
-            );
-            $HttpSocket = $syncTool->createHttpSocket($params);
-        }
-        foreach($attributes as &$attribute) {
-            $response = $HttpSocket->get(
-                sprintf(
-                    '%s:%s/r/all/%s?val=%s',
-                    $host,
-                    $port,
-                    $attribute['type'],
-                    rtrim(str_replace('/', '_', str_replace('+', '-', base64_encode($attribute['value']))), '=')
-                )
-            );
-            if ($response->code == 200) {
-                $responseData = json_decode($response->body, true);
-                if ($responseData !== false) {
-                    if (!isset($responseData['error'])) {
-                        $attribute['SightingDB'] = $responseData;
-                    }
-                }
-            }
-        }
-        return $attributes;
-    }
-
     public function saveSightings($id, $values, $timestamp, $user, $type = false, $source = false, $sighting_uuid = false, $publish = false, $saveOnBehalfOf = false)
     {
         if (!in_array($type, array(0, 1, 2))) {
@@ -379,7 +321,8 @@ class Sighting extends AppModel
             // if sighting with given uuid already exists and quit early
             $existing_sighting = $this->find('count', array(
                 'recursive' => -1,
-                'conditions' => array('uuid' => $sighting_uuid)
+                'conditions' => array('uuid' => $sighting_uuid),
+                'callbacks' => false,
             ));
             if ($existing_sighting) {
                 return 0;
@@ -409,7 +352,10 @@ class Sighting extends AppModel
                 }
             }
         }
-        $attributes = $this->Attribute->fetchAttributes($user, array('conditions' => $conditions, 'flatten' => 1));
+        $attributes = $this->Attribute->fetchAttributesSimple($user, [
+            'conditions' => $conditions,
+            'fields' => ['Attribute.id', 'Attribute.event_id'],
+        ]);
         if (empty($attributes)) {
             return 'No valid attributes found that match the criteria.';
         }
@@ -417,16 +363,20 @@ class Sighting extends AppModel
         foreach ($attributes as $attribute) {
             if ($type === '2') {
                 // remove existing expiration by the same org if it exists
-                $this->deleteAll(array('Sighting.org_id' => $user['org_id'], 'Sighting.type' => $type, 'Sighting.attribute_id' => $attribute['Attribute']['id']));
+                $this->deleteAll(array(
+                    'Sighting.org_id' => $user['org_id'],
+                    'Sighting.type' => $type,
+                    'Sighting.attribute_id' => $attribute['Attribute']['id'],
+                ));
             }
             $this->create();
             $sighting = array(
-                    'attribute_id' => $attribute['Attribute']['id'],
-                    'event_id' => $attribute['Attribute']['event_id'],
-                    'org_id' => ($saveOnBehalfOf === false) ? $user['org_id'] : $saveOnBehalfOf,
-                    'date_sighting' => $timestamp,
-                    'type' => $type,
-                    'source' => $source
+                'attribute_id' => $attribute['Attribute']['id'],
+                'event_id' => $attribute['Attribute']['event_id'],
+                'org_id' => ($saveOnBehalfOf === false) ? $user['org_id'] : $saveOnBehalfOf,
+                'date_sighting' => $timestamp,
+                'type' => $type,
+                'source' => $source,
             );
             // zeroq: allow setting a specific uuid
             if ($sighting_uuid) {
@@ -471,11 +421,6 @@ class Sighting extends AppModel
         return $result;
     }
 
-    public function generateRandomFileName()
-    {
-        return (new RandomTool())->random_str(false, 12);
-    }
-
     public function addUuids()
     {
         $sightings = $this->find('all', array(
@@ -538,11 +483,18 @@ class Sighting extends AppModel
         return $sightingsRearranged;
     }
 
-    public function getSightingsForObjectIds($user, $tagList, $context = 'event', $type = '0')
+    /**
+     * @param $user - not used
+     * @param array $tagIds
+     * @param string $context 'event' or 'attribute'
+     * @param string|false $type
+     * @return array
+     */
+    public function getSightingsForObjectIds($user, array $tagIds, $context = 'event', $type = '0')
     {
         $conditions = array(
             'Sighting.date_sighting >' => $this->getMaximumRange(),
-            ucfirst($context) . 'Tag.tag_id' => $tagList
+            ucfirst($context) . 'Tag.tag_id' => $tagIds
         );
         if ($type !== false) {
             $conditions['Sighting.type'] = $type;
@@ -799,10 +751,7 @@ class Sighting extends AppModel
      */
     public function bulkSaveSightings($eventId, $sightings, $user, $passAlong = null)
     {
-        $event = $this->Event->fetchEvent($user, array_merge(array(
-             'metadata' => 1,
-             'flatten' => true,
-        ), is_string($eventId) ? array('event_uuid' => $eventId) : array('eventid' => $eventId)));
+        $event = $this->Event->fetchSimpleEvent($user, $eventId);
         if (empty($event)) {
             return 'Event not found or not accessible by this user.';
         }
@@ -824,7 +773,7 @@ class Sighting extends AppModel
             }
         }
         if ($saved > 0) {
-            $this->Event->publishRouter($eventId, $passAlong, $user, 'sightings');
+            $this->Event->publishRouter($event['Event']['id'], $passAlong, $user, 'sightings');
         }
         return $saved;
     }

app/Model/Tag.php

@@ -1,10 +1,6 @@
 <?php
 App::uses('AppModel', 'Model');
 
-/**
- * @property EventTag $EventTag
- * @property AttributeTag $AttributeTag
- */
 class Tag extends AppModel
 {
     public $useTable = 'tags';

app/Model/Taxonomy.php

@@ -156,6 +156,7 @@ class Taxonomy extends AppModel
                     if (isset($entry['numerical_value']) && $entry['numerical_value'] !== null) {
                         $temp['numerical_value'] = $entry['numerical_value'];
                     }
+                    $temp['exclusive_predicate'] = $predicate['exclusive'];
                     $entries[] = $temp;
                 }
             } else {

app/Model/User.php

@@ -222,7 +222,7 @@ class User extends AppModel
         'Containable'
     );
 
-    /** @var Crypt_GPG|null|false */
+    /** @var CryptGpgExtended|null|false */
     private $gpg;
 
     public function beforeValidate($options = array())
@@ -302,9 +302,11 @@ class User extends AppModel
         return true;
     }
 
-    // Checks if the GnuPG key is a valid key, but also import it in the keychain.
-    // this will NOT fail on keys that can only be used for signing but not encryption!
-    // the method in verifyUsers will fail in that case.
+    /**
+     * Checks if the GnuPG key is a valid key.
+     * @param array $check
+     * @return bool
+     */
     public function validateGpgkey($check)
     {
         // LATER first remove the old gpgkey from the keychain
@@ -319,12 +321,11 @@ class User extends AppModel
             return true;
         }
         try {
-            $keyImportOutput = $gpg->importKey($check['gpgkey']);
-            if (!empty($keyImportOutput['fingerprint'])) {
-                return true;
-            }
+            $gpgTool = new GpgTool($gpg);
+            $gpgTool->validateGpgKey($check['gpgkey']);
+            return true;
         } catch (Exception $e) {
-            $this->logException("Exception during importing GPG key", $e);
+            $this->logException("Exception during validating GPG key", $e, LOG_NOTICE);
             return false;
         }
     }
@@ -452,25 +453,40 @@ class User extends AppModel
                 )));
     }
 
-    public function verifySingleGPG($user, $gpg = null)
+    /**
+     * 0 - true if key is valid
+     * 1 - User e-mail
+     * 2 - Error message
+     * 3 - Not used
+     * 4 - Key fingerprint
+     * 5 - Key fingerprint
+     * @param array $user
+     * @return array
+     */
+    public function verifySingleGPG(array $user)
     {
-        if ($gpg === null) {
-            $gpg = $this->initializeGpg();
-            if (!$gpg) {
-                $result[2] = 'GnuPG is not configured on this system.';
-                $result[0] = true;
-                return $result;
-            }
+        $result = [0 => false, 1 => $user['User']['email']];
+
+        $gpg = $this->initializeGpg();
+        if (!$gpg) {
+            $result[2] = 'GnuPG is not configured on this system.';
+            return $result;
         }
-        $result = array();
+
         try {
             $currentTimestamp = time();
-            $temp = $gpg->importKey($user['User']['gpgkey']);
-            $key = $gpg->getKeys($temp['fingerprint']);
-            $result[5] = $temp['fingerprint'];
-            $subKeys = $key[0]->getSubKeys();
-            $sortedKeys = array('valid' => 0, 'expired' => 0, 'noEncrypt' => 0);
-            foreach ($subKeys as $subKey) {
+            $keys = $gpg->keyInfo($user['User']['gpgkey']);
+            if (count($keys) !== 1) {
+                $result[2] = 'Multiple or no key found';
+                return $result;
+            }
+
+            $key = $keys[0];
+            $result[4] = $key->getPrimaryKey()->getFingerprint();
+            $result[5] = $result[4];
+
+            $sortedKeys = ['valid' => 0, 'expired' => 0, 'noEncrypt' => 0];
+            foreach ($key->getSubKeys() as $subKey) {
                 $expiration = $subKey->getExpirationDate();
                 if ($expiration != 0 && $currentTimestamp > $expiration) {
                     $sortedKeys['expired']++;
@@ -490,14 +506,12 @@ class User extends AppModel
                 if ($sortedKeys['noEncrypt']) {
                     $result[2] .= ' Found ' . $sortedKeys['noEncrypt'] . ' subkey(s) that are sign only.';
                 }
+            } else {
                 $result[0] = true;
             }
         } catch (Exception $e) {
             $result[2] = $e->getMessage();
-            $result[0] = true;
         }
-        $result[1] = $user['User']['email'];
-        $result[4] = $temp['fingerprint'];
         return $result;
     }
 
@@ -521,7 +535,7 @@ class User extends AppModel
         }
         $results = [];
         foreach ($users as $k => $user) {
-            $results[$user['User']['id']] = $this->verifySingleGPG($user, $gpg);
+            $results[$user['User']['id']] = $this->verifySingleGPG($user);
         }
         return $results;
     }
@@ -817,7 +831,7 @@ class User extends AppModel
      */
     public function searchGpgKey($email)
     {
-        $gpgTool = new GpgTool();
+        $gpgTool = new GpgTool(null);
         return $gpgTool->searchGpgKey($email);
     }
 
@@ -828,7 +842,7 @@ class User extends AppModel
      */
     public function fetchGpgKey($fingerprint)
     {
-        $gpgTool = new GpgTool();
+        $gpgTool = new GpgTool($this->initializeGpg());
         return $gpgTool->fetchGpgKey($fingerprint);
     }
 
@@ -1310,10 +1324,28 @@ class User extends AppModel
         }
     }
 
+    /**
+     * Updates `current_login` and `last_login` time in database.
+     *
+     * @param array $user
+     * @return array|bool
+     * @throws Exception
+     */
+    public function updateLoginTimes(array $user)
+    {
+        if (!isset($user['id'])) {
+            throw new InvalidArgumentException("Invalid user object provided.");
+        }
+        $user['action'] = 'login'; // for afterSave callbacks
+        $user['last_login'] = $user['current_login'];
+        $user['current_login'] = time();
+        return $this->save($user, true, array('id', 'last_login', 'current_login'));
+    }
+
     /**
      * Initialize GPG. Returns `null` if initialization failed.
      *
-     * @return null|Crypt_GPG
+     * @return null|CryptGpgExtended
      */
     private function initializeGpg()
     {
@@ -1326,8 +1358,7 @@ class User extends AppModel
         }
 
         try {
-            $gpgTool = new GpgTool();
-            $this->gpg = $gpgTool->initializeGpg();
+            $this->gpg = GpgTool::initializeGpg();
             return $this->gpg;
         } catch (Exception $e) {
             $this->logException("GPG couldn't be initialized, GPG encryption and signing will be not available.", $e, LOG_NOTICE);

app/Model/Warninglist.php

@@ -1,5 +1,10 @@
 <?php
 App::uses('AppModel', 'Model');
+
+/**
+ * @property WarninglistType $WarninglistType
+ * @property WarninglistEntry $WarninglistEntry
+ */
 class Warninglist extends AppModel
 {
     public $useTable = 'warninglists';
@@ -23,27 +28,138 @@ class Warninglist extends AppModel
     );
 
     public $hasMany = array(
-            'WarninglistEntry' => array(
-                'dependent' => true
-            ),
-            'WarninglistType' => array(
-                'dependent' => true
-            )
+        'WarninglistEntry' => array(
+            'dependent' => true
+        ),
+        'WarninglistType' => array(
+            'dependent' => true
+        )
     );
 
     private $__tlds = array(
         'TLDs as known by IANA'
     );
 
-    public function beforeValidate($options = array())
+    /** @var array */
+    private $entriesCache = [];
+
+    /** @var array|null */
+    private $enabledCache = null;
+
+    private $showForAll;
+
+    public function __construct($id = false, $table = null, $ds = null)
     {
-        parent::beforeValidate();
-        return true;
+        parent::__construct($id, $table, $ds);
+        $this->showForAll = Configure::read('MISP.warning_for_all');
     }
 
-    public function getTldLists()
+    /**
+     * Attach warninglist matches to attributes or proposals with IDS mark.
+     *
+     * @param array $attributes
+     * @return array Warninglist ID => name
+     */
+    public function attachWarninglistToAttributes(array &$attributes)
     {
-        return $this->__tlds;
+        if (empty($attributes)) {
+            return [];
+        }
+
+        $enabledWarninglists = $this->getEnabled();
+        if (empty($enabledWarninglists)) {
+            return []; // no warninglist is enabled
+        }
+
+        try {
+            $redis = $this->setupRedisWithException();
+        } catch (Exception $e) {
+            // fallback to default implementation when redis is not available
+            $eventWarnings = [];
+            foreach ($attributes as $pos => $attribute) {
+                $attributes[$pos] = $this->checkForWarning($attribute, $enabledWarninglists);
+                if (isset($attributes[$pos]['warnings'])) {
+                    foreach ($attribute['warnings'] as $match) {
+                        $eventWarnings[$match['warninglist_id']] = $match['warninglist_name'];
+                    }
+                }
+            }
+            return $eventWarnings;
+        }
+
+        $warninglistIdToName = [];
+        $enabledTypes = [];
+        foreach ($enabledWarninglists as $warninglist) {
+            $warninglistIdToName[$warninglist['Warninglist']['id']] = $warninglist['Warninglist']['name'];
+            foreach ($warninglist['types'] as $type) {
+                $enabledTypes[$type] = true;
+            }
+        }
+
+        $redisResultToAttributePos = [];
+        $keysToGet = [];
+        foreach ($attributes as $pos => $attribute) {
+            if (($attribute['to_ids'] || $this->showForAll) && (isset($enabledTypes[$attribute['type']]) || isset($enabledTypes['ALL']))) {
+                $redisResultToAttributePos[] = $pos;
+                // Use hash as binary string to save memory and CPU time
+                // Hash contains just attribute type and value, so can be reused in another event attributes
+                $keysToGet[] = 'misp:wlc:' . md5($attribute['type'] . ':' . $attribute['value'], true);
+            }
+        }
+
+        if (empty($keysToGet)) {
+            return []; // no attribute suitable for warninglist check
+        }
+
+        $eventWarnings = [];
+        $saveToCache = [];
+        foreach ($redis->mget($keysToGet) as $pos => $result) {
+            if ($result === false) { // not in cache
+                $attribute = $attributes[$redisResultToAttributePos[$pos]];
+                $attribute = $this->checkForWarning($attribute, $enabledWarninglists);
+
+                $store = [];
+                if (isset($attribute['warnings'])) {
+                    foreach ($attribute['warnings'] as $match) {
+                        $warninglistId = $match['warninglist_id'];
+                        $attributes[$redisResultToAttributePos[$pos]]['warnings'][] = [
+                            'value' => $match['value'],
+                            'match' => $match['match'],
+                            'warninglist_id' => $warninglistId,
+                            'warninglist_name' => $warninglistIdToName[$warninglistId],
+                        ];
+                        $eventWarnings[$warninglistId] = $warninglistIdToName[$warninglistId];
+
+                        $store[$warninglistId] = [$match['value'], $match['match']];
+                    }
+                }
+
+                $attributeKey = $keysToGet[$pos];
+                $saveToCache[$attributeKey] = empty($store) ? '' : json_encode($store);
+
+            } elseif (!empty($result)) { // skip empty string that means no warning list match
+                $matchedWarningList = json_decode($result, true);
+                foreach ($matchedWarningList as $warninglistId => $matched) {
+                    $attributes[$redisResultToAttributePos[$pos]]['warnings'][] = [
+                        'value' => $matched[0],
+                        'match' => $matched[1],
+                        'warninglist_id' => $warninglistId,
+                        'warninglist_name' => $warninglistIdToName[$warninglistId],
+                    ];
+                    $eventWarnings[$warninglistId] = $warninglistIdToName[$warninglistId];
+                }
+            }
+        }
+
+        if (!empty($saveToCache)) {
+            $pipe = $redis->multi(Redis::PIPELINE);
+            foreach ($saveToCache as $attributeKey => $json) {
+                $redis->setex($attributeKey, 3600, $json); // cache for one hour
+            }
+            $pipe->exec();
+        }
+
+        return $eventWarnings;
     }
 
     public function update()
@@ -52,8 +168,9 @@ class Warninglist extends AppModel
         $updated = array('success' => [], 'fails' => []);
         foreach ($directories as $dir) {
             $file = new File($dir . DS . 'list.json');
-            $list = json_decode($file->read(), true);
+            $list = $this->jsonDecode($file->read());
             $file->close();
+
             if (!isset($list['version'])) {
                 $list['version'] = 1;
             }
@@ -63,9 +180,9 @@ class Warninglist extends AppModel
                 $list['type'] = $list['type'][0];
             }
             $current = $this->find('first', array(
-                    'conditions' => array('name' => $list['name']),
-                    'recursive' => -1,
-                    'fields' => array('*')
+                'conditions' => array('name' => $list['name']),
+                'recursive' => -1,
+                'fields' => array('*')
             ));
             if (empty($current) || $list['version'] > $current['Warninglist']['version']) {
                 $result = $this->__updateList($list, $current);
@@ -99,7 +216,7 @@ class Warninglist extends AppModel
         return $result;
     }
 
-    private function __updateList($list, $current)
+    private function __updateList(array $list, array $current)
     {
         $list['enabled'] = 0;
         $warninglist = array();
@@ -149,31 +266,53 @@ class Warninglist extends AppModel
         }
     }
 
-    // regenerate the warninglist caches, but if an ID is passed along, only regen the entries for the given ID.
-    // This allows us to enable/disable a single warninglist without regenerating all caches
-    public function regenerateWarninglistCaches($id = false)
+    /**
+     * Regenerate the warninglist caches, but if an ID is passed along, only regen the entries for the given ID.
+     * This allows us to enable/disable a single warninglist without regenerating all caches.
+     * @param int|null $id
+     * @return bool
+     */
+    public function regenerateWarninglistCaches($id = null)
     {
         $redis = $this->setupRedis();
         if ($redis === false) {
             return false;
         }
-        $warninglists = $this->find('all', array('contain' => array('WarninglistType'), 'conditions' => array('enabled' => 1)));
+
+        if (method_exists($redis, 'unlink')) {
+            // Delete attributes cache non blocking way if available
+            $redis->unlink($redis->keys('misp:wlc:*'));
+        } else {
+            $redis->del($redis->keys('misp:wlc:*'));
+        }
+
+        if ($id === null) {
+            // delete all cached entries when regenerating whole cache
+            $redis->del($redis->keys('misp:warninglist_entries_cache:*'));
+        }
+
+        $warninglists = $this->find('all', array(
+            'contain' => array('WarninglistType'),
+            'conditions' => array('enabled' => 1),
+            'fields' => ['id', 'name', 'type'],
+        ));
         $this->cacheWarninglists($warninglists);
+
         foreach ($warninglists as $warninglist) {
             if ($id && $warninglist['Warninglist']['id'] != $id) {
                 continue;
             }
             $entries = $this->WarninglistEntry->find('list', array(
-                    'recursive' => -1,
-                    'conditions' => array('warninglist_id' => $warninglist['Warninglist']['id']),
-                    'fields' => array('value')
+                'recursive' => -1,
+                'conditions' => array('warninglist_id' => $warninglist['Warninglist']['id']),
+                'fields' => array('value')
             ));
             $this->cacheWarninglistEntries($entries, $warninglist['Warninglist']['id']);
         }
         return true;
     }
 
-    public function cacheWarninglists($warninglists)
+    private function cacheWarninglists(array $warninglists)
     {
         $redis = $this->setupRedis();
         if ($redis !== false) {
@@ -186,7 +325,7 @@ class Warninglist extends AppModel
         return false;
     }
 
-    public function cacheWarninglistEntries($warninglistEntries, $id)
+    private function cacheWarninglistEntries(array $warninglistEntries, $id)
     {
         $redis = $this->setupRedis();
         if ($redis !== false) {
@@ -196,7 +335,7 @@ class Warninglist extends AppModel
                 $redis->sAddArray($key, $warninglistEntries);
             } else {
                 foreach ($warninglistEntries as $entry) {
-                    $redis->sAdd('misp:warninglist_entries_cache:' . $id, $entry);
+                    $redis->sAdd($key, $entry);
                 }
             }
             return true;
@@ -204,62 +343,59 @@ class Warninglist extends AppModel
         return false;
     }
 
-    public function getWarninglists($conditions)
+    /**
+     * @return array
+     */
+    public function getEnabled()
     {
-        $redis = $this->setupRedis();
-        if ($redis !== false) {
-            if ($redis->sCard('misp:warninglist_cache') === 0) {
-                if (!empty($conditions)) {
-                    $warninglists = $this->find('all', array('contain' => array('WarninglistType'), 'conditions' => $conditions));
-                } else {
-                    $warninglists = $this->find('all', array('contain' => array('WarninglistType'), 'conditions' => array('enabled' => 1)));
-                }
-                if (empty($conditions)) {
-                    $this->cacheWarninglists($warninglists);
-                }
-                return $warninglists;
-            } else {
-                $warninglists = $redis->sMembers('misp:warninglist_cache');
-                foreach ($warninglists as $k => $v) {
-                    $warninglists[$k] = json_decode($v, true);
-                }
-                if (!empty($conditions)) {
-                    foreach ($warninglists as $k => $v) {
-                        foreach ($conditions as $k2 => $v2) {
-                            if ($v['Warninglist'][$k2] != $v2) {
-                                unset($warninglists[$k]);
-                                continue 2;
-                            }
-                        }
-                    }
-                }
-                return $warninglists;
-            }
+        if (isset($this->enabledCache)) {
+            return $this->enabledCache;
         }
-    }
 
-    public function getWarninglistEntries($id)
-    {
         $redis = $this->setupRedis();
-        if ($redis !== false) {
-            if ($redis->sCard('misp:warninglist_entries_cache:' . $id) === 0) {
-                $entries = $this->WarninglistEntry->find('list', array(
-                        'recursive' => -1,
-                        'conditions' => array('warninglist_id' => $id),
-                        'fields' => array('value')
-                ));
-                $this->cacheWarninglistEntries($entries, $id);
-            } else {
-                $entries = $redis->sMembers('misp:warninglist_entries_cache:' . $id);
+        if ($redis !== false && $redis->exists('misp:warninglist_cache')) {
+            $warninglists = $redis->sMembers('misp:warninglist_cache');
+            foreach ($warninglists as $k => $v) {
+                $warninglists[$k] = json_decode($v, true);
             }
         } else {
-            $entries = $this->WarninglistEntry->find('list', array(
-                    'recursive' => -1,
-                    'conditions' => array('warninglist_id' => $id),
-                    'fields' => array('value')
+            $warninglists = $this->find('all', array(
+                'contain' => array('WarninglistType'),
+                'conditions' => array('enabled' => 1),
+                'fields' => ['id', 'name', 'type'],
             ));
+            $this->cacheWarninglists($warninglists);
+        }
+
+        foreach ($warninglists as &$warninglist) {
+            $warninglist['types'] = [];
+            foreach ($warninglist['WarninglistType'] as $wt) {
+                $warninglist['types'][] = $wt['type'];
+            }
+            unset($warninglist['WarninglistType']);
+        }
+        $this->enabledCache = $warninglists;
+        return $warninglists;
+    }
+
+    /**
+     * @param int $id
+     * @return array
+     */
+    private function getWarninglistEntries($id)
+    {
+        $redis = $this->setupRedis();
+        if ($redis !== false && $redis->exists('misp:warninglist_entries_cache:' . $id)) {
+            return $redis->sMembers('misp:warninglist_entries_cache:' . $id);
+        } else {
+            $entries = array_values($this->WarninglistEntry->find('list', array(
+                'recursive' => -1,
+                'conditions' => array('warninglist_id' => $id),
+                'fields' => array('value')
+            )));
+            $this->cacheWarninglistEntries($entries, $id);
+            return $entries;
         }
-        return $entries;
     }
 
     /**
@@ -271,6 +407,7 @@ class Warninglist extends AppModel
     {
         $outputValues = [];
         foreach ($inputValues as $v) {
+            $v = strtolower($v);
             $parts = explode('/', $v, 2);
             if (filter_var($parts[0], FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) {
                 $maximumNetmask = 32;
@@ -289,58 +426,69 @@ class Warninglist extends AppModel
                 continue;
             }
 
-            $outputValues[$v] = $v;
+            $outputValues[$v] = true;
         }
         return $outputValues;
     }
 
-    public function fetchForEventView()
+    /**
+     * For 'hostname', 'string' and 'cidr' warninglist type, values are just in keys to save memory.
+     *
+     * @param array $warninglist
+     * @return array
+     */
+    public function getFilteredEntries(array $warninglist)
     {
-        $warninglists = $this->getWarninglists(array('enabled' => 1));
-        if (empty($warninglists)) {
-            return array();
+        $id = $warninglist['Warninglist']['id'];
+        if (isset($this->entriesCache[$id])) {
+            return $this->entriesCache[$id];
         }
-        foreach ($warninglists as $k => &$t) {
-            $t['values'] = $this->getWarninglistEntries($t['Warninglist']['id']);
-            $t['values'] = array_values($t['values']);
 
-            if ($t['Warninglist']['type'] === 'hostname') {
-                $values = [];
-                foreach ($t['values'] as $v) {
-                    $v = rtrim($v, '.');
-                    $values[$v] = $v;
-                }
-                $t['values'] = $values;
-            } else if ($t['Warninglist']['type'] === 'string') {
-                $t['values'] = array_combine($t['values'], $t['values']);
-            } else if ($t['Warninglist']['type'] === 'cidr') {
-                $t['values'] = $this->filterCidrList($t['values']);
+        $values = $this->getWarninglistEntries($id);
+        if ($warninglist['Warninglist']['type'] === 'hostname') {
+            $output = [];
+            foreach ($values as $v) {
+                $v = strtolower(trim($v, '.'));
+                $output[$v] = true;
             }
-
-            foreach ($t['WarninglistType'] as $wt) {
-                $t['types'][] = $wt['type'];
+            $values = $output;
+        } else if ($warninglist['Warninglist']['type'] === 'string') {
+            $output = [];
+            foreach ($values as $v) {
+                $output[$v] = true;
             }
-            unset($warninglists[$k]['WarninglistType']);
+            $values = $output;
+        } else if ($warninglist['Warninglist']['type'] === 'cidr') {
+            $values = $this->filterCidrList($values);
         }
-        return $warninglists;
+
+        $this->entriesCache[$id] = $values;
+
+        return $values;
     }
 
-    public function simpleCheckForWarning($object, $warninglists, $returnVerboseValue = false)
+    /**
+     * @param array $object
+     * @param array|null $warninglists If null, all enabled warninglists will be used
+     * @return array
+     */
+    public function checkForWarning(array $object, $warninglists = null)
     {
-        if ($object['to_ids']) {
+        if ($warninglists === null) {
+            $warninglists = $this->getEnabled();
+        }
+
+        if ($object['to_ids'] || $this->showForAll) {
             foreach ($warninglists as $list) {
                 if (in_array('ALL', $list['types']) || in_array($object['type'], $list['types'])) {
-                    $result = $this->__checkValue($list['values'], $object['value'], $object['type'], $list['Warninglist']['type']);
-                    if (!empty($result)) {
-                        if ($returnVerboseValue) {
-                            $object['warnings'][] = array(
-                                'value' => $result,
-                                'warninglist_name' => $list['Warninglist']['name'],
-                                'warninglist_id' => $list['Warninglist']['id']
-                            );
-                        } else {
-                            $object['warnings'][$result][] = $list['Warninglist']['name'];
-                        }
+                    $result = $this->__checkValue($this->getFilteredEntries($list), $object['value'], $object['type'], $list['Warninglist']['type']);
+                    if ($result !== false) {
+                        $object['warnings'][] = array(
+                            'match' => $result[0],
+                            'value' => $result[1],
+                            'warninglist_name' => $list['Warninglist']['name'],
+                            'warninglist_id' => $list['Warninglist']['id'],
+                        );
                     }
                 }
             }
@@ -348,85 +496,36 @@ class Warninglist extends AppModel
         return $object;
     }
 
-    public function checkForWarning($object, &$eventWarnings, $warningLists, $returnVerboseValue = false)
-    {
-        if ($object['to_ids']) {
-            foreach ($warningLists as $list) {
-                if (in_array('ALL', $list['types']) || in_array($object['type'], $list['types'])) {
-                    $result = $this->__checkValue($list['values'], $object['value'], $object['type'], $list['Warninglist']['type'], $returnVerboseValue);
-                    if (!empty($result)) {
-                        if ($returnVerboseValue) {
-                            $object['warnings'][] = array(
-                                'value' => $result,
-                                'warninglist_name' => $list['Warninglist']['name'],
-                                'warninglist_id' => $list['Warninglist']['id']
-                            );
-                        } else {
-                            $object['warnings'][$result][] = $list['Warninglist']['name'];
-                        }
-                        if (!isset($eventWarnings[$list['Warninglist']['id']])) {
-                            $eventWarnings[$list['Warninglist']['id']] = $list['Warninglist']['name'];
-                        }
-                    }
-                }
-            }
-        }
-        return $object;
-    }
-
-    public function setWarnings(&$event, &$warninglists)
-    {
-        if (empty($event['objects'])) {
-            return $event;
-        }
-        $eventWarnings = array();
-        foreach ($event['objects'] as &$object) {
-            if ($object['to_ids']) {
-                foreach ($warninglists as $list) {
-                    if (in_array('ALL', $list['types']) || in_array($object['type'], $list['types'])) {
-                        $result = $this->__checkValue($list['values'], $object['value'], $object['type'], $list['Warninglist']['type']);
-                        if (!empty($result)) {
-                            $object['warnings'][$result][] = $list['Warninglist']['name'];
-                            if (!in_array($list['Warninglist']['name'], $eventWarnings)) {
-                                $eventWarnings[$list['Warninglist']['id']] = $list['Warninglist']['name'];
-                            }
-                        }
-                    }
-                }
-            }
-        }
-        $event['Event']['warnings'] = $eventWarnings;
-        return $event;
-    }
-
-    private function __checkValue($listValues, $value, $type, $listType, $returnVerboseValue = false)
+    /**
+     * @param array $listValues
+     * @param string $value
+     * @param string $type
+     * @param string $listType
+     * @return array|false [Matched value, attribute value that matched]
+     */
+    private function __checkValue($listValues, $value, $type, $listType)
     {
         if ($type === 'malware-sample' || strpos($type, '|') !== false) {
-            $value = explode('|', $value);
+            $value = explode('|', $value, 2);
         } else {
             $value = array($value);
         }
-        $components = array(0, 1);
-        foreach ($components as $component) {
-            if (!isset($value[$component])) {
-                continue;
-            }
+        foreach ($value as $v) {
             if ($listType === 'cidr') {
-                $result = $this->__evalCIDRList($listValues, $value[$component]);
+                $result = $this->__evalCidrList($listValues, $v);
             } elseif ($listType === 'string') {
-                $result = $this->__evalString($listValues, $value[$component]);
+                $result = $this->__evalString($listValues, $v);
             } elseif ($listType === 'substring') {
-                $result = $this->__evalSubString($listValues, $value[$component]);
+                $result = $this->__evalSubString($listValues, $v);
             } elseif ($listType === 'hostname') {
-                $result = $this->__evalHostname($listValues, $value[$component]);
+                $result = $this->__evalHostname($listValues, $v);
             } elseif ($listType === 'regex') {
-                $result = $this->__evalRegex($listValues, $value[$component]);
+                $result = $this->__evalRegex($listValues, $v);
+            } else {
+                $result = false;
             }
-            if (!empty($result)) {
-                if ($returnVerboseValue) {
-                    return $value[$component];
-                }
-                return ($component + 1);
+            if ($result !== false) {
+                return [$result, $v];
             }
         }
         return false;
@@ -434,47 +533,59 @@ class Warninglist extends AppModel
 
     public function quickCheckValue($listValues, $value, $type)
     {
-        $typeMapping = array(
-            'cidr' => '__evalCIDRList',
-            'string' => '__evalString',
-            'substring' => '__evalSubString',
-            'hostname' => '__evalHostname',
-            'regex' => '__evalRegex'
-        );
-        $result = $this->{$typeMapping[$type]}($listValues, $value);
-        return (!empty($result) ? 1 : false);
+        return $this->__checkValue($listValues, $value, '', $type) !== false;
     }
 
-    // This requires an IP type attribute in a non CIDR notation format
-    // For the future we can expand this to look for CIDR overlaps?
-    private function __evalCIDRList($listValues, $value)
+    private function __evalCidrList($listValues, $value)
     {
+        $valueMask = null;
+        if (strpos($value, '/') !== false) {
+            list($value, $valueMask) = explode('/', $value);
+        }
+
+        $match = false;
         if (filter_var($value, FILTER_VALIDATE_IP, FILTER_FLAG_IPV4)) {
             // This code converts IP address to all possible CIDRs that can contains given IP address
             // and then check if given hash table contains that CIDR.
             $ip = ip2long($value);
-            for ($bits = 0; $bits <= 32; $bits++) {
+            // Start from 1, because doesn't make sense to check 0.0.0.0/0 match
+            for ($bits = 1; $bits <= 32; $bits++) {
                 $mask = -1 << (32 - $bits);
                 $needle = long2ip($ip & $mask) . "/$bits";
                 if (isset($listValues[$needle])) {
-                    return true;
+                    $match = $needle;
+                    break;
                 }
             }
 
         } elseif (filter_var($value, FILTER_VALIDATE_IP, FILTER_FLAG_IPV6)) {
-            foreach ($listValues as $lv) {
-                if (strpos($lv, ':') !== false) { // IPv6 CIDR must contain dot
+            foreach ($listValues as $lv => $foo) {
+                if (strpos($lv, ':') !== false) { // Filter out IPv4 CIDR, IPv6 CIDR must contain colon
                     if ($this->__ipv6InCidr($value, $lv)) {
-                        return true;
+                        $match = $lv;
+                        break;
                     }
                 }
             }
         }
 
-        return false;
+        if ($match && $valueMask) {
+            $matchMask = explode('/', $match)[1];
+            if ($valueMask < $matchMask) {
+                return false;
+            }
+        }
+
+        return $match;
     }
 
-    // Using solution from https://github.com/symfony/symfony/blob/master/src/Symfony/Component/HttpFoundation/IpUtils.php
+    /**
+     * Using solution from https://github.com/symfony/symfony/blob/master/src/Symfony/Component/HttpFoundation/IpUtils.php
+     *
+     * @param string $ip
+     * @param string $cidr
+     * @return bool
+     */
     private function __ipv6InCidr($ip, $cidr)
     {
         list($address, $netmask) = explode('/', $cidr);
@@ -494,10 +605,17 @@ class Warninglist extends AppModel
         return true;
     }
 
+    /**
+     * Check for exact match.
+     *
+     * @param array $listValues
+     * @param string $value
+     * @return false
+     */
     private function __evalString($listValues, $value)
     {
         if (isset($listValues[$value])) {
-            return true;
+            return $value;
         }
         return false;
     }
@@ -506,7 +624,7 @@ class Warninglist extends AppModel
     {
         foreach ($listValues as $listValue) {
             if (strpos($value, $listValue) !== false) {
-                return true;
+                return $listValue;
             }
         }
         return false;
@@ -515,7 +633,7 @@ class Warninglist extends AppModel
     private function __evalHostname($listValues, $value)
     {
         // php's parse_url is dumb, so let's use some hacky workarounds
-        if (strpos($value, '//') == false) {
+        if (strpos($value, '//') === false) {
             $value = explode('/', $value);
             $hostname = $value[0];
         } else {
@@ -529,14 +647,14 @@ class Warninglist extends AppModel
         $hostname = rtrim($hostname, '.');
         $hostname = explode('.', $hostname);
         $rebuilt = '';
-        foreach (array_reverse($hostname) as $k => $piece) {
+        foreach (array_reverse($hostname) as $piece) {
             if (empty($rebuilt)) {
                 $rebuilt = $piece;
             } else {
                 $rebuilt = $piece . '.' . $rebuilt;
             }
             if (isset($listValues[$rebuilt])) {
-                return true;
+                return $rebuilt;
             }
         }
         return false;
@@ -546,23 +664,30 @@ class Warninglist extends AppModel
     {
         foreach ($listValues as $listValue) {
             if (preg_match($listValue, $value)) {
-                return true;
+                return $listValue;
             }
         }
         return false;
     }
 
+    /**
+     * @return array
+     */
     public function fetchTLDLists()
     {
-        $tldLists = $this->find('list', array('conditions' => array('Warninglist.name' => $this->__tlds), 'recursive' => -1, 'fields' => array('Warninglist.id', 'Warninglist.name')));
+        $tldLists = $this->find('list', array(
+            'conditions' => array('Warninglist.name' => $this->__tlds),
+            'recursive' => -1,
+            'fields' => array('Warninglist.id', 'Warninglist.id')
+        ));
         $tlds = array();
         if (!empty($tldLists)) {
-            $tldLists = array_keys($tldLists);
-            $tlds = $this->WarninglistEntry->find('list', array('conditions' => array('WarninglistEntry.warninglist_id' => $tldLists), 'fields' => array('WarninglistEntry.value')));
-            if (!empty($tlds)) {
-                foreach ($tlds as $key => $value) {
-                    $tlds[$key] = strtolower($value);
-                }
+            $tlds = $this->WarninglistEntry->find('list', array(
+                'conditions' => array('WarninglistEntry.warninglist_id' => $tldLists),
+                'fields' => array('WarninglistEntry.value')
+            ));
+            foreach ($tlds as $key => $value) {
+                $tlds[$key] = strtolower($value);
             }
         }
         if (!in_array('onion', $tlds)) {
@@ -571,11 +696,20 @@ class Warninglist extends AppModel
         return $tlds;
     }
 
-    public function filterWarninglistAttributes($warninglists, $attribute)
+    /**
+     * @param array $attribute
+     * @param array|null $warninglists If null, all enabled warninglists will be used
+     * @return bool
+     */
+    public function filterWarninglistAttribute(array $attribute, $warninglists = null)
     {
+        if ($warninglists === null) {
+            $warninglists = $this->getEnabled();
+        }
+
         foreach ($warninglists as $warninglist) {
             if (in_array('ALL', $warninglist['types']) || in_array($attribute['type'], $warninglist['types'])) {
-                $result = $this->__checkValue($warninglist['values'], $attribute['value'], $attribute['type'], $warninglist['Warninglist']['type']);
+                $result = $this->__checkValue($this->getFilteredEntries($warninglist), $attribute['value'], $attribute['type'], $warninglist['Warninglist']['type']);
                 if ($result !== false) {
                     return false;
                 }
@@ -583,4 +717,20 @@ class Warninglist extends AppModel
         }
         return true;
     }
+
+    public function missingTldLists()
+    {
+        $missingTldLists = array();
+        foreach ($this->__tlds as $tldList) {
+            $temp = $this->find('first', array(
+                'recursive' => -1,
+                'conditions' => array('Warninglist.name' => $tldList),
+                'fields' => array('Warninglist.id')
+            ));
+            if (empty($temp)) {
+                $missingTldLists[] = $tldList;
+            }
+        }
+        return $missingTldLists;
+    }
 }

app/Plugin/ShibbAuth/README.md

@@ -5,6 +5,72 @@ This plugin enables CakePHP applications to use Single Sign-On to authenticate i
 
 ## Usage
 
+### Prerequisites - Shibboleth Service Provider
+The MISP plugin takes care of the mapping of your shibboleth session attributes to MISP, but you will still need to install the service provider (SP) and configure it yourself. The documentation for Shibboleth Service Provider 3 can be found at https://wiki.shibboleth.net/confluence/display/SP3/Home.
+
+To install Shibboleth SP3 on Ubuntu, you can use the instructions provided by SWITCH at https://www.switch.ch/aai/guides/sp/installation/ and then follow the below steps. If you already installed and configured Shibboleth you can skip this section.
+
+Create signing and encryption certificate. The value following -e should be your entity ID, for example https://<host>/shibboleth.
+```bash
+sudo shib-keygen -f -u _shibd -h <host> -y 5 -e https://<host>/shibboleth -o /etc/shibboleth
+```
+
+Edit /etc/shibboleth/shibboleth2.xml to use the created certificate for both signing and encryption (change the values for key and certificate).
+```xml
+    <CredentialResolver type="File" use="signing"
+        key="sp-key.pem" certificate="sp-cert.pem"/>
+    <CredentialResolver type="File" use="encryption"
+        key="sp-key.pem" certificate="sp-cert.pem"/>
+```
+
+Edit /etc/shibboleth/shibboleth2.xml to set secure cookie properties (cookieProps) if you want to.
+```xml
+<Sessions lifetime="28800" timeout="3600" relayState="ss:mem"
+                  checkAddress="false" handlerSSL="false" cookieProps="https"
+                  redirectLimit="exact">
+```
+
+At this point, you should already be able to test your configuration. The last line of the output should be "overall configuration is loadable, check console for non-fatal problems".
+```bash
+sudo shibd -t
+```
+
+Set entityID in /etc/shibboleth/shibboleth2.xml.
+```xml
+<ApplicationDefaults entityID="https://<host>/shibboleth"
+        REMOTE_USER="eppn subject-id pairwise-id persistent-id"
+        cipherSuites="DEFAULT:!EXP:!LOW:!aNULL:!eNULL:!DES:!IDEA:!SEED:!RC4:!3DES:!kRSA:!SSLv2:!SSLv3:!TLSv1:!TLSv1.1">
+```
+
+Copy your identity provider metadata to /etc/shibboleth, for example to /etc/shibboleth/idp-metadata.xml and refer to it in /etc/shibboleth/shibboleth2.xml. Uncomment and edit the relevant line.
+```xml
+<MetadataProvider type="XML" validate="true" path="idp-metadata.xml"/>
+```
+
+Optionally, you can make sure the service provider does not create a session if some attributes, like OrgTag and GroupTag are missing. If users attempt to login an this happens, they will receive a pre-configured reply (default at /etc/shibboleth/attrChecker.html).
+In /etc/shibboleth/shibboleth2.xml, edit ApplicationDefaults by adding the sessionHook:
+```xml
+<ApplicationDefaults entityID="https://<HOST>/shibboleth"
+  REMOTE_USER="eppn persistent-id targeted-id"
+  signing="front" encryption="false"
+  sessionHook="/Shibboleth.sso/AttrChecker"
+```
+Optional for attribute checking: add your checks (note that the incoming attribute names can be different for you, for more info on possible checks refer to https://wiki.shibboleth.net/confluence/display/SP3/Attribute+Checker+Handler):
+```xml
+<Handler type="AttributeChecker" Location="/AttrChecker" template="attrChecker.html" attributes="OrgTag GroupTag" flushSession="true"/>
+```
+
+At this point you will have to send your metadata to your identity provider. You can get template metadata based on your configuration from https://&lt;host&gt;/Shibboleth.sso/Metadata.
+
+### MISP plugin configuration
+
+Edit your MISP apache configuration by adding the below (location depends on your handler path, /Shibboleth.sso by default).
+```Apache
+  <Location /Shibboleth.sso>
+    SetHandler shib
+  </Locations>
+```
+
 Enable the plugin at bootstrap.php:
 
 ```php
@@ -18,43 +84,68 @@ Uncomment the following line to enable SSO authorization
 'auth'=>array('ShibbAuth.ApacheShibb'),
 ```
 
-And configure it. MailTag, OrgTag and GroupTag are the string that represent the key for the values needed by the plugin.
-For example if you are using ADFS OrgTag will be ADFS_FEDERATION, GroupTag will be ADFS_GROUP, etc. meaning the key for the values needed.
-DefaultOrg are values that come by default just in case they are not defined or obtained from the environment variables.
-The GroupRoleMatching is an array that allows the definition and correlation between groups and roles in MISP, being them updated
+If the line does not exist, add it to 'Security' array, for example like below. Note that you should just add the line to your own existing config.
+```php
+'Security' =>
+  array (
+    'level' => 'medium',
+    'salt' => '',
+    'cipherSeed' => '',
+    'password_policy_length' => 12,
+    'password_policy_complexity' => '/^((?=.*\\d)|(?=.*\\W+))(?![\\n])(?=.*[A-Z])(?=.*[a-z]).*$|.{16,}/',
+    'self_registration_message' => 'If you would like to send us a registration request, please fill out the form below. Make sure you fill out as much information as possible in order to ease the task of the administrators.',
+    'auth'=>array('ShibbAuth.ApacheShibb'),
+  )
+```
+
+And configure it. MailTag, OrgTag and GroupTag are the keys for the values needed by the plugin.
+For example if you are using ADFS you should replace IDP_FEDERATION_TAG by ADFS_FEDERATION, IDP_GROUP_TAG by ADFS_GROUP, etc.
+Replace MISP_DEFAULT_ORG by the organization you want users to be assigned to in case no organization value is given by the identity provider.
+The GroupRoleMatching is an array that allows the definition and correlation between groups and roles in MISP. These get updated
 if the groups are updated (i.e. a user that was admin and their groups changed inside the organization will have his role changed in MISP
 upon the next login being now user or org admin respectively). The GroupSeparator is the character used to separate the different groups
-in the list given by apache.
+in the list given by apache. By default, you can leave it at ';'.
 
 ```php
 'ApacheShibbAuth' =>                      // Configuration for shibboleth authentication
     array(
-         'MailTag' => 'EMAIL_TAG',
-         'OrgTag' => 'FEDERATION_TAG',
-	 'GroupTag' => 'GROUP_TAG',
+         'MailTag' => 'IDP_EMAIL_TAG',
+         'OrgTag' => 'IDP_FEDERATION_TAG',
+	 'GroupTag' => 'IDP_GROUP_TAG',
 	 'GroupSeparator' => ';',
          'GroupRoleMatching' => array(                // 3:User, 1:admin. May be good to set "1" for the first user
-               'group_three' => '3',
-	       'group_two' => 2,
-	       'group_one' => 1,
+               'possible_group_attribute_value_3' => '3',
+	       'possible_group_attribute_value_2' => 2,
+	       'possible_group_attribute_value_1' => 1,
           ),
-         'DefaultOrg' => 'DEFAULT_ORG',
+         'DefaultOrg' => 'MISP_DEFAULT_ORG',
     ),
 ```
-If used with Apache as webserver it might be useful to make a distinction to filter out API/Syncs from SSO login. It can be added to the vhost as follows:
+If used with Apache as webserver it might be useful to make a distinction to filter out API/Syncs from SSO login. It can be added to the vhost as follows (Added lines are the If/Else clauses):
 
 ```Apache
-  <If "-T req('Authorization')">
-    Require all granted
-    AuthType None
-  </If>
-  <Else>
-    Require valid-user
-    AuthType shibboleth
-    ShibRequestSetting requiresession On
-    ShibRequestSetting shibexportassertion Off
-    ShibUseHeaders On
-  </Else>
+  <Directory /var/www/MISP/app/webroot>
+    Options -Indexes
+    AllowOverride all
+    <If "-T req('Authorization')">
+        Require all granted
+        AuthType None
+    </If>
+    <Else>
+        Require valid-user
+        AuthType shibboleth
+        ShibRequestSetting requiresession On
+        ShibRequestSetting shibexportassertion Off
+        ShibUseHeaders On
+    </Else>
+  </Directory>
+```
+
+If you want the logout button to work for killing your session, you can use the CustomAuth plugin to configure a custom logout url, by default the url should be https://&lt;host&gt;/Shibboleth.sso/Logout. This leads to a local logout. If you want to also trigger a logout at the identity provider, you can use the return mechanism. In this case you will need to change the allowed redirects. Your logout url will look like https://&lt;host&gt;/Shibboleth.sso/Logout?return=https://<idp_host>/Logout. Edit your shibboleth configuration (often at /etc/shibboleth/shibboleth2.xml) as necessary. Relevant shibboleth documentation can be found at https://wiki.shibboleth.net/confluence/display/SP3/Logout and https://wiki.shibboleth.net/confluence/display/SP3/Sessions.
+```xml
+<Sessions lifetime="28800" timeout="3600" relayState="ss:mem"
+                  checkAddress="false" handlerSSL="false" cookieProps="https"
+                  redirectLimit="exact+whitelist" redirectWhitelist="https://<idp_host>">
 ```
 
 

app/Plugin/SysLogLogable/Model/Behavior/SysLogLogableBehavior.php

@@ -256,23 +256,32 @@ class SysLogLogableBehavior extends LogableBehavior {
 		$this->Log->save(null, array('validate' => false));
 	}
 
-	function setup(Model $Model, $config = array()) {
-		if (!is_array($config)) {
-			$config = array();
-		}
-		$this->settings[$Model->alias] = array_merge($this->defaults, $config);
-		$this->settings[$Model->alias]['ignore'][] = $Model->primaryKey;
+    function setup(Model $Model, $config = array())
+    {
+        if (!is_array($config)) {
+            $config = array();
+        }
+        $this->settings[$Model->alias] = array_merge($this->defaults, $config);
+        $this->settings[$Model->alias]['ignore'][] = $Model->primaryKey;
 
-		$this->Log = ClassRegistry::init('Log');
-		if ($this->settings[$Model->alias]['userModel'] != $Model->alias) {
-			$this->UserModel = ClassRegistry::init($this->settings[$Model->alias]['userModel']);
-		} else {
-			$this->UserModel = $Model;
-		}
-		$this->schema = $this->Log->schema();
-		App::uses('AuthComponent', 'Controller/Component');
-		$user = AuthComponent::user();
-		if (!empty($user)) $this->user[$this->settings[$Model->alias]['userModel']] = AuthComponent::user();
-		else $this->user['User'] = array('email' => 'SYSTEM', 'Organisation' => array('name' => 'SYSTEM'), 'id' => 0);
-	}
+        $this->Log = ClassRegistry::init('Log');
+        if ($this->settings[$Model->alias]['userModel'] != $Model->alias) {
+            $this->UserModel = ClassRegistry::init($this->settings[$Model->alias]['userModel']);
+        } else {
+            $this->UserModel = $Model;
+        }
+        $this->schema = $this->Log->schema();
+
+        $user = array('email' => 'SYSTEM', 'Organisation' => array('name' => 'SYSTEM'), 'id' => 0);
+        $isShell = defined('CAKEPHP_SHELL') && CAKEPHP_SHELL; // do not start session for shell commands
+        if (!$isShell) {
+            App::uses('AuthComponent', 'Controller/Component');
+            $authUser = AuthComponent::user();
+            if (!empty($authUser)) {
+                $user = $authUser;
+            }
+        }
+
+        $this->user['User'] = $user;
+    }
 }

app/Test/ComplexTypeToolTest.php

@@ -420,6 +420,15 @@ EOT;
         $this->assertEquals('vulnerability', $results[0]['default_type']);
     }
 
+    public function testCheckFreeTextCveLowercase(): void
+    {
+        $complexTypeTool = new ComplexTypeTool();
+        $results = $complexTypeTool->checkFreeText('cve-2019-16202');
+        $this->assertCount(1, $results);
+        $this->assertEquals('CVE-2019-16202', $results[0]['value']);
+        $this->assertEquals('vulnerability', $results[0]['default_type']);
+    }
+
     public function testCheckFreeTextAs(): void
     {
         $complexTypeTool = new ComplexTypeTool();
@@ -438,6 +447,15 @@ EOT;
         $this->assertEquals('md5', $results[0]['default_type']);
     }
 
+    public function testCheckFreeTextMd5Uppercase(): void
+    {
+        $complexTypeTool = new ComplexTypeTool();
+        $results = $complexTypeTool->checkFreeText('9E107D9D372BB6826BD81D3542A419D6');
+        $this->assertCount(1, $results);
+        $this->assertEquals('9E107D9D372BB6826BD81D3542A419D6', $results[0]['value']);
+        $this->assertEquals('md5', $results[0]['default_type']);
+    }
+
     public function testCheckFreeTextSha1(): void
     {
         $complexTypeTool = new ComplexTypeTool();

app/View/Attributes/ajax/attributeEditCategoryForm.ctp

@@ -2,8 +2,8 @@
     echo $this->Form->create('Attribute', array('class' => 'inline-form inline-field-form', 'id' => 'Attribute_' . $object['id'] . '_category_form', 'url' => $baseurl . '/attributes/editField/' . $object['id']));
 ?>
 <div class='inline-input inline-input-container'>
-<div class="inline-input-accept inline-input-button inline-input-passive"><span class = "icon-ok" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
-<div class="inline-input-decline inline-input-button inline-input-passive"><span class = "icon-remove" role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
+<div class="inline-input-accept inline-input-button inline-input-passive"><span class="fas fa-check" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
+<div class="inline-input-decline inline-input-button inline-input-passive"><span class="fas fa-times" role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
 <?php
     echo $this->Form->input('category', array(
         'options' => array(array_combine($typeCategory[$object['type']], $typeCategory[$object['type']])),

app/View/Attributes/ajax/attributeEditCommentForm.ctp

@@ -2,8 +2,8 @@
     echo $this->Form->create('Attribute', array('class' => 'inline-form inline-field-form', 'id' => 'Attribute_' . $object['id'] . '_comment_form', 'url' => $baseurl . '/attributes/editField/' . $object['id']));
 ?>
     <div class='inline-input inline-input-container'>
-    <div class="inline-input-accept inline-input-button inline-input-passive"><span class = "icon-ok" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
-    <div class="inline-input-decline inline-input-button inline-input-passive"><span class = "icon-remove" role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
+    <div class="inline-input-accept inline-input-button inline-input-passive"><span class="fas fa-check" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
+    <div class="inline-input-decline inline-input-button inline-input-passive"><span class="fas fa-times" role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
         <?php
             echo $this->Form->input('comment', array(
                     'type' => 'textarea',

app/View/Attributes/ajax/attributeEditDistributionForm.ctp

@@ -2,8 +2,8 @@
     echo $this->Form->create('Attribute', array('class' => 'inline-form inline-field-form', 'id' => 'Attribute_' . $object['id'] . '_distribution_form', 'url' => $baseurl . '/attributes/editField/' . $object['id']));
 ?>
 <div class='inline-input inline-input-container'>
-    <div class="inline-input-accept inline-input-button inline-input-passive"><span class = "icon-ok" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
-    <div class="inline-input-decline inline-input-button inline-input-passive"><span class = "icon-remove" role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
+    <div class="inline-input-accept inline-input-button inline-input-passive"><span class="fas fa-check" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
+    <div class="inline-input-decline inline-input-button inline-input-passive"><span class="fas fa-times"role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
     <?php
         echo $this->Form->input('distribution', array(
                 'options' => array($distributionLevels),

app/View/Attributes/ajax/attributeEditTypeForm.ctp

@@ -2,8 +2,8 @@
     echo $this->Form->create('Attribute', array('class' => 'inline-form inline-field-form', 'id' => 'Attribute_' . $object['id'] . '_type_form', 'url' => $baseurl . '/attributes/editField/' . $object['id']));
 ?>
 <div class='inline-input inline-input-container'>
-    <div class="inline-input-accept inline-input-button inline-input-passive"><span class = "icon-ok" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
-    <div class="inline-input-decline inline-input-button inline-input-passive"><span class = "icon-remove" role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
+    <div class="inline-input-accept inline-input-button inline-input-passive"><span class="fas fa-check" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
+    <div class="inline-input-decline inline-input-button inline-input-passive"><span class="fas fa-times" role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
 <?php
     echo $this->Form->input('type', array(
         'options' => array(array_combine($categoryDefinitions[$object['category']]['types'], $categoryDefinitions[$object['category']]['types'])),

app/View/Attributes/ajax/attributeEditValueForm.ctp

@@ -3,8 +3,8 @@
     echo $this->Form->create('Attribute', array('class' => 'inline-form inline-field-form', 'url' => $baseurl . '/attributes/editField/' . $object['id'], 'id' => 'Attribute_' . $object['id'] . '_value_form', 'default' => false));
 ?>
     <div class='inline-input inline-input-container'>
-    <div class="inline-input-accept inline-input-button inline-input-passive"><span class = "icon-ok" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
-    <div class="inline-input-decline inline-input-button inline-input-passive"><span class = "icon-remove" role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
+    <div class="inline-input-accept inline-input-button inline-input-passive"><span class="fas fa-check" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>
+    <div class="inline-input-decline inline-input-button inline-input-passive"><span class="fas fa-times" role="button" tabindex="0" aria-label="<?php echo __('Discard change'); ?>"></span></div>
 <?php
     echo $this->Form->input('value', array(
             'type' => 'textarea',

app/View/Attributes/ajax/attributeViewFieldForm.ctp

@@ -1,8 +1,12 @@
 <?php
-if ($value === 'No') {
-    echo '<input type="checkbox" disabled>';
-} else if ($value === 'Yes') {
-    echo '<input type="checkbox" checked disabled>';
+if ($field === 'value') {
+    echo $this->element('Events/View/value_field', ['object' => $object['Attribute']]);
 } else {
-    echo nl2br(h($value)) . '&nbsp;';
+    if ($value === 'No') {
+        echo '<input type="checkbox" disabled>';
+    } else if ($value === 'Yes') {
+        echo '<input type="checkbox" checked disabled>';
+    } else {
+        echo nl2br(h($value)) . '&nbsp;';
+    }
 }

app/View/Attributes/ajax/hover_enrichment.ctp

@@ -1,49 +1,59 @@
-<div style="overflow-y:auto;max-height:75vh">
-    <?php
-        foreach ($results as $enrichment_type => $enrichment_values):
-            echo sprintf('<h5><span class="hover_enrichment_title blue">%s</span>:</h5>', Inflector::humanize(h($enrichment_type)));
-            if (empty($enrichment_values)) {
-                echo '<div style="padding: 2px;"><span class="empty_results_text red">Empty results</span></div>';
-                continue;
+<div style="overflow-y:auto;<?= $persistent ? 'max-height:98%;padding: .3em 1em' : 'max-height:75vh' ?>">
+<?php
+$formatValue = function (array $attribute) {
+    switch ($attribute['type']) {
+        case 'link':
+            return '<a href="' . h($attribute['value']) . '" rel="noreferrer noopener" target="_blank">' . h($attribute['value']) . '</a>';
+        default:
+            return h($attribute['value']);
+    }
+};
+foreach ($results as $enrichment_type => $enrichment_values) {
+    echo sprintf('<h5 class="hover_enrichment_title blue">%s:</h5>', Inflector::humanize(h($enrichment_type)));
+    if (isset($enrichment_values['error'])) {
+        echo '<div style="padding: 2px;" class="red">' . __('Error: %s', h($enrichment_values['error']))  . '</div>';
+        continue;
+    }
+    if (empty($enrichment_values)) {
+        echo '<div style="padding: 2px;" class="red">' . __('Empty results') . '</div>';
+        continue;
+    }
+    if (!empty($enrichment_values['Object'])) {
+        foreach ($enrichment_values['Object'] as $object) {
+            echo '<h6 class="bold blue">' . __('Object: %s', h($object['name'])) . '</h6>';
+            echo '<table class="table table-striped table-condensed">';
+            foreach ($object['Attribute'] as $object_attribute) {
+                echo '<tr><th style="width: 15em">' . h($object_attribute['object_relation']) . '</th><td>' . $formatValue($object_attribute) . '</td></tr>';
             }
-            if (!empty($enrichment_values['Object'])) {
-                echo '<h6><span class="bold blue">Objects</span></h6>';
-                foreach ($enrichment_values['Object'] as $object) {
-                    echo '<span class="object_name bold blue">' . h($object['name']) . '</span><br />';
-                    foreach ($object['Attribute'] as $object_attribute) {
-                        echo '<div style="padding: 2px;"><pre class="object_attribute">';
-                        echo '<span class="attribute_object_relation bold blue">' . h($object_attribute['object_relation']) . '</span>';
-                        echo ': <span class="attribute_value red">' . h($object_attribute['value']) . '</span></pre></div>';
+            echo '</table>';
+        }
+        unset($enrichment_values['Object']);
+    }
+    if (!empty($enrichment_values['Attribute'])) {
+        echo '<h6 class="bold blue">' . __('Attributes') . '</h6>';
+        echo '<table class="table table-striped table-condensed">';
+        foreach ($enrichment_values['Attribute'] as $attribute) {
+            echo '<tr><th style="width: 15em">' . h($attribute['type']). '</th><td>' . $formatValue($attribute) . '</td></tr>';
+        }
+        echo '</table>';
+        unset($enrichment_values['Attribute']);
+    }
+    foreach ($enrichment_values as $attributes) {
+        foreach ($attributes as $attribute) {
+            echo '<div style="padding: 2px;">';
+            if (is_array($attribute)) {
+                foreach ($attribute as $attribute_name => $attribute_value) {
+                    if (!is_numeric($attribute_name)) {
+                        echo '<strong>' . h($attribute_name) . ':</strong> ';
                     }
+                    echo ' ' . h($attribute_value);
                 }
-                unset($enrichment_values['Object']);
+            } else {
+                echo h($attribute);
             }
-            if (!empty($enrichment_values['Attribute'])) {
-                echo '<h6><span class="bold blue">Attributes</span><br />';
-                foreach ($enrichment_values['Attribute'] as $attribute) {
-                    echo '<div style="padding: 2px;"><pre class="attribute">';
-                    echo '<span class="attribute_type bold blue">' . h($attribute['type']) . '</span>';
-                    echo ': <span class="attribute_value red">' . h($attribute['value']) . '</span></pre></div>';
-                }
-                unset($enrichment_values['Attribute']);
-            }
-            foreach ($enrichment_values as $attributes):
-                foreach ($attributes as $attribute):
-                    echo '<div style="padding: 2px;">';
-                    if (is_array($attribute)) {
-                        foreach ($attribute as $attribute_name => $attribute_value) {
-                            if (!is_numeric($attribute_name)) {
-                                echo '<span class="hover_enrichment_text blue">' . h($attribute_name) . ':</span>';
-                            }
-                            echo '<span><pre class="hover_enrichment_text red">' . h($attribute_value) . '</pre></span>';
-                        }
-                    } else {
-                        echo '<span><pre class="hover_enrichment_text red ">' . h($attribute) . '</pre></span>';
-                    }
-                    echo '</div>';
-                endforeach;
-            endforeach;
-            echo "<br/>";
-        endforeach;
-    ?>
+            echo '</div>';
+        }
+    }
+}
+?>
 </div>

app/View/Attributes/index.ctp

@@ -63,11 +63,12 @@
             __('Actions')
         );
         foreach ($headers as $k => &$header) {
-            $header = sprintf('<th>%s</th>', $header);
+            $header = "<th>$header</th>";
         }
-        $rows = array(
-            sprintf('<tr>%s</tr>', implode('', $headers))
-        );
+        ?>
+    <table class="table table-striped table-hover table-condensed">
+        <tr><?= implode('', $headers) ?></tr>
+        <?php
         $currentCount = 0;
         if ($isSearch == 1) {
             // sanitize data
@@ -94,11 +95,10 @@
             $mayModify = ($isSiteAdmin || ($isAclModify && $event['Event']['user_id'] == $me['id'] && $attribute['Event']['orgc_id'] == $me['org_id']) || ($isAclModifyOrg && $attribute['Event']['orgc_id'] == $me['org_id']));
             $mayPublish = ($isAclPublish && $attribute['Event']['orgc_id'] == $me['org_id']);
             $mayChangeCorrelation = !Configure::read('MISP.completely_disable_correlation') && ($isSiteAdmin || ($mayModify && Configure::read('MISP.allow_disabling_correlation')));
-            $mayModify = $attribute['Event']['orgc_id'] === $me['org_id'] ? true : false;
             if (!empty($attribute['Attribute']['RelatedAttribute'])) {
                 $event['RelatedAttribute'] = array($attribute['Attribute']['id'] => $attribute['Attribute']['RelatedAttribute']);
             }
-            $rows[] =  $this->element('/Events/View/row_attribute', array(
+            echo $this->element('/Events/View/row_attribute', array(
                 'object' => $attribute['Attribute'],
                 'k' => $k,
                 'mayModify' => $mayModify,
@@ -113,8 +113,8 @@
                 'context' => 'list'
             ));
         }
-        echo sprintf('<table class="table table-striped table-hover table-condensed">%s</table>', implode('', $rows));
     ?>
+    </table>
     <p>
     <?php
     echo $this->Paginator->counter(array(
@@ -144,7 +144,7 @@ if ($isSearch == 1){
 ?>
 <script type="text/javascript">
 // tooltips
-$(document).ready(function () {
+$(function () {
     $("td, div").tooltip({
         'placement': 'top',
         'container' : 'body',

app/View/Communities/index.ctp

@@ -100,10 +100,5 @@
         $('#quickFilterButton').click(function() {
             runIndexQuickFilter('/context:' + passedArgsArray['context']);
         });
-        $('#quickFilterField').on('keypress', function (e) {
-            if(e.which === 13) {
-                runIndexQuickFilter('/context:' + passedArgsArray['context']);
-            }
-        });
     });
 </script>

app/View/Dashboards/list_templates.ctp

@@ -98,10 +98,5 @@
         $('#quickFilterButton').click(function() {
             runIndexQuickFilter();
         });
-        $('#quickFilterField').on('keypress', function (e) {
-            if(e.which === 13) {
-                runIndexQuickFilter();
-            }
-        });
     });
 </script>

app/View/Elements/EventReports/reportEditor.ctp

@@ -0,0 +1,19 @@
+<script>
+    'use strict';
+    var proxyMISPElements = null
+    var eventid = '<?= !isset($eventid) ? '' : h($eventid) ?>'
+    var reportid = '<?= h($reportid) ?>'
+    var invalidMessage = '<?= __('invalid scope or id') ?>'
+</script>
+
+<?php
+    echo $this->element('genericElements/assetLoader', [
+        'js' => [
+            'markdownEditor/event-report',
+        ],
+        'css' => [
+            'markdownEditor/event-report',
+            ]
+    ]);
+    
+?>

app/View/Elements/EventReports/reportHelpModal.ctp

@@ -0,0 +1,74 @@
+<?php
+    $formatDifferences = [
+        __('No html support, typographer & autolinker'),
+        __('An additional syntax to reference MISP Elements'),
+    ];
+    $allowedScopes = ['attribute', 'object', 'galaxymatrix' ,'tag'];
+    $allowedScopesHtml = '<code>' . implode('</code> <code>', $allowedScopes) . '</code>';
+?>
+
+<h2><?= __('Markdown format') ?></h2>
+<p><?= __('The suported markdown format is similar to %s with some differences:', sprintf('<a href="%s" target="_blank">GFM</a>', 'https://github.github.com/gfm/')) ?></p>
+<ul>
+    <?php foreach($formatDifferences as $formatDifference): ?>
+        <li><?= $formatDifference ?></li>
+    <?php endforeach; ?>
+</ul>
+
+<h2><?= __('Markdown extended format') ?></h2>
+<p><?= __('In order to have a visually pleasant report but more importantly, avoid hardcoding element\'s value or ID, MISP elements such as attributes and objects can be referenced with the following special syntax') ?></p>
+<h4 style="text-align: center;">
+    <code style="font-size: 14px;">@[scope](UUID)</code>
+</h4>
+<span><?= __('Where:') ?></span>
+<ul>
+    <li><b>scope</b>: <?= __('Is the scope to which the UUID is related to.') ?></li>
+    <ul>
+        <li><?= __('Can be one of the following: %s', $allowedScopesHtml) ?></li>
+    </ul>
+    <li><b>UUID</b>: <?= __('Is the UUID of the MISP element with only one exception for the tag') ?></li>
+</ul>
+<span><?= __('Examples:') ?></span>
+<ul>
+    <li><code>@[attribute](5f1accda-cde4-47fc-baf1-6ab8f331dc3b)</code></li>
+    <li><code>@[object](5f1accda-cde4-47fc-baf1-6ab8f331dc3b)</code></li>
+    <li><code>@[galaxymatrix](5f1accda-cde4-47fc-baf1-6ab8f331dc3b)</code></li>
+</ul>
+
+<h4><?= __('Pictures from attachment-type attributes') ?></h4>
+<p><?= __('Syntax for pictures is like the syntax for referencing MISP elements but with two differences:') ?></p>
+<ul>
+    <li><?= __('The addition of the %s character to indicate that the picture should be displayed and not the atttribute', '<code>!</code>') ?></li>
+    <li><?= __('The scope is fixed to %s as only attributes can contain a file', '<code>attribute</code>') ?></li>
+</ul>
+
+<h4 style="text-align: center;">
+    <code style="font-size: 14px;">@![attribute](UUID)</code>
+</h4>
+<span><?= __('Examples:') ?></span>
+<ul>
+    <li><code>@![attribute](5f1accda-cde4-47fc-baf1-6ab8f331dc3b)</code></li>
+</ul>
+
+<h4><?= __('Tags') ?></h4>
+<p><?=  __('Syntax for representing tags is similar the syntax for referencing MISP elements but with two differences:') ?></p>
+<ul>
+    <li><?= __('The scope is fixed to %s', '<code>tag</code>') ?></li>
+    <li><?= __('The UUID is replaced by the tag name sa tags don\'t have UUID') ?></li>
+</ul>
+<span><?= __('Examples:') ?></span>
+<ul>
+    <li><code>@[tag](tlp:green)</code></li>
+    <li><code>@[tag](misp-galaxy:threat-actor="APT 29")</code></li>
+</ul>
+
+<h4><?= __('Event\'s Galaxy matrixes') ?></h4>
+<p><?=  __('Syntax for embedding the ATT&CK matrix or any other galaxy matrixes is similar to the syntax for referencing MISP elements:') ?></p>
+<ul>
+    <li><?= __('The scope is fixed to %s', '<code>galaxymatrix</code>') ?></li>
+    <li><?= __('The matrix will be generated for the whole event for which the report is linked to') ?></li>
+</ul>
+<span><?= __('Examples:') ?></span>
+<ul>
+    <li><code>@[galaxymatrix](5f1accda-cde4-47fc-baf1-6ab8f331dc3b)</code></li>
+</ul>

app/View/Elements/Events/View/attribute_correlations.ctp

@@ -14,7 +14,7 @@ $event['Related' . $scope][$object['id']] = array_values($event['Related' . $sco
 $count = count($event['Related' . $scope][$object['id']]);
 foreach ($event['Related' . $scope][$object['id']] as $relatedAttribute) {
   if ($i == 4 && $count > 5) {
-    $expandButton = __('Show ') . ($count - 4) . __(' more...');
+    $expandButton = __('Show %s more...', $count - 4);
     echo sprintf(
       '<li class="no-side-padding correlation-expand-button useCursorPointer linkButton %s">%s</li>',
       $linkColour,
@@ -24,12 +24,12 @@ foreach ($event['Related' . $scope][$object['id']] as $relatedAttribute) {
   $relatedData = array(
     'Orgc' => !empty($orgTable[$relatedAttribute['org_id']]) ? $orgTable[$relatedAttribute['org_id']] : 'N/A',
     'Date' => isset($relatedAttribute['date']) ? $relatedAttribute['date'] : 'N/A',
-    'Info' => $relatedAttribute['info'],
+    'Event' => $relatedAttribute['info'],
     'Correlating Value' => $relatedAttribute['value']
   );
   $popover = '';
   foreach ($relatedData as $k => $v) {
-    $popover .= '<span class=\'bold black\'>' . h($k) . '</span>: <span class="blue">' . h($v) . '</span><br />';
+    $popover .= '<span class="bold black">' . h($k) . '</span>: <span class="blue">' . h($v) . '</span><br>';
   }
   $link = $this->Html->link(
     $relatedAttribute['id'],
@@ -53,4 +53,3 @@ if ($i > 5) {
     __('Collapse…')
   );
 }
-?>

app/View/Elements/Events/View/eventSightingValue.ctp

@@ -1,21 +1,29 @@
 <?php
-    echo sprintf(
-        '%s (%s) %s %s',
-        sprintf(
-            '<span id="eventSightingCount" class="bold sightingsCounter" data-toggle="popover" data-trigger="hover" data-content="%s">%s</span>',
-            $sightingPopover,
-            count($event['Sighting'])
-        ),
-        sprintf(
-            '<span id="eventOwnSightingCount" class="green bold sightingsCounter" data-toggle="popover" data-trigger="hover" data-content="%s">%s</span>',
-            $sightingPopover,
-            isset($ownSightings) ? count($ownSightings) : 0
-        ),
-        (Configure::read('Plugin.Sightings_policy')) ? '' : __('- restricted to own organisation only.'),
-        sprintf(
-            '<span class="icon-wrench useCursorPointer sightings_advanced_add" title="%s" role="button" tabindex="0" aria-label="%s" data-object-id="%s" data-object-context="event">&nbsp;</span>',
-             __('Advanced Sightings'),
-            __('Advanced Sightings'),
-            h($event['Event']['id'])
-        )
-    );
+$ownOrgSightingsCount = 0;
+if (isset($event['Sighting'])) {
+    $meOrgId = $this->get('me')['org_id'];
+    foreach ($event['Sighting'] as $sighting) {
+        if (isset($sighting['org_id']) && $sighting['org_id'] == $meOrgId) {
+            ++$ownOrgSightingsCount;
+        }
+    }
+}
+
+echo sprintf(
+    '%s (%s) %s %s',
+    sprintf(
+        '<span id="eventSightingCount" class="bold sightingsCounter">%s</span>',
+        count($event['Sighting'])
+    ),
+    sprintf(
+        '<span id="eventOwnSightingCount" class="green bold sightingsCounter">%s</span>',
+        $ownOrgSightingsCount
+    ),
+    (Configure::read('Plugin.Sightings_policy')) ? '' : __('- restricted to own organisation only.'),
+    sprintf(
+        '<span class="fas fa-wrench useCursorPointer sightings_advanced_add" title="%s" role="button" tabindex="0" aria-label="%s" data-object-id="%s" data-object-context="event">&nbsp;</span>',
+        __('Advanced Sightings'),
+        __('Advanced Sightings'),
+        h($event['Event']['id'])
+    )
+);

app/View/Elements/Events/View/related_event.ctp

@@ -4,26 +4,27 @@
         $href_url .= sprintf('/%s/%s', 1, $from_id);
     }
     $hide = isset($hide) ? $hide : false;
+    $correlationCount = isset($relatedEventCorrelationCount[$related['id']]) ? (int)$relatedEventCorrelationCount[$related['id']] : null;
 ?>
 <span class="<?php echo $hide ? 'hidden correlation-expanded-area' : '' ?>">
-    <span style="display: inline-block; border: 1px solid #ddd; border-radius: 5px; padding: 3px; background-color: white;">
+    <span style="display: inline-block; border: 1px solid #ddd; border-radius: 5px; padding: 3px; background-color: white; line-height: 14px;">
         <table>
             <tbody>
                 <tr>
                     <td rowspan="2" style="border-right: 1px solid #ddd; padding-right: 2px; min-width: 24px; max-width: 24px; overflow: hidden; font-size: xx-small; text-overflow: ellipsis;" title="<?php echo h($related['Orgc']['name']); ?>">
                         <?php echo $this->OrgImg->getOrgImg(array('name' => $related['Orgc']['name'], 'id' => $related['Orgc']['id'], 'size' => 24)); ?>
                     </td>
-                    <td style="line-height: 14px; padding-left: 2px; white-space: nowrap; text-overflow: ellipsis; overflow: hidden; max-width: 410px;">
+                    <td style="padding-left: 2px; white-space: nowrap; text-overflow: ellipsis; overflow: hidden; max-width: 410px;">
                         <a title="<?php echo h($related['info']); ?>" href="<?php echo h($href_url)?>">
                             <?php echo h($related['info']) ?>
                         </a>
                     </td>
                 </tr>
                 <tr>
-                    <td style="line-height: 14px; padding-left: 2px;">
+                    <td style="padding-left: 2px;">
                         <i><?php echo h($related['date']); ?></i>
-                        <?php if (isset($relatedEventCorrelationCount[$related['id']])): ?>
-                            <b style="margin-left: 5px; float: right; cursor: help;" title="<?php echo __('This related event contains %s unique correlation(s)', h($relatedEventCorrelationCount[$related['id']])); ?>"> <?php echo h($relatedEventCorrelationCount[$related['id']]) ?></b>
+                        <?php if (isset($correlationCount)): ?>
+                            <b style="margin-left: 5px; float: right; cursor: help;" title="<?= __n('This related event contains %s unique correlation', 'This related event contains %s unique correlations', $correlationCount, $correlationCount); ?>"> <?=  $correlationCount ?></b>
                         <?php endif; ?>
                     </td>
                 </tr>

app/View/Elements/Events/View/row_attribute.ctp

@@ -1,46 +1,62 @@
 <?php
-  $tr_class = '';
-  if (empty($context)) {
-      $context = 'event';
-  }
-  $linkClass = 'blue';
-  if ($event['Event']['id'] != $object['event_id']) {
+$tr_class = '';
+if (empty($context)) {
+    $context = 'event';
+}
+$linkClass = 'blue';
+if ($event['Event']['id'] != $object['event_id']) {
     if (!$isSiteAdmin && $event['extensionEvents'][$object['event_id']]['Orgc']['id'] != $me['org_id']) {
-      $mayModify = false;
+        $mayModify = false;
     }
-  }
-  $editScope = ($isSiteAdmin || $mayModify) ? 'Attribute' : 'ShadowAttribute';
-  if (!empty($child)) {
+}
+$editScope = ($isSiteAdmin || $mayModify) ? 'Attribute' : 'ShadowAttribute';
+if (!empty($child)) {
     if ($child === 'last' && empty($object['ShadowAttribute'])) {
-      $tr_class .= ' tableHighlightBorderBottom borderBlue';
+        $tr_class .= ' tableHighlightBorderBottom borderBlue';
     } else {
-      $tr_class .= ' tableHighlightBorderCenter borderBlue';
+        $tr_class .= ' tableHighlightBorderCenter borderBlue';
     }
     if (!empty($object['ShadowAttribute'])) {
-      $tr_class .= ' tableInsetOrangeFirst';
+        $tr_class .= ' tableInsetOrangeFirst';
     }
-  } else {
+} else {
     $child = false;
     if (!empty($object['ShadowAttribute'])) {
-      $tr_class .= ' tableHighlightBorderTop borderOrange';
+        $tr_class .= ' tableHighlightBorderTop borderOrange';
     }
-  }
-  if (!empty($object['deleted'])) {
+}
+if (!empty($object['deleted'])) {
     $tr_class .= ' deleted-attribute';
-  }
-  if (!empty($k)) {
+}
+if (!empty($k)) {
     $tr_class .= ' row_' . h($k);
-  }
+}
+
+$objectId = h($object['id']);
+
+$quickEdit = function($fieldName) use ($editScope, $object, $event) {
+    if ($object['deleted']) {
+        return ''; // deleted attributes are not editable
+    }
+    if ($editScope === 'ShadowAttribute') {
+        return ''; // currently it is not supported to create proposals trough quick edit
+    }
+    if ($fieldName === 'value' && ($object['type'] === 'attachment' || $object['type'] === 'malware-sample')) {
+        return '';
+    }
+    return " onmouseenter=\"quickEditHover(this, '$editScope', '{$object['id']}', '$fieldName', {$event['Event']['id']});\"";
+}
+
 ?>
-<tr id = "Attribute_<?php echo h($object['id']); ?>_tr" class="<?php echo $tr_class; ?>" tabindex="0">
+<tr id="Attribute_<?= $objectId ?>_tr" class="<?php echo $tr_class; ?>" tabindex="0">
   <?php
     if (($mayModify || !empty($extended)) && empty($disable_multi_select)):
   ?>
-      <td style="width:10px;" data-position="<?php echo 'attribute_' . h($object['id']); ?>">
+      <td style="width:10px;" data-position="<?php echo 'attribute_' . $objectId ?>">
       <?php
         if ($mayModify):
       ?>
-          <input id = "select_<?php echo $object['id']; ?>" class="select_attribute row_checkbox" type="checkbox" data-id="<?php echo $object['id'];?>" aria-label="<?php echo __('Select attribute');?>" />
+          <input id="select_<?= $objectId ?>" class="select_attribute row_checkbox" type="checkbox" data-id="<?= $objectId ?>" aria-label="<?php echo __('Select attribute');?>" />
       <?php
         endif;
       ?>
@@ -49,11 +65,9 @@
     endif;
   ?>
     <td class="short context hidden">
-      <?php echo h($object['id']); ?>
-    </td>
-    <td class="short context hidden">
-      <?php echo h($object['uuid']); ?>
+      <?= $objectId ?>
     </td>
+    <td class="short context hidden uuid quickSelect"><?php echo h($object['uuid']); ?></td>
     <td class="short context hidden">
         <?php echo $this->element('/Events/View/seen_field', array('object' => $object)); ?>
     </td>
@@ -65,13 +79,10 @@
     ?>
       <td class="short">
         <?php
-            $event_info = '';
-            if (!empty($extended)) {
-                $event_info = sprintf('title="%s%s"',
-                    __('Event info') . ':&#10;     ',
-                    $object['event_id'] != $event['Event']['id'] ? h($event['extensionEvents'][$object['event_id']]['info']) : h($event['Event']['info'])
-                );
-            }
+            $event_info = sprintf('title="%s%s"',
+                __('Event info') . ':&#10;     ',
+                $object['event_id'] != $event['Event']['id'] ? h($event['extensionEvents'][$object['event_id']]['info']) : h($event['Event']['info'])
+            );
         ?>
         <?php echo '<a href="' . $baseurl . '/events/view/' . h($object['event_id']) . '" ' . $event_info . '>' . h($object['event_id']) . '</a>'; ?>
       </td>
@@ -91,13 +102,13 @@
       ?>
       &nbsp;
     </td>
-    <td class="short" onmouseenter="quickEditHover(this, '<?php echo $editScope; ?>', '<?php echo $object['id']; ?>', 'category', <?php echo $event['Event']['id'];?>);">
-      <div id = "Attribute_<?php echo $object['id']; ?>_category_placeholder" class = "inline-field-placeholder"></div>
-      <div id = "Attribute_<?php echo $object['id']; ?>_category_solid" class="inline-field-solid">
+    <td class="short"<?= $quickEdit('category') ?>>
+      <div id="Attribute_<?= $objectId ?>_category_placeholder" class="inline-field-placeholder"></div>
+      <div id="Attribute_<?= $objectId ?>_category_solid" class="inline-field-solid">
         <?php echo h($object['category']); ?>
       </div>
     </td>
-    <td class="short" onmouseenter="quickEditHover(this, '<?php echo $editScope; ?>', '<?php echo $object['id']; ?>', 'type', <?php echo $event['Event']['id'];?>);">
+    <td class="short"<?= $quickEdit('type') ?>>
       <?php
         if (!empty($object['object_relation'])):
       ?>
@@ -105,60 +116,33 @@
       <?php
         endif;
       ?>
-      <div></div>
-      <div id = "Attribute_<?php echo $object['id']; ?>_type_placeholder" class = "inline-field-placeholder"></div>
-      <div id = "Attribute_<?php echo $object['id']; ?>_type_solid" class="inline-field-solid">
+      <div id="Attribute_<?= $objectId ?>_type_placeholder" class="inline-field-placeholder"></div>
+      <div id="Attribute_<?= $objectId ?>_type_solid" class="inline-field-solid">
         <?php echo h($object['type']); ?>
       </div>
     </td>
-    <?php
-        if ('attachment' !== $object['type'] && 'malware-sample' !== $object['type']):
-            $editable = ' onmouseenter="quickEditHover(this, \'' . $editScope . '\', \'' . $object['id'] . '\', \'value\', \'' . $event['Event']['id'] . '\' );"';
-        else:
-            $editable = '';
-        endif;
-    ?>
-    <td id="Attribute_<?php echo h($object['id']); ?>_container" class="showspaces limitedWidth shortish" <?php echo $editable; ?>>
-    <div id="Attribute_<?php echo $object['id']; ?>_value_placeholder" class="inline-field-placeholder"></div>
-      <div id = "Attribute_<?php echo $object['id']; ?>_value_solid" class="inline-field-solid">
-        <span>
+    <td id="Attribute_<?= $objectId ?>_container" class="showspaces limitedWidth shortish"<?= $quickEdit('value') ?>>
+      <div id="Attribute_<?= $objectId ?>_value_placeholder" class="inline-field-placeholder"></div>
+      <div id="Attribute_<?= $objectId ?>_value_solid" class="inline-field-solid">
         <?php
-            $spanExtra = '';
-            $popupButton = '';
             if (Configure::read('Plugin.Enrichment_hover_enable') && isset($modules) && isset($modules['hover_type'][$object['type']])) {
-                $commonDataFields = sprintf(
-                    'data-object-type="%s" data-object-id="%s"',
-                    "Attribute",
-                    h($object['id'])
+                $commonDataFields = sprintf('data-object-type="Attribute" data-object-id="%s"', $objectId);
+                $spanExtra = Configure::read('Plugin.Enrichment_hover_popover_only') ? '' : sprintf(' class="eventViewAttributeHover" %s', $commonDataFields);
+                $popupButton = sprintf('<i class="fa fa-search-plus useCursorPointer eventViewAttributePopup noPrint" title="%s" %s></i>', __('Show hover enrichment'), $commonDataFields);
+                echo sprintf(
+                    '<span%s>%s</span> %s',
+                    $spanExtra,
+                    $this->element('/Events/View/value_field', array('object' => $object, 'linkClass' => $linkClass)),
+                    $popupButton
                 );
-                $spanExtra = sprintf(' class="eventViewAttributeHover" %s', $commonDataFields);
-                $popupButton = sprintf('<i class="fa fa-search-plus useCursorPointer eventViewAttributePopup" %s></i>', $commonDataFields);
+            } else {
+                echo $this->element('/Events/View/value_field', array('object' => $object, 'linkClass' => $linkClass));
             }
-            echo sprintf(
-                '<span%s style="white-space: pre-wrap;">%s</span> %s',
-                $spanExtra,
-                $this->element('/Events/View/value_field', array('object' => $object, 'linkClass' => $linkClass)),
-                $popupButton
-            );
-        ?>
-        </span>
-        <?php
-          if (isset($object['warnings'])) {
-            $temp = '';
-            $components = array(1 => 0, 2 => 1);
-            $valueParts = explode('|', $object['value']);
-            foreach ($components as $component => $valuePart) {
-              if (isset($object['warnings'][$component]) && isset($valueParts[$valuePart])) {
-                foreach ($object['warnings'][$component] as $warning) $temp .= '<span class=\'bold\'>' . h($valueParts[$valuePart]) . '</span>: <span class=\'red\'>' . h($warning) . '</span><br />';
-              }
-            }
-            echo ' <span aria-label="' . __('warning') . '" role="img" tabindex="0" class="fa fa-exclamation-triangle" data-placement="right" data-toggle="popover" data-content="' . h($temp) . '" data-trigger="hover" data-placement="right">&nbsp;</span>';
-          }
         ?>
       </div>
     </td>
     <td class="short">
-      <div class="attributeTagContainer" id="#Attribute_<?php echo h($object['id']);?>_tr .attributeTagContainer">
+      <div class="attributeTagContainer">
         <?php echo $this->element('ajaxTags', array('attributeId' => $object['id'], 'tags' => $object['AttributeTag'], 'tagAccess' => ($isSiteAdmin || $mayModify || $me['org_id'] == $event['Event']['org_id']), 'context' => $context, 'scope' => 'attribute', 'tagConflicts' => isset($object['tagConflicts']) ? $object['tagConflicts'] : array())); ?>
       </div>
     </td>
@@ -170,13 +154,12 @@
             }
             echo sprintf(
                 '<td class="shortish"><div %s>%s</div></td>',
-                'class="attributeRelatedTagContainer" id="#Attribute_' . h($object['id']) . 'Related_tr .attributeTagContainer"',
+                'class="attributeRelatedTagContainer" id="#Attribute_' . $objectId . 'Related_tr .attributeTagContainer"',
                 $element
             );
         }
     ?>
-    <?php $rowId = sprintf('attribute_%s_galaxy', h($object['id'])); ?>
-    <td class="short" id="<?= $rowId ?>">
+    <td class="short" id="<?= sprintf('attribute_%s_galaxy', h($objectId)) ?>">
       <?php
         echo $this->element('galaxyQuickViewMini', array(
           'mayModify' => $mayModify,
@@ -188,20 +171,20 @@
         ));
       ?>
     </td>
-    <td class="showspaces bitwider" onmouseenter="quickEditHover(this, '<?php echo $editScope; ?>', '<?php echo $object['id']; ?>', 'comment', <?php echo $event['Event']['id'];?>);">
-      <div id = "Attribute_<?php echo $object['id']; ?>_comment_placeholder" class = "inline-field-placeholder"></div>
-      <div id = "Attribute_<?php echo $object['id']; ?>_comment_solid" class="inline-field-solid">
+    <td class="showspaces bitwider"<?= $quickEdit('comment') ?>>
+      <div id="Attribute_<?= $objectId ?>_comment_placeholder" class="inline-field-placeholder"></div>
+      <div id="Attribute_<?= $objectId ?>_comment_solid" class="inline-field-solid">
         <?php echo nl2br(h($object['comment'])); ?>&nbsp;
       </div>
     </td>
     <td class="short" style="padding-top:3px;">
       <input
-        id="correlation_toggle_<?php echo h($object['id']); ?>"
+        id="correlation_toggle_<?= $objectId ?>"
         class="correlation-toggle"
         aria-label="<?php echo __('Toggle correlation');?>"
         title="<?php echo __('Toggle correlation');?>"
         type="checkbox"
-        data-attribute-id="<?php echo h($object['id']); ?>"
+        data-attribute-id="<?= $objectId ?>"
         <?php
           echo $object['disable_correlation'] ? '' : ' checked';
           echo ($mayChangeCorrelation && empty($event['Event']['disable_correlation'])) ? '' : ' disabled';
@@ -209,78 +192,73 @@
       >
     </td>
     <td class="shortish">
-      <ul class="inline" style="margin:0px;">
         <?php
           if (!empty($event['RelatedAttribute'][$object['id']])) {
-            echo $this->element('Events/View/attribute_correlations', array(
-              'scope' => 'Attribute',
-              'object' => $object,
-              'event' => $event,
-            ));
+              echo '<ul class="inline" style="margin:0">';
+              echo $this->element('Events/View/attribute_correlations', array(
+                  'scope' => 'Attribute',
+                  'object' => $object,
+                  'event' => $event,
+              ));
+              echo '</ul>';
           }
         ?>
-      </ul>
     </td>
     <td class="shortish">
-      <ul class="inline" style="margin:0px;">
+      <ul class="inline" style="margin:0">
         <?php
-            if (!empty($object['Feed'])) {
+            if (isset($object['Feed'])) {
                 foreach ($object['Feed'] as $feed) {
-                    $popover = '';
-                    foreach ($feed as $k => $v) {
-                        if ($k == 'id') continue;
-                        if (is_array($v)) {
-                            foreach ($v as $k2 => $v2) {
-                                $v[$k2] = h($v2);
-                            }
-                            $v = implode('<br />', $v);
-                        } else {
-                            $v = h($v);
-                        }
-                        $popover .= '<span class=\'bold black\'>' . Inflector::humanize(h($k)) . '</span>: <span class="blue">' . $v . '</span><br />';
+                    $relatedData = array(
+                        __('Name') => $feed['name'],
+                        __('URL') => $feed['url'],
+                        __('Provider') => $feed['provider'],
+                    );
+                    if (isset($feed['event_uuids'])) {
+                        $relatedData[__('Event UUIDs')] = implode('<br>', $feed['event_uuids']);
                     }
-                    $liContents = '';
-                    if ($isSiteAdmin) {
-                        if ($feed['source_format'] == 'misp') {
-                            $liContents .= sprintf(
-                                '<form action="%s/feeds/previewIndex/%s" method="post" style="margin:0px;line-height:auto;">%s%s</form>',
+                    $popover = '';
+                    foreach ($relatedData as $k => $v) {
+                        $popover .= '<span class="bold black">' . h($k) . '</span>: <span class="blue">' . h($v) . '</span><br>';
+                    }
+                    if ($isSiteAdmin || $hostOrgUser) {
+                        if ($feed['source_format'] === 'misp') {
+                            $liContents = sprintf(
+                                '<form action="%s/feeds/previewIndex/%s" method="post" style="margin:0;line-height:auto;">%s%s</form>',
                                 $baseurl,
                                 h($feed['id']),
                                 sprintf(
                                     '<input type="hidden" name="data[Feed][eventid]" value="%s">',
-                                    h(json_encode($feed['event_uuids'], true))
+                                    h(json_encode($feed['event_uuids']))
                                 ),
                                 sprintf(
-                                    '<input type="submit" class="linkButton useCursorPointer" value="%s" data-toggle="popover" data-content="%s" data-trigger="hover" style="margin-right:3px;line-height:normal;vertical-align: text-top;" />',
+                                    '<input type="submit" class="linkButton useCursorPointer" value="%s" data-toggle="popover" data-content="%s" data-trigger="hover" style="margin-right:3px;line-height:normal;vertical-align: text-top;">',
                                     h($feed['id']),
                                     h($popover)
                                 )
                             );
                         } else {
-                            $liContents .= sprintf(
-                                '<form>%s</form>',
-                                sprintf(
-                                    '<a href="%s/feeds/previewIndex/%s" style="margin-right:3px;" data-toggle="popover" data-content="%s" data-trigger="hover">%s</a>',
-                                    $baseurl,
-                                    h($feed['id']),
-                                    h($popover),
-                                    h($feed['id'])
-                                )
+                            $liContents = sprintf(
+                                '<a href="%s/feeds/previewIndex/%s" style="margin-right:3px;" data-toggle="popover" data-content="%s" data-trigger="hover">%s</a>',
+                                $baseurl,
+                                h($feed['id']),
+                                h($popover),
+                                h($feed['id'])
                             );
                         }
                     } else {
-                        $liContents .= sprintf(
+                        $liContents = sprintf(
                             '<span style="margin-right:3px;">%s</span>',
                             h($feed['id'])
                         );
                     }
                     echo sprintf(
-                        '<li style="padding-right: 0px; padding-left:0px;">%s</li>',
+                        '<li style="padding-right: 0; padding-left:0;">%s</li>',
                         $liContents
                     );
                 }
             }
-            if (!empty($object['Server'])) {
+            if (isset($object['Server'])) {
                 foreach ($object['Server'] as $server) {
                     $popover = '';
                     foreach ($server as $k => $v) {
@@ -313,7 +291,7 @@
                             );
                         }
                         echo sprintf(
-                            '<li style="padding-right: 0px; padding-left:0px;">%s</li>',
+                            '<li style="padding-right:0; padding-left:0;">%s</li>',
                             $liContents
                         );
                     }
@@ -323,20 +301,20 @@
       </ul>
     </td>
     <td class="short">
-        <div id = "Attribute_<?php echo $object['id']; ?>_to_ids_placeholder" class = "inline-field-placeholder"></div>
-        <div id = "Attribute_<?php echo $object['id']; ?>_to_ids_solid" class="inline-field-solid">
-            <input type="checkbox" class="toids-toggle" id="toids_toggle_<?php echo h($object['id']); ?>" data-attribute-id="<?php echo h($object['id']); ?>" aria-label="<?php echo __('Toggle IDS flag');?>" title="<?php echo __('Toggle IDS flag');?>" <?php echo $object['to_ids'] ? 'checked' : ''; ?> >
+        <div id="Attribute_<?= $objectId ?>_to_ids_placeholder" class="inline-field-placeholder"></div>
+        <div id="Attribute_<?= $objectId ?>_to_ids_solid" class="inline-field-solid">
+            <input type="checkbox" class="toids-toggle" id="toids_toggle_<?= $objectId ?>" data-attribute-id="<?= $objectId ?>" aria-label="<?= __('Toggle IDS flag') ?>" title="<?= __('Toggle IDS flag') ?>"<?= $object['to_ids'] ? ' checked' : ''; ?><?= $mayModify ? '' : ' disabled' ?>>
         </div>
     </td>
-    <td class="short" onmouseenter="quickEditHover(this, '<?php echo $editScope; ?>', '<?php echo $object['id']; ?>', 'distribution', <?php echo $event['Event']['id'];?>);">
+    <td class="short"<?= $quickEdit('distribution') ?>>
         <?php
             $turnRed = '';
             if ($object['distribution'] == 0) {
                 $turnRed = 'style="color:red"';
             }
         ?>
-        <div id = "Attribute_<?php echo $object['id']; ?>_distribution_placeholder" class = "inline-field-placeholder"></div>
-        <div id = "Attribute_<?php echo $object['id']; ?>_distribution_solid" <?php echo $turnRed; ?> class="inline-field-solid">
+        <div id="Attribute_<?= $objectId ?>_distribution_placeholder" class="inline-field-placeholder"></div>
+        <div id="Attribute_<?= $objectId ?>_distribution_solid" <?php echo $turnRed; ?> class="inline-field-solid">
             <?php
                 if ($object['distribution'] == 4):
             ?>
@@ -351,19 +329,15 @@
     <?php
         echo $this->element('/Events/View/sighting_field', array(
           'object' => $object,
-          'tr_class' => $tr_class,
-          'page' => $page
         ));
         if (!empty($includeSightingdb)) {
             echo $this->element('/Events/View/sightingdb_field', array(
               'object' => $object,
-              'tr_class' => $tr_class,
-              'page' => $page
             ));
         }
         if (!empty($includeDecayScore)): ?>
             <td class="decayingScoreField">
-                  <div id = "Attribute_<?php echo h($object['id']); ?>_score_solid" class="inline-field-solid">
+                  <div id="Attribute_<?= $objectId ?>_score_solid" class="inline-field-solid">
                     <?php echo $this->element('DecayingModels/View/attribute_decay_score', array('scope' => 'object', 'object' => $object, 'uselink' => true)); ?>
                   </div>
             </td>
@@ -375,25 +349,25 @@
         if ($object['deleted']):
           if ($isSiteAdmin || $mayModify):
       ?>
-          <span class="icon-repeat useCursorPointer" title="<?php echo __('Restore attribute');?>" role="button" tabindex="0" aria-label="<?php echo __('Restore attribute');?>" onClick="deleteObject('attributes', 'restore', '<?php echo h($object['id']); ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
-          <span class="fa fa-trash useCursorPointer" title="<?php echo __('Permanently delete attribute');?>" role="button" tabindex="0" aria-label="i<?php echo __('Permanently delete attribute');?>" onClick="deleteObject('attributes', 'delete', '<?php echo h($object['id']) . '/true'; ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
+          <span class="fas fa-redo useCursorPointer" title="<?php echo __('Restore attribute');?>" role="button" tabindex="0" aria-label="<?php echo __('Restore attribute');?>" onClick="deleteObject('attributes', 'restore', '<?= $objectId ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
+          <span class="fa fa-trash useCursorPointer" title="<?php echo __('Permanently delete attribute');?>" role="button" tabindex="0" aria-label="i<?php echo __('Permanently delete attribute');?>" onClick="deleteObject('attributes', 'delete', '<?= $objectId . '/true'; ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
       <?php
           endif;
         else:
           if ($isAclAdd && ($isSiteAdmin || !$mayModify)):
             if (isset($modules) && isset($modules['types'][$object['type']])):
       ?>
-        <span class="icon-asterisk useCursorPointer" title="<?php echo __('Query enrichment');?>" role="button" tabindex="0" aria-label="<?php echo __('Query enrichment');?>" onClick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?php echo h($object['id']);?>/ShadowAttribute');" title="<?php echo __('Propose enrichment');?>">&nbsp;</span>
+        <span class="fas fa-asterisk useCursorPointer" role="button" tabindex="0" aria-label="<?php echo __('Query enrichment');?>" onClick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/ShadowAttribute');" title="<?php echo __('Propose enrichment');?>">&nbsp;</span>
       <?php
             endif;
             if (isset($cortex_modules) && isset($cortex_modules['types'][$object['type']])):
       ?>
-        <span class="icon-eye-open useCursorPointer" title="<?php echo __('Query Cortex');?>" role="button" tabindex="0" aria-label="<?php echo __('Query Cortex');?>" onClick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?php echo h($object['id']);?>/ShadowAttribute/Cortex');" title="<?php echo __('Propose enrichment through Cortex');?>"></span>
+        <span class="icon-eye-open useCursorPointer" title="<?php echo __('Query Cortex');?>" role="button" tabindex="0" aria-label="<?php echo __('Query Cortex');?>" onClick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/ShadowAttribute/Cortex');" title="<?php echo __('Propose enrichment through Cortex');?>"></span>
       <?php
             endif;
       ?>
-            <a href="<?php echo $baseurl;?>/shadow_attributes/edit/<?php echo $object['id']; ?>" title="<?php echo __('Propose Edit');?>" aria-label="<?php echo __('Propose Edit');?>" class="fa fa-comment useCursorPointer"></a>
-            <span class="fa fa-trash useCursorPointer" title="<?php echo __('Propose Deletion');?>" role="button" tabindex="0" aria-label="Propose deletion" onClick="deleteObject('shadow_attributes', 'delete', '<?php echo h($object['id']); ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
+            <a href="<?php echo $baseurl;?>/shadow_attributes/edit/<?= $objectId ?>" title="<?php echo __('Propose Edit');?>" aria-label="<?php echo __('Propose Edit');?>" class="fa fa-comment useCursorPointer"></a>
+            <span class="fa fa-trash useCursorPointer" title="<?php echo __('Propose Deletion');?>" role="button" tabindex="0" aria-label="Propose deletion" onClick="deleteObject('shadow_attributes', 'delete', '<?= $objectId ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
       <?php
             if ($isSiteAdmin):
       ?>
@@ -403,24 +377,24 @@
           if ($isSiteAdmin || $mayModify):
             if (isset($modules) && isset($modules['types'][$object['type']])):
       ?>
-        <span class="icon-asterisk useCursorPointer" onClick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?php echo h($object['id']);?>/Attribute');" title="<?php echo __('Add enrichment');?>" role="button" tabindex="0" aria-label="<?php echo __('Add enrichment');?>">&nbsp;</span>
+        <span class="fas fa-asterisk useCursorPointer" onClick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/Attribute');" title="<?php echo __('Add enrichment');?>" role="button" tabindex="0" aria-label="<?php echo __('Add enrichment');?>">&nbsp;</span>
       <?php
             endif;
             if (isset($cortex_modules) && isset($cortex_modules['types'][$object['type']])):
       ?>
-        <span class="icon-eye-open useCursorPointer" onClick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?php echo h($object['id']);?>/Attribute/Cortex');" title="<?php echo __('Add enrichment');?>" role="button" tabindex="0" aria-label="<?php echo __('Add enrichment via Cortex');?>"></span>
+        <span class="icon-eye-open useCursorPointer" onClick="simplePopup('<?php echo $baseurl;?>/events/queryEnrichment/<?= $objectId ?>/Attribute/Cortex');" title="<?php echo __('Add enrichment');?>" role="button" tabindex="0" aria-label="<?php echo __('Add enrichment via Cortex');?>"></span>
       <?php
             endif;
       ?>
-            <a href="<?php echo $baseurl;?>/attributes/edit/<?php echo $object['id']; ?>" title="<?php echo __('Edit');?>" aria-label="<?php echo __('Edit');?>" class="fa fa-edit useCursorPointer"></a>
+            <a href="<?php echo $baseurl;?>/attributes/edit/<?= $objectId ?>" title="<?php echo __('Edit');?>" aria-label="<?php echo __('Edit');?>" class="fa fa-edit useCursorPointer"></a>
           <?php
             if (empty($event['Event']['publish_timestamp'])):
           ?>
-            <span class="fa fa-trash useCursorPointer" title="<?php echo __('Permanently delete attribute');?>" role="button" tabindex="0" aria-label="i<?php echo __('Permanently delete attribute');?>" onClick="deleteObject('attributes', 'delete', '<?php echo h($object['id']) . '/true'; ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
+            <span class="fa fa-trash useCursorPointer" title="<?php echo __('Permanently delete attribute');?>" role="button" tabindex="0" aria-label="i<?php echo __('Permanently delete attribute');?>" onClick="deleteObject('attributes', 'delete', '<?= $objectId . '/true'; ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
           <?php
             else:
           ?>
-            <span class="fa fa-trash useCursorPointer" title="<?php echo __('Soft-delete attribute');?>" role="button" tabindex="0" aria-label="<?php echo __('Soft-delete attribute');?>" onClick="deleteObject('attributes', 'delete', '<?php echo h($object['id']); ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
+            <span class="fa fa-trash useCursorPointer" title="<?php echo __('Soft-delete attribute');?>" role="button" tabindex="0" aria-label="<?php echo __('Soft-delete attribute');?>" onClick="deleteObject('attributes', 'delete', '<?= $objectId ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
           <?php
             endif;
           endif;
@@ -429,19 +403,18 @@
   </td>
 </tr>
 <?php
-  if (!empty($object['ShadowAttribute'])) {
+if (!empty($object['ShadowAttribute'])) {
     end($object['ShadowAttribute']);
     $lastElement = key($object['ShadowAttribute']);
     foreach ($object['ShadowAttribute'] as $propKey => $proposal) {
-      echo $this->element('/Events/View/row_' . $proposal['objectType'], array(
-        'object' => $proposal,
-        'mayModify' => $mayModify,
-        'mayChangeCorrelation' => $mayChangeCorrelation,
-        'page' => $page,
-        'fieldCount' => $fieldCount,
-        'child' => $propKey == $lastElement ? 'last' : true,
-        'objectContainer' => $child
-      ));
+        echo $this->element('/Events/View/row_' . $proposal['objectType'], array(
+            'object' => $proposal,
+            'mayModify' => $mayModify,
+            'mayChangeCorrelation' => $mayChangeCorrelation,
+            'fieldCount' => $fieldCount,
+            'child' => $propKey == $lastElement ? 'last' : true,
+            'objectContainer' => $child
+        ));
     }
-  }
-?>
+}
+

app/View/Elements/Events/View/row_object.ctp

@@ -1,5 +1,4 @@
 <?php
-  $tr_class = '';
   $linkClass = 'white';
   $currentType = 'Object';
   $tr_class = 'tableHighlightBorderTop borderBlue';
@@ -18,24 +17,14 @@
   <?php
     if ($mayModify || $extended):
   ?>
-    <td style="width:10px;" data-position="<?php echo h($object['objectType']) . '_' . h($object['id']); ?>">
-      <?php
-        if ($mayModify):
-      ?>
-        <input id="select_object_<?php echo $object['id']; ?>" class="select_object row_checkbox" type="checkbox" data-id="<?php echo $object['id'];?>" />
-      <?php
-        endif;
-      ?>
-    </td>
+    <td style="width:10px;"></td>
   <?php
     endif;
   ?>
   <td class="short context hidden">
     <?php echo h($object['id']); ?>
   </td>
-  <td class="short context hidden">
-    <?php echo h($object['uuid']); ?>
-  </td>
+  <td class="short context hidden uuid quickSelect"><?php echo h($object['uuid']); ?></td>
   <td class="short context hidden">
       <?php echo $this->element('/Events/View/seen_field', array('object' => $object)); ?>
   </td>
@@ -99,12 +88,11 @@
   <td class="shortish" onmouseenter="quickEditHover(this, 'Object', '<?php echo $object['id']; ?>', 'distribution', <?php echo $event['Event']['id'];?>);">
     <?php
       $turnRed = '';
-      if ($object['objectType'] == 0 && $object['distribution'] == 0) $turnRed = 'style="color:red"';
+      if ($object['distribution'] == 0) $turnRed = 'style="color:red"';
     ?>
     <div id="<?php echo $currentType . '_' . $object['id'] . '_distribution_placeholder'; ?>" class="inline-field-placeholder"></div>
     <div id="<?php echo $currentType . '_' . $object['id'] . '_distribution_solid'; ?>" <?php echo $turnRed; ?> class="inline-field-solid">
       <?php
-        if ($object['objectType'] == 0) {
           if ($object['distribution'] == 4):
       ?>
         <a href="<?php echo $baseurl; ?>/sharing_groups/view/<?php echo h($object['sharing_group_id']); ?>"><?php echo h($object['SharingGroup']['name']);?></a>
@@ -112,7 +100,6 @@
           else:
             echo h($shortDist[$object['distribution']]);
           endif;
-        }
       ?>&nbsp;
     </div>
   </td>
@@ -130,9 +117,11 @@
     <?php
       if ($mayModify && empty($object['deleted'])) {
         echo sprintf(
-          '<a href="%s/objects/edit/%s" title="Edit" aria-label="Edit" class="fa fa-edit white useCursorPointer"></a>',
+          '<a href="%s/objects/edit/%s" title="%s" aria-label="%s" class="fa fa-edit white useCursorPointer"></a> ',
           $baseurl,
-          h($object['id'])
+          h($object['id']),
+          __('Edit'),
+          __('Edit')
         );
         echo sprintf(
           '<span class="fa fa-trash white useCursorPointer" title="%1$s" role="button" tabindex="0" aria-label="%1$s" onClick="%2$s"></span>',
@@ -156,7 +145,6 @@
         'object' => $attribute,
         'mayModify' => $mayModify,
         'mayChangeCorrelation' => $mayChangeCorrelation,
-        'page' => $page,
         'fieldCount' => $fieldCount,
         'child' => $attrKey == $lastElement ? 'last' : true
       ));

app/View/Elements/Events/View/row_object_reference.ctp

@@ -32,7 +32,7 @@
 <div id="Object_<?php echo $object['id']; ?>_references_collapsible" class="collapse">
 <?php
   foreach ($object['ObjectReference'] as $reference):
-    if (!empty($reference['Object'])) {
+    if (isset($reference['Object'])) {
       $uuid = $reference['Object']['uuid'];
       $output = ' (' . $reference['Object']['name'] . ': ' . $reference['Object']['name'] . ')';
       $objectType = 'Object';
@@ -43,13 +43,12 @@
     }
     $uuid = empty($reference['Object']) ? $reference['Attribute']['uuid'] : $reference['Object']['uuid'];
     $idref = $reference['deleted'] ? $reference['id'] . '/1' : $reference['id'];
+    $linkText = h($reference['relationship_type']) . ' ' . $objectType . ' ' . $reference['referenced_id'] . h($output);
 ?>
     &nbsp;&nbsp;
-    <a class="bold white useCursorPointer <?php echo $reference['deleted'] ? 'strikethrough' : ''; ?>" onClick="pivotObjectReferences('<?php echo h($currentUri); ?>', '<?php echo $uuid; ?>')">
-      <?php echo h($reference['relationship_type']) . ' ' . $objectType . ' ' . $reference['referenced_id'] . h($output);?>
-    </a>
+    <a class="bold white useCursorPointer<?= $reference['deleted'] ? ' strikethrough' : ''; ?>" onclick="pivotObjectReferences('<?= h($currentUri) ?>', '<?= $uuid ?>')"><?= $linkText ?></a>
     <span class="fa fa-trash icon-white useCursorPointer" title="<?php echo __('Delete object reference');?>" role="button" tabindex="0" aria-label="<?php echo __('Delete object reference');?>" onClick="deleteObject('object_references', 'delete', '<?php echo h($idref); ?>', '<?php echo h($event['Event']['id']); ?>');"></span>
-    <br />
+    <br>
 <?php
   endforeach;
 ?>

app/View/Elements/Events/View/row_proposal.ctp

@@ -33,9 +33,8 @@
       $tr_class .= ' tableHighlightBorder borderOrange';
     }
   }
-  $identifier = (empty($k)) ? '' : ' id="row_' . h($k) . '" tabindex="0"';
 ?>
-<tr id = "proposal<?php echo '_' . $object['id'] . '_tr'; ?>" class="<?php echo $tr_class; ?>" <?php echo $identifier; ?>>
+<tr id="proposal_<?= $object['id'] ?>_tr" class="<?php echo $tr_class; ?>">
   <?php if ($mayModify): ?>
     <td style="width:10px;" data-position="<?php echo h($object['objectType']) . '_' . h($object['id']); ?>">
       <input id = "select_proposal_<?php echo $object['id']; ?>" class="select_proposal row_checkbox" type="checkbox" aria-label="<?php __('Select proposal');?>" data-id="<?php echo $object['id'];?>" />
@@ -43,17 +42,15 @@
   <?php endif; ?>
   <td class="short context hidden">
     <?php
-      echo $object['objectType'] == 0 ? h($object['id']) : '&nbsp;';
+      echo h($object['id']);
     ?>
   </td>
-  <td class="short context hidden">
-    <?php echo $object['objectType'] == 0 ? h($object['uuid']) : '&nbsp;'; ?>
-  </td>
+  <td class="short context hidden uuid quickSelect"><?= h($object['uuid']) ?></td>
   <td class="short context hidden">
       <?php echo $this->element('/Events/View/seen_field', array('object' => $object)); ?>
   </td>
   <td class="short">
-    <div id = "<?php echo $currentType . '_' . $object['id'] . '_timestamp_solid'; ?>">
+    <div id="<?php echo $currentType . '_' . $object['id'] . '_timestamp_solid'; ?>">
       <?php
         if (isset($object['timestamp'])) echo date('Y-m-d', $object['timestamp']);
         else echo '&nbsp';
@@ -77,49 +74,30 @@
   ?>
   </td>
   <td class="short">
-    <div id = "<?php echo $currentType . '_' . $object['id'] . '_category_placeholder'; ?>" class = "inline-field-placeholder"></div>
-    <div id = "<?php echo $currentType . '_' . $object['id'] . '_category_solid'; ?>" class="inline-field-solid" ondblclick="activateField('<?php echo $currentType; ?>', '<?php echo $object['id']; ?>', 'category', <?php echo $event['Event']['id'];?>);">
+    <div id="<?php echo $currentType . '_' . $object['id'] . '_category_placeholder'; ?>" class="inline-field-placeholder"></div>
+    <div id="<?php echo $currentType . '_' . $object['id'] . '_category_solid'; ?>" class="inline-field-solid">
       <?php echo h($object['category']); ?>
     </div>
   </td>
   <td class="short">
-    <div id = "<?php echo $currentType . '_' . $object['id'] . '_type_placeholder'; ?>" class = "inline-field-placeholder"></div>
-    <div id = "<?php echo $currentType . '_' . $object['id'] . '_type_solid'; ?>" class="inline-field-solid" ondblclick="activateField('<?php echo $currentType; ?>', '<?php echo $object['id']; ?>', 'type', <?php echo $event['Event']['id'];?>);">
+    <div id="<?php echo $currentType . '_' . $object['id'] . '_type_placeholder'; ?>" class="inline-field-placeholder"></div>
+    <div id="<?php echo $currentType . '_' . $object['id'] . '_type_solid'; ?>" class="inline-field-solid">
       <?php echo h($object['type']); ?>
     </div>
   </td>
   <td id="<?php echo h($currentType) . '_' . h($object['id']) . '_container'; ?>" class="showspaces limitedWidth shortish">
-    <div id = "<?php echo $currentType . '_' . $object['id'] . '_value_placeholder'; ?>" class = "inline-field-placeholder"></div>
-    <?php
-      if ('attachment' !== $object['type'] && 'malware-sample' !== $object['type']) $editable = ' ondblclick="activateField(\'' . $currentType . '\', \'' . $object['id'] . '\', \'value\', \'' . $event['Event']['id'] . '\');"';
-      else $editable = '';
-    ?>
-    <div id = "<?php echo $currentType; ?>_<?php echo $object['id']; ?>_value_solid" class="inline-field-solid" <?php echo $editable; ?>>
-      <span <?php if (Configure::read('Plugin.Enrichment_hover_enable') && isset($modules) && isset($modules['hover_type'][$object['type']])) echo 'class="eventViewAttributeHover" data-object-type="' . h($currentType) . '" data-object-id="' . h($object['id']) . '"'?>>
+    <div id="<?php echo $currentType . '_' . $object['id'] . '_value_placeholder'; ?>" class="inline-field-placeholder"></div>
+    <div id="<?php echo $currentType; ?>_<?php echo $object['id']; ?>_value_solid" class="inline-field-solid">
     <?php
         echo $this->element('/Events/View/value_field', array('object' => $object, 'linkClass' => $linkClass));
     ?>
-      </span>
-      <?php
-        if (isset($object['warnings'])) {
-          $temp = '';
-          $components = array(1 => 0, 2 => 1);
-          $valueParts = explode('|', $object['value']);
-          foreach ($components as $component => $valuePart) {
-            if (isset($object['warnings'][$component]) && isset($valueParts[$valuePart])) {
-              foreach ($object['warnings'][$component] as $warning) $temp .= '<span class=\'bold\'>' . h($valueParts[$valuePart]) . '</span>: <span class=\'red\'>' . h($warning) . '</span><br />';
-            }
-          }
-          echo ' <span aria-label="' . __('warning') . '" role="img" tabindex="0" class="fa fa-exclamation-triangle white" data-placement="right" data-toggle="popover" data-content="' . h($temp) . '" data-trigger="hover">&nbsp;</span>';
-        }
-      ?>
     </div>
   </td>
   <td class="shortish">&nbsp;</td>
   <td class="shortish">&nbsp;</td>
   <td class="showspaces bitwider">
-    <div id = "<?php echo $currentType . '_' . $object['id'] . '_comment_placeholder'; ?>" class = "inline-field-placeholder"></div>
-    <div id = "<?php echo $currentType . '_' . $object['id'] . '_comment_solid'; ?>" class="inline-field-solid" ondblclick="activateField('<?php echo $currentType; ?>', '<?php echo $object['id']; ?>', 'comment', <?php echo $event['Event']['id'];?>);">
+    <div id="<?php echo $currentType . '_' . $object['id'] . '_comment_placeholder'; ?>" class="inline-field-placeholder"></div>
+    <div id="<?php echo $currentType . '_' . $object['id'] . '_comment_solid'; ?>" class="inline-field-solid">
       <?php echo nl2br(h($object['comment'])); ?>&nbsp;
     </div>
   </td>
@@ -148,7 +126,7 @@
               $popover .= '<span class=\'bold black\'>' . Inflector::humanize(h($k)) . '</span>: <span class="blue">' . h($v) . '</span><br />';
             endforeach;
           ?>
-            <li style="padding-right: 0px; padding-left:0px;"  data-toggle="popover" data-content="<?php echo h($popover);?>" data-trigger="hover"><span>
+            <li style="padding-right: 0px; padding-left:0px;" data-toggle="popover" data-content="<?php echo h($popover);?>" data-trigger="hover"><span>
               <?php
                 if ($isSiteAdmin):
                   echo $this->Html->link($feed['id'], array('controller' => 'feeds', 'action' => 'previewIndex', $feed['id']), array('style' => 'margin-right:3px;'));
@@ -166,8 +144,8 @@
     </ul>
   </td>
   <td class="short">
-    <div id = "<?php echo $currentType . '_' . $object['id'] . '_to_ids_placeholder'; ?>" class = "inline-field-placeholder"></div>
-    <div id = "<?php echo $currentType . '_' . $object['id'] . '_to_ids_solid'; ?>" class="inline-field-solid" ondblclick="activateField('<?php echo $currentType; ?>', '<?php echo $object['id']; ?>', 'to_ids', <?php echo $event['Event']['id'];?>);">
+    <div id="<?php echo $currentType . '_' . $object['id'] . '_to_ids_placeholder'; ?>" class="inline-field-placeholder"></div>
+    <div id="<?php echo $currentType . '_' . $object['id'] . '_to_ids_solid'; ?>" class="inline-field-solid">
       <?php
         if ($object['to_ids']) echo 'Yes';
         else echo 'No';
@@ -191,7 +169,7 @@
           echo $this->Form->create('Shadow_Attribute', array('id' => 'ShadowAttribute_' . $object['id'] . '_accept', 'url' => $baseurl . '/shadow_attributes/accept/' . $object['id'], 'style' => 'display:none;'));
           echo $this->Form->end();
         ?>
-          <span class="icon-ok icon-white useCursorPointer" title="<?php echo __('Accept Proposal');?>" role="button" tabindex="0" aria-label="<?php echo __('Accept proposal');?>" onClick="acceptObject('shadow_attributes', '<?php echo $object['id']; ?>', '<?php echo $event['Event']['id']; ?>');"></span>
+          <span class="fas fa-check white useCursorPointer" title="<?php echo __('Accept Proposal');?>" role="button" tabindex="0" aria-label="<?php echo __('Accept proposal');?>" onClick="acceptObject('shadow_attributes', '<?php echo $object['id']; ?>', '<?php echo $event['Event']['id']; ?>');"></span>
         <?php
         }
         if (($event['Orgc']['id'] == $me['org_id'] && $mayModify) || $isSiteAdmin || ($object['org_id'] == $me['org_id'])) {

app/View/Elements/Events/View/row_proposal_delete.ctp

@@ -30,16 +30,12 @@
   }
   $identifier = (empty($k)) ? '' : ' id="row_' . h($k) . '" tabindex="0"';
 ?>
-<tr id = "<?php echo $currentType . '_' . $object['id'] . '_tr'; ?>" class="<?php echo $tr_class; ?>" <?php echo $identifier; ?>>
+<tr id="<?php echo $currentType . '_' . $object['id'] . '_tr'; ?>" class="<?php echo $tr_class; ?>" <?php echo $identifier; ?>>
   <?php
     if ($mayModify):
   ?>
       <td style="width:10px;" data-position="<?php echo h($object['objectType']) . '_' . h($object['id']); ?>">
-        <?php if ($object['objectType'] == 0): ?>
-          <input id = "select_<?php echo $object['id']; ?>" class="select_attribute row_checkbox" type="checkbox" data-id="<?php echo $object['id'];?>" />
-        <?php else: ?>
-          <input id = "select_proposal_<?php echo $object['id']; ?>" class="select_proposal row_checkbox" type="checkbox" data-id="<?php echo $object['id'];?>" />
-        <?php endif; ?>
+          <input id="select_proposal_<?php echo $object['id']; ?>" class="select_proposal row_checkbox" type="checkbox" data-id="<?php echo $object['id'];?>" />
       </td>
   <?php
     endif;
@@ -53,7 +49,7 @@
   <td class="short context hidden">
       <?php echo $this->element('/Events/View/seen_field', array('object' => $object)); ?>
   </td>
-  <td style="font-weight:bold;text-align:left;">DELETE</td>
+  <td style="font-weight:bold;text-align:left;"><?= __('DELETE') ?></td>
   <?php
     if ($extended):
   ?>
@@ -77,7 +73,7 @@
           echo $this->Form->create('Shadow_Attribute', array('id' => 'ShadowAttribute_' . $object['id'] . '_accept', 'url' => $baseurl . '/shadow_attributes/accept/' . $object['id'], 'style' => 'display:none;'));
           echo $this->Form->end();
         ?>
-          <span class="icon-ok icon-white useCursorPointer" title="<?php echo __('Accept Proposal');?>" role="button" tabindex="0" aria-label="<?php echo __('Accept proposal');?>" onClick="acceptObject('shadow_attributes', '<?php echo $object['id']; ?>', '<?php echo $event['Event']['id']; ?>');"></span>
+          <span class="fas fa-check white useCursorPointer" title="<?php echo __('Accept Proposal');?>" role="button" tabindex="0" aria-label="<?php echo __('Accept proposal');?>" onClick="acceptObject('shadow_attributes', '<?php echo $object['id']; ?>', '<?php echo $event['Event']['id']; ?>');"></span>
         <?php
         }
         if (($event['Orgc']['id'] == $me['org_id'] && $mayModify) || $isSiteAdmin || ($object['org_id'] == $me['org_id'])) {

app/View/Elements/Events/View/seen_field.ctp

@@ -4,6 +4,4 @@
         <i style="display: block; text-align: center;" class="fas fa-arrow-down"></i>
         <div><?php echo $object['last_seen'] != null ? h($object['last_seen']) : '<span style="display: block; text-align:center;">_</span>'; ?></div>
     </div>
-<?php else: ?>
-    <div></div>
-<?php endif; ?>
+<?php endif ?>

app/View/Elements/Events/View/sighting_field.ctp

@@ -1,35 +1,53 @@
+<?php
+$objectId = intval($object['id']);
+$html = '';
+if (isset($sightingsData['data'][$objectId])) {
+    $objectSighting = $sightingsData['data'][$objectId];
+    foreach ($objectSighting as $type => $typeData) {
+        $name = $type !== 'expiration' ? Inflector::pluralize($type) : $type;
+        $html .= '<span class="blue bold">' . ucfirst(h($name)) . '</span><br>';
+        foreach ($typeData['orgs'] as $org => $orgData) {
+            $extra = $org === $me['Organisation']['name'] ? ' class="bold"' : "";
+            if ($type == 'expiration') {
+                $html .= '<span' . $extra . '>' . h($org) . '</span>: <span class="orange bold">' . date('Y-m-d H:i:s', $orgData['date']) . '</span><br>';
+            } else {
+                $html .= '<span' . $extra . '>' . h($org) . '</span>: <span class="' . ($type === 'sighting' ? 'green' : 'red') . ' bold">' . h($orgData['count']) . ' (' . date('Y-m-d H:i:s', $orgData['date']) . ')</span><br>';
+            }
+        }
+    }
+
+    $s = isset($objectSighting['sighting']['count']) ? intval($objectSighting['sighting']['count']) : 0;
+    $f = isset($objectSighting['false-positive']['count']) ? intval($objectSighting['false-positive']['count']) : 0;
+    $e = isset($objectSighting['expiration']['count']) ? intval($objectSighting['expiration']['count']) : 0;
+} else {
+    $s = $f = $e = 0;
+}
+?>
 <td class="shortish">
-  <span id="sightingForm_<?php echo h($object['id']);?>">
+  <span id="sightingForm_<?= $objectId ?>">
   <?php
-      echo $this->Form->create('Sighting', array('id' => 'Sighting_' . $object['id'], 'url' => $baseurl . '/sightings/add/' . $object['id'], 'style' => 'display:none;'));
-      echo $this->Form->input('type', array('label' => false, 'id' => 'Sighting_' . $object['id'] . '_type'));
+      echo $this->Form->create('Sighting', array('id' => 'Sighting_' . $objectId, 'url' => $baseurl . '/sightings/add/' . $objectId, 'style' => 'display:none;'));
+      echo $this->Form->input('type', array('label' => false, 'id' => 'Sighting_' . $objectId . '_type'));
       echo $this->Form->end();
   ?>
   </span>
   <?php
     if ($isAclSighting):
   ?>
-      <i class="icon-thumbs-up useCursorPointer" title="<?php echo __('Add sighting');?>" role="button" tabindex="0" aria-label="<?php echo __('Add sighting');?>" onmouseover="flexibleAddSighting(this, '0', '<?php echo h($object['id']); ?>', '<?php echo h($object['event_id']);?>', '<?php echo h($page); ?>', 'top');" onclick="addSighting('0', '<?php echo h($object['id']); ?>', '<?php echo h($object['event_id']);?>', '<?php echo h($page); ?>');">&nbsp;</i>
-      <i class="icon-thumbs-down useCursorPointer" title="<?php echo __('Mark as false-positive');?>" role="button" tabindex="0" aria-label="<?php echo __('Mark as false-positive');?>" onmouseover="flexibleAddSighting(this, '1', '<?php echo h($object['id']); ?>', '<?php echo h($object['event_id']);?>', '<?php echo h($page); ?>', 'bottom');" onclick="addSighting('1', '<?php echo h($object['id']); ?>', '<?php echo h($object['event_id']);?>', '<?php echo h($page); ?>');">&nbsp;</i>
-      <i class="icon-wrench useCursorPointer sightings_advanced_add" title="<?php echo __('Advanced sightings');?>" role="button" tabindex="0" aria-label="<?php echo __('Advanced sightings');?>" data-object-id="<?php echo h($object['id']); ?>" data-object-context="attribute">&nbsp;</i>
+      <i class="far fa-thumbs-up useCursorPointer" title="<?php echo __('Add sighting');?>" role="button" tabindex="0" aria-label="<?php echo __('Add sighting');?>" onmouseover="flexibleAddSighting(this, '0', '<?= $objectId ?>', '<?php echo h($object['event_id']);?>', 'top');" onclick="addSighting('0', '<?= $objectId ?>', '<?php echo h($object['event_id']);?>');">&nbsp;</i>
+      <i class="far fa-thumbs-down useCursorPointer" title="<?php echo __('Mark as false-positive');?>" role="button" tabindex="0" aria-label="<?php echo __('Mark as false-positive');?>" onmouseover="flexibleAddSighting(this, '1', '<?= $objectId ?>', '<?php echo h($object['event_id']);?>', 'bottom');" onclick="addSighting('1', '<?= $objectId ?>', '<?php echo h($object['event_id']);?>');">&nbsp;</i>
+      <i class="fas fa-wrench useCursorPointer sightings_advanced_add" title="<?php echo __('Advanced sightings');?>" role="button" tabindex="0" aria-label="<?php echo __('Advanced sightings');?>" data-object-id="<?= $objectId ?>" data-object-context="attribute">&nbsp;</i>
   <?php
     endif;
   ?>
-  <span id="sightingCount_<?php echo h($object['id']); ?>" class="bold sightingsCounter_<?php echo h($object['id']); ?>" data-placement="top" data-toggle="popover" data-trigger="hover" data-content="<?php echo isset($sightingsData['data'][$object['id']]['html']) ? $sightingsData['data'][$object['id']]['html'] : ''; ?>">
-    <?php
-      $s = (!empty($sightingsData['data'][$object['id']]['sighting']['count']) ? $sightingsData['data'][$object['id']]['sighting']['count'] : 0);
-      $f = (!empty($sightingsData['data'][$object['id']]['false-positive']['count']) ? $sightingsData['data'][$object['id']]['false-positive']['count'] : 0);
-      $e = (!empty($sightingsData['data'][$object['id']]['expiration']['count']) ? $sightingsData['data'][$object['id']]['expiration']['count'] : 0);
-    ?>
-  </span>
-  <span id="ownSightingCount_<?php echo h($object['id']); ?>" class="bold sightingsCounter_<?php echo h($object['id']); ?>" data-placement="top" data-toggle="popover" data-trigger="hover" data-content="<?php echo isset($sightingsData['data'][$object['id']]['html']) ? $sightingsData['data'][$object['id']]['html'] : ''; ?>">
-    <?php echo '(<span class="green">' . h($s) . '</span>/<span class="red">' . h($f) . '</span>/<span class="orange">' . h($e) . '</span>)'; ?>
+  <span id="sightingCount_<?php echo $objectId; ?>" class="bold" data-placement="top" data-toggle="popover" data-trigger="hover" data-content="<?= h($html) ?>">
+    (<span class="green"><?= $s ?></span>/<span class="red"><?= $f ?></span>/<span class="orange"><?= $e ?></span>)
   </span>
 </td>
 <td class="short">
   <?php
-    if (!empty($sightingsData['csv'][$object['id']])) {
-      echo $this->element('sparkline', array('scope' => 'object', 'id' => $object['id'], 'csv' => $sightingsData['csv'][$object['id']]));
+    if (!empty($sightingsData['csv'][$objectId])) {
+      echo $this->element('sparkline', array('scope' => 'object', 'id' => $objectId, 'csv' => $sightingsData['csv'][$objectId]));
     }
   ?>
 </td>

app/View/Elements/Events/View/value_field.ctp

@@ -1,5 +1,26 @@
 <?php
-$sigDisplay = $object['value'];
+$truncateLongText = function ($text, $maxLength = 500, $maxLines = 10) {
+    $truncated = false;
+    if (mb_strlen($text) > $maxLength) {
+        $text = mb_substr($text, 0, 500);
+        $truncated = true;
+    }
+
+    if (substr_count($text, "\n") > $maxLines) {
+        $lines = explode("\n", $text);
+        $text = implode("\n", array_slice($lines, 0, $maxLines));
+        $truncated = true;
+    }
+
+    if ($truncated) {
+        return $text;
+    }
+    return null;
+};
+
+if (!isset($linkClass)) {
+    $linkClass = null;
+}
 
 switch ($object['type']) {
     case 'attachment':
@@ -24,32 +45,59 @@ switch ($object['type']) {
                 $filename = $filenameHash[0];
             }
 
-            $controller = isset($object['objectType']) && $object['objectType'] === 'proposal' ? 'shadow_attributes' : 'attributes';
-            $url = array('controller' => $controller, 'action' => 'download', $object['id']);
-            echo $this->Html->link($filename, $url, array('class' => $linkClass));
+            if (isset($object['objectType'])) {
+                if (array_key_exists('infected', $object) && $object['infected'] !== false) { // it is not possible to use isset
+                    if ($object['infected'] === null) {
+                        $confirm = __('This file was not checked by AV scan. Do you really want to download it?');
+                    } else {
+                        $confirm = __('According to AV scan, this file contains %s malware. Do you really want to download it?', $object['infected']);
+                    }
+                } else {
+                    $confirm = null;
+                }
+
+                $controller = $object['objectType'] === 'proposal' ? 'shadow_attributes' : 'attributes';
+                $url = array('controller' => $controller, 'action' => 'download', $object['id']);
+                echo $this->Html->link($filename, $url, array('class' => $linkClass), $confirm);
+            } else {
+                echo $filename;
+            }
             if (isset($filenameHash[1])) {
-                echo '<br />' . $filenameHash[1];
+                echo '<br>' . $filenameHash[1];
+            }
+            if (isset($object['infected']) && $object['infected'] !== false) {
+                echo ' <i class="fas fa-virus" title="' . __('This file contains malware %s', $object['infected'])  . '"></i>';
             }
         }
         break;
 
     case 'vulnerability':
         $cveUrl = Configure::read('MISP.cveurl') ?: 'https://cve.circl.lu/cve/';
-        echo $this->Html->link($sigDisplay, $cveUrl . $sigDisplay, array('target' => '_blank', 'class' => $linkClass));
+        echo $this->Html->link($object['value'], $cveUrl . $object['value'], [
+            'target' => '_blank',
+            'class' => $linkClass,
+            'rel' => 'noreferrer noopener',
+            'title' => __('Show more information about this vulnerability in external tool'),
+        ]);
         break;
 
     case 'weakness':
         $cweUrl = Configure::read('MISP.cweurl') ?: 'https://cve.circl.lu/cwe/';
-        $link = $cweUrl . explode("-", $sigDisplay)[1];
-        echo $this->Html->link($sigDisplay, $link, array('target' => '_blank', 'class' => $linkClass));
+        $link = $cweUrl . explode("-", $object['value'])[1];
+        echo $this->Html->link($object['value'], $link, [
+            'target' => '_blank',
+            'class' => $linkClass,
+            'rel' => 'noreferrer noopener',
+            'title' => __('Show more information about this weakness in external tool'),
+        ]);
         break;
 
     case 'link':
-        echo $this->Html->link($sigDisplay, $sigDisplay, array('class' => $linkClass));
+        echo $this->Html->link($object['value'], $object['value'], ['class' => $linkClass, 'rel' => 'noreferrer noopener']);
         break;
 
     case 'cortex':
-        echo '<div class="cortex-json" data-cortex-json="' . h($object['value']) . '">' . __('Cortex object') . '</div>';
+        echo '<span data-full="' . h($object['value']) . '" data-full-type="cortex"><a href="#">' . __('Cortex object') . '</a></span>';
         break;
 
     case 'text':
@@ -57,48 +105,70 @@ switch ($object['type']) {
             $url = array('controller' => 'events', 'action' => 'view', $object['value']);
             echo $this->Html->link($object['value'], $url, array('class' => $linkClass));
         } else {
-            $sigDisplay = str_replace("\r", '', h($sigDisplay));
-            $sigDisplay = str_replace(" ", '&nbsp;', $sigDisplay);
-            echo $sigDisplay;
+            $value = str_replace("\r", '', $object['value']);
+            $truncated = $truncateLongText($value);
+            if ($truncated) {
+                echo '<span style="white-space: pre-wrap;" data-full="' . h($object['value']) .'" data-full-type="text">' .
+                    str_replace(" ", '&nbsp;', h(rtrim($truncated)));
+                echo ' <b>&hellip;</b><br><a href="#">' . __('Show all') . '</a></span>';
+            } else {
+                echo '<span style="white-space: pre-wrap;">' . str_replace(" ", '&nbsp;', h($value)) . '</span>';
+            }
         }
         break;
 
     case 'hex':
-        $sigDisplay = str_replace("\r", '', $sigDisplay);
-        echo '<span class="hex-value" title="' . __('Hexadecimal representation') . '">' . nl2br(h($sigDisplay)) . '</span>&nbsp;';
-        echo '<span role="button" tabindex="0" aria-label="' . __('Switch to binary representation') . '" class="icon-repeat hex-value-convert useCursorPointer" title="' . __('Switch to binary representation') . '"></span>';
+        echo '<span class="hex-value" title="' . __('Hexadecimal representation') . '">' . h($object['value']) . '</span>&nbsp;';
+        echo '<span role="button" tabindex="0" aria-label="' . __('Switch to binary representation') . '" class="fas fa-redo hex-value-convert useCursorPointer" title="' . __('Switch to binary representation') . '"></span>';
+        break;
+
+    case 'ip-dst|port':
+    case 'ip-src|port':
+    case 'hostname|port':
+        $valuePieces = explode('|', $object['value']);
+        if (substr_count($valuePieces[0], ':') >= 2) {
+            echo '[' . h($valuePieces[0]) . ']:' . h($valuePieces[1]); // IPv6 style
+        } else {
+            echo h($valuePieces[0]) . ':' . h($valuePieces[1]);
+        }
         break;
 
     /** @noinspection PhpMissingBreakStatementInspection */
     case 'domain':
-        if (strpos($sigDisplay, 'xn--') !== false && function_exists('idn_to_utf8')) {
-            echo '<span title="' . h(idn_to_utf8($sigDisplay)) . '">' . h($sigDisplay) . '</span>';
+        if (strpos($object['value'], 'xn--') !== false && function_exists('idn_to_utf8')) {
+            echo '<span title="' . h(idn_to_utf8($object['value'])) . '">' . h($object['value']) . '</span>';
             break;
         }
 
     default:
         if (strpos($object['type'], '|') !== false) {
-            if (in_array($object['type'], array('ip-dst|port', 'ip-src|port'))) {
-                if (substr_count($object['value'], ':') >= 2) {
-                    $object['value'] = '[' . $object['value']; // prepend `[` for a nicer display
-                    $separator = ']:';
-                } else {
-                    $separator = ':';
-                }
-            } else {
-                $separator = '<br />';
-            }
             $valuePieces = explode('|', $object['value']);
             foreach ($valuePieces as $k => $v) {
                 $valuePieces[$k] = h($v);
             }
-            echo implode($separator, $valuePieces);
+            echo implode('<br>', $valuePieces);
         } else {
-            $sigDisplay = str_replace("\r", '', $sigDisplay);
-            echo h($sigDisplay);
+            $value = str_replace("\r", '', $object['value']);
+            $truncated = $truncateLongText($value);
+            if ($truncated) {
+                $rawTypes = ['email-header', 'yara', 'pgp-private-key', 'pgp-public-key', 'url'];
+                $dataFullType = in_array($object['type'], $rawTypes) ? 'raw' : 'text';
+                echo '<span style="white-space: pre-wrap;" data-full="' . h($value) .'" data-full-type="' . $dataFullType .'">' . h($truncated) .
+                    ' <b>&hellip;</b><br><a href="#">' . __('Show all') . '</a></span>';
+            } else {
+                echo '<span style="white-space: pre-wrap;">' . h($value) . '</span>';
+            }
         }
 }
 
 if (isset($object['validationIssue'])) {
-    echo ' <span class="fa fa-exclamation-triangle" title="' . __('Warning, this doesn\'t seem to be a legitimate ') . strtoupper(h($object['type'])) . __(' value') . '">&nbsp;</span>';
+    echo ' <span class="fa fa-exclamation-triangle" title="' . __('Warning, this doesn\'t seem to be a legitimate %s value', strtoupper(h($object['type']))) . '">&nbsp;</span>';
+}
+
+if (isset($object['warnings'])) {
+    $temp = '';
+    foreach ($object['warnings'] as $warning) {
+        $temp .= '<span class="bold">' . h($warning['match']) . ':</span> <span class="red">' . h($warning['warninglist_name']) . '</span><br>';
+    }
+    echo ' <span aria-label="' . __('warning') . '" role="img" tabindex="0" class="fa fa-exclamation-triangle" data-placement="right" data-toggle="popover" data-content="' . h($temp) . '" data-trigger="hover">&nbsp;</span>';
 }

app/View/Elements/Events/eventIndexTable.ctp

@@ -31,7 +31,7 @@
             $date = time();
             $day = 86400;
         ?>
-        <th><?php echo $this->Paginator->sort('id', null, array('direction' => 'desc'));?></th>
+        <th><?php echo $this->Paginator->sort('id', __('ID'), array('direction' => 'desc'));?></th>
         <th><?php echo __('Clusters');?></th>
         <?php if (Configure::read('MISP.tagging')): ?>
             <th class="filter"><?php echo __('Tags');?></th>
@@ -50,7 +50,7 @@
             <th title="<?php echo __('Post Count');?>"><?php echo __('#Posts');?></th>
         <?php endif; ?>
         <?php if ($isSiteAdmin): ?>
-        <th><?php echo $this->Paginator->sort('user_id', __('Email'));?></th>
+        <th><?php echo $this->Paginator->sort('user_id', __('Creator user'));?></th>
         <?php endif; ?>
         <th class="filter"><?php echo $this->Paginator->sort('date', null, array('direction' => 'desc'));?></th>
         <th class="filter"><?php echo $this->Paginator->sort('info');?></th>
@@ -61,7 +61,7 @@
 
     </tr>
     <?php foreach ($events as $event): ?>
-    <tr <?php if ($event['Event']['distribution'] == 0) echo 'class = "privateRed"'?> id="event_<?php echo h($event['Event']['id']);?>">
+    <tr <?php if ($event['Event']['distribution'] == 0) echo 'class="privateRed"'?> id="event_<?php echo h($event['Event']['id']);?>">
             <?php
                 if ($isSiteAdmin || ($event['Event']['orgc_id'] == $me['org_id'])):
             ?>
@@ -76,16 +76,9 @@
                 endif;
             ?>
         <td class="short" ondblclick="document.location.href ='<?php echo $baseurl."/events/view/".$event['Event']['id'];?>'">
-            <?php
-            if ($event['Event']['published'] == 1) {
-            ?>
-                <a href="<?php echo $baseurl."/events/view/".$event['Event']['id'] ?>" title = "<?php echo __('View');?>" aria-label = "<?php echo __('View');?>"><i class="black fa fa-check"></i></a>
-            <?php
-            } else {
-            ?>
-                <a href="<?php echo $baseurl."/events/view/".$event['Event']['id'] ?>" title = "<?php echo __('View');?>" aria-label = "<?php echo __('View');?>"><i class="black fa fa-times"></i></a>
-            <?php
-            }?>&nbsp;
+            <a href="<?= "$baseurl/events/view/{$event['Event']['id']}" ?>" title="<?= __('View') ?>" aria-label="<?= __('View') ?>">
+                <i class="black fa <?= $event['Event']['published'] == 1 ? 'fa-check' : 'fa-times' ?>"></i>
+            </a>
         </td>
         <?php if (Configure::read('MISP.showorg') || $isAdmin): ?>
             <td class="short" ondblclick="document.location.href ='<?php echo $baseurl . "/events/index/searchorg:" . $event['Orgc']['id'];?>'">
@@ -152,7 +145,7 @@
             <?php echo $event['Event']['attribute_count']; ?>&nbsp;
         </td>
         <?php if (Configure::read('MISP.showCorrelationsOnIndex')):?>
-            <td class = "bold" style="width:30px;">
+            <td class="bold" style="width:30px;">
                 <?php if (!empty($event['Event']['correlation_count'])): ?>
                     <a href="<?php echo $baseurl."/events/view/" . h($event['Event']['id']) . '/correlation:1';?>" title="<?php echo h($event['Event']['correlation_count']) . __(' correlation(s). Show filtered event with correlation only.');?>">
                         <?php echo h($event['Event']['correlation_count']); ?>&nbsp;
@@ -161,23 +154,21 @@
             </td>
         <?php endif; ?>
         <?php if (Configure::read('MISP.showSightingsCountOnIndex')):?>
-            <td class = "bold" style="width:30px;">
+            <td class="bold" style="width:30px;">
                 <?php if (!empty($event['Event']['sightings_count'])): ?>
                     <a href="<?php echo $baseurl."/events/view/" . h($event['Event']['id']) . '/sighting:1';?>" title="<?php echo (!empty($event['Event']['sightings_count']) ? h($event['Event']['sightings_count']) : '0') . ' sighting(s). Show filtered event with sighting(s) only.';?>">
                         <?php echo h($event['Event']['sightings_count']); ?>&nbsp;
                     </a>
                 <?php endif; ?>
             </td>
-
-
         <?php endif; ?>
         <?php if (Configure::read('MISP.showProposalsOnIndex')): ?>
-            <td class = "bold" style="width:30px;" ondblclick="location.href ='<?php echo $baseurl."/events/view/".$event['Event']['id'];?>'" title="<?php echo (!empty($event['Event']['proposals_count']) ? h($event['Event']['proposals_count']) : '0') . __(' proposal(s)');?>">
+            <td class="bold" style="width:30px;" ondblclick="location.href ='<?php echo $baseurl."/events/view/".$event['Event']['id'];?>'" title="<?php echo (!empty($event['Event']['proposals_count']) ? h($event['Event']['proposals_count']) : '0') . __(' proposal(s)');?>">
                 <?php echo !empty($event['Event']['proposals_count']) ? h($event['Event']['proposals_count']) : ''; ?>&nbsp;
             </td>
         <?php endif;?>
         <?php if (Configure::read('MISP.showDiscussionsCountOnIndex')): ?>
-            <td class = "bold" style="width:30px;" ondblclick="location.href ='<?php echo $baseurl."/events/view/".$event['Event']['id'];?>'" title="<?php echo (!empty($event['Event']['proposals_count']) ? h($event['Event']['proposals_count']) : '0') . __(' proposal(s)');?>">
+            <td class="bold" style="width:30px;" ondblclick="location.href ='<?php echo $baseurl."/events/view/".$event['Event']['id'];?>'" title="<?php echo (!empty($event['Event']['proposals_count']) ? h($event['Event']['proposals_count']) : '0') . __(' proposal(s)');?>">
                 <?php
                     if (!empty($event['Event']['post_count'])) {
                         $post_count = h($event['Event']['post_count']);
@@ -202,7 +193,7 @@
         <td ondblclick="location.href ='<?php echo $baseurl."/events/view/".$event['Event']['id'];?>'">
             <?php echo nl2br(h($event['Event']['info'])); ?>&nbsp;
         </td>
-        <td class="short <?php if ($event['Event']['distribution'] == 0) echo 'privateRedText';?>" ondblclick="location.href ='<?php echo $baseurl; ?>/events/view/<?php echo $event['Event']['id'];?>'" title = "<?php echo $event['Event']['distribution'] != 3 ? $distributionLevels[$event['Event']['distribution']] : __('All');?>">
+        <td class="short <?php if ($event['Event']['distribution'] == 0) echo 'privateRedText';?>" ondblclick="location.href ='<?php echo $baseurl; ?>/events/view/<?php echo $event['Event']['id'];?>'" title="<?php echo $event['Event']['distribution'] != 3 ? $distributionLevels[$event['Event']['distribution']] : __('All');?>">
             <?php if ($event['Event']['distribution'] == 4):?>
                 <a href="<?php echo $baseurl;?>/sharingGroups/view/<?php echo h($event['SharingGroup']['id']); ?>"><?php echo h($event['SharingGroup']['name']);?></a>
             <?php else:
@@ -228,24 +219,23 @@
 
                 if ($isSiteAdmin || ($isAclModify && $event['Event']['user_id'] == $me['id']) || ($isAclModifyOrg && $event['Event']['orgc_id'] == $me['org_id'])):
             ?>
-                    <a href='<?php echo $baseurl."/events/edit/".$event['Event']['id'];?>' title = "<?php echo __('Edit');?>" aria-label = "<?php echo __('Edit');?>"><i class="black fa fa-edit"></i></a>
+                    <a href="<?php echo $baseurl."/events/edit/".$event['Event']['id'];?>" title="<?php echo __('Edit');?>" aria-label="<?php echo __('Edit');?>"><i class="black fa fa-edit"></i></a>
             <?php
 
                     echo sprintf('<a class="useCursorPointer fa fa-trash" title="%s" aria-label="%s" onclick="deleteEvent(%s)"></a>', __('Delete'), __('Delete'), h($event['Event']['id']));
                 endif;
             ?>
-            <a href='<?php echo $baseurl."/events/view/".$event['Event']['id'];?>' title = "<?php echo __('View');?>" aria-label = "<?php echo __('View');?>"><i class="fa black fa-eye"></i></a>
+            <a href="<?php echo $baseurl."/events/view/".$event['Event']['id'];?>" title="<?php echo __('View');?>" aria-label="<?php echo __('View');?>"><i class="fa black fa-eye"></i></a>
         </td>
     </tr>
     <?php endforeach; ?>
 </table>
 <script type="text/javascript">
     var lastSelected = false;
-    $(document).ready(function() {
+    $(function() {
         $('.select').on('change', function() {
             listCheckboxesChecked();
-        });
-        $('.select').click(function(e) {
+        }).click(function(e) {
             if ($(this).is(':checked')) {
                 if (e.shiftKey) {
                     selectAllInbetween(lastSelected, this.id);
@@ -263,7 +253,7 @@
     });
 
     function deleteEvent(id) {
-        var message = "<?= __('Are you sure you want to delete # ') ?>" + id + "?"
+        var message = "<?= __('Are you sure you want to delete #') ?>" + id + "?"
         var url = '<?= $baseurl ?>/events/delete/' + id
         if (confirm(message)) {
             fetchFormDataAjax(url, function(formData) {

app/View/Elements/Feeds/View/row_attribute.ctp

@@ -51,17 +51,13 @@
           ?>
         </span>
         <?php
-          if (isset($object['warnings'])) {
+        if (isset($object['warnings'])) {
             $temp = '';
-            $components = array(1 => 0, 2 => 1);
-            $valueParts = explode('|', $object['value']);
-            foreach ($components as $component => $valuePart) {
-              if (isset($object['warnings'][$component]) && isset($valueParts[$valuePart])) {
-                foreach ($object['warnings'][$component] as $warning) $temp .= '<span class=\'bold\'>' . h($valueParts[$valuePart]) . '</span>: <span class=\'red\'>' . h($warning) . '</span><br />';
-              }
+            foreach ($object['warnings'] as $warning) {
+                $temp .= '<span class="bold">' . h($warning['match']) . ':</span> <span class="red">' . h($warning['warninglist_name']) . '</span><br>';
             }
-            echo ' <span aria-label="' . __('warning') . '" role="img" tabindex="0" class="icon-warning-sign" data-placement="right" data-toggle="popover" data-content="' . h($temp) . '" data-trigger="hover">&nbsp;</span>';
-          }
+            echo ' <span aria-label="' . __('warning') . '" role="img" tabindex="0" class="fa fa-exclamation-triangle" data-placement="right" data-toggle="popover" data-content="' . h($temp) . '" data-trigger="hover" data-placement="right">&nbsp;</span>';
+        }
         ?>
       </div>
     </td>
@@ -89,5 +85,4 @@
         <?php echo $object['to_ids'] ? __('Yes') : __('No'); ?>
       </div>
     </td>
-  </td>
 </tr>

app/View/Elements/Feeds/eventattribute.ctp

@@ -117,32 +117,6 @@
             screenshotPopup($(this).attr('src'), $(this).attr('title'));
         });
     });
-    $('.hex-value-convert').click(function() {
-        var val = $(this).parent().children(':first-child').text();
-        if ($(this).parent().children(':first-child').attr('data-original-title') == 'Hexadecimal representation') {
-            var bin = [];
-            var temp;
-            val.split('').forEach(function(entry) {
-                temp = parseInt(entry, 16).toString(2);
-                bin.push(Array(5 - (temp.length)).join('0') + temp);
-            });
-            bin = bin.join(' ');
-            $(this).parent().children(':first-child').text(bin);
-            $(this).parent().children(':first-child').attr('data-original-title', __('Binary representation'));
-            $(this).parent().children(':nth-child(2)').attr('data-original-title', __('Switch to hexadecimal representation'));
-            $(this).parent().children(':nth-child(2)').attr('aria-label', __('Switch to hexadecimal representation'));
-        } else {
-            val = val.split(' ');
-            hex = '';
-            val.forEach(function(entry) {
-                hex += parseInt(entry , 2).toString(16).toUpperCase();
-            });
-            $(this).parent().children(':first-child').text(hex);
-            $(this).parent().children(':first-child').attr('data-original-title', __('Hexadecimal representation'));
-            $(this).parent().children(':nth-child(2)').attr('data-original-title', __('Switch to binary representation'));
-            $(this).parent().children(':nth-child(2)').attr('aria-label', __('Switch to binary representation'));
-        }
-    });
 </script>
 <?php
     echo $this->Js->writeBuffer();

app/View/Elements/Objects/object_add_attributes.ctp

@@ -37,7 +37,7 @@
       echo $this->Form->input('Attribute.' . $k . '.type', $formSettings);
       echo '<span class="bold">' . Inflector::humanize(h($element['object_relation'])) . '</span>';
       if (!empty($template['ObjectTemplate']['requirements']['required']) && in_array($element['object_relation'], $template['ObjectTemplate']['requirements']['required'])) {
-        echo '<span class="bold red">' . '(*)' . '</span>';
+        echo '<span class="red" style="vertical-align: super;font-size: 8px;margin-left: 2px;" title="' . __('Required') . '"><i class="fas fa-asterisk"></i></span>';
       }
       echo ' :: ' . h($element['type']) . '';
     ?>

app/View/Elements/Servers/View/row_attribute.ctp

@@ -58,17 +58,13 @@
           ?>
         </span>
         <?php
-          if (isset($object['warnings'])) {
+        if (isset($object['warnings'])) {
             $temp = '';
-            $components = array(1 => 0, 2 => 1);
-            $valueParts = explode('|', $object['value']);
-            foreach ($components as $component => $valuePart) {
-              if (isset($object['warnings'][$component]) && isset($valueParts[$valuePart])) {
-                foreach ($object['warnings'][$component] as $warning) $temp .= '<span class=\'bold\'>' . h($valueParts[$valuePart]) . '</span>: <span class=\'red\'>' . h($warning) . '</span><br />';
-              }
+            foreach ($object['warnings'] as $warning) {
+                $temp .= '<span class="bold">' . h($warning['match']) . ':</span> <span class="red">' . h($warning['warninglist_name']) . '</span><br>';
             }
-            echo ' <span aria-label="' . __('warning') . '" role="img" tabindex="0" class="icon-warning-sign" data-placement="right" data-toggle="popover" data-content="' . h($temp) . '" data-trigger="hover">&nbsp;</span>';
-          }
+            echo ' <span aria-label="' . __('warning') . '" role="img" tabindex="0" class="fa fa-exclamation-triangle" data-placement="right" data-toggle="popover" data-content="' . h($temp) . '" data-trigger="hover" data-placement="right">&nbsp;</span>';
+        }
         ?>
       </div>
     </td>

app/View/Elements/Servers/eventattribute.ctp

@@ -117,32 +117,6 @@
             screenshotPopup($(this).attr('src'), $(this).attr('title'));
         });
     });
-    $('.hex-value-convert').click(function() {
-        var val = $(this).parent().children(':first-child').text();
-        if ($(this).parent().children(':first-child').attr('data-original-title') == 'Hexadecimal representation') {
-            var bin = [];
-            var temp;
-            val.split('').forEach(function(entry) {
-                temp = parseInt(entry, 16).toString(2);
-                bin.push(Array(5 - (temp.length)).join('0') + temp);
-            });
-            bin = bin.join(' ');
-            $(this).parent().children(':first-child').text(bin);
-            $(this).parent().children(':first-child').attr('data-original-title', __('Binary representation'));
-            $(this).parent().children(':nth-child(2)').attr('data-original-title', __('Switch to hexadecimal representation'));
-            $(this).parent().children(':nth-child(2)').attr('aria-label', __('Switch to hexadecimal representation'));
-        } else {
-            val = val.split(' ');
-            hex = '';
-            val.forEach(function(entry) {
-                hex += parseInt(entry , 2).toString(16).toUpperCase();
-            });
-            $(this).parent().children(':first-child').text(hex);
-            $(this).parent().children(':first-child').attr('data-original-title', __('Hexadecimal representation'));
-            $(this).parent().children(':nth-child(2)').attr('data-original-title', __('Switch to binary representation'));
-            $(this).parent().children(':nth-child(2)').attr('aria-label', __('Switch to binary representation'));
-        }
-    });
 </script>
 <?php
     echo $this->Js->writeBuffer();

app/View/Elements/Users/userIndexTable.ctp

@@ -1,6 +1,6 @@
 <table class="table table-striped table-hover table-condensed">
     <tr>
-        <th><?php echo $this->Paginator->sort('id');?></th>
+        <th><?php echo $this->Paginator->sort('id', __('ID'));?></th>
         <th><?php echo $this->Paginator->sort('org_ci', __('Org'));?></th>
         <th><?php echo $this->Paginator->sort('role_id', __('Role'));?></th>
         <th><?php echo $this->Paginator->sort('email');?></th>
@@ -40,8 +40,8 @@
                 <td ondblclick="document.location ='<?php echo $this->Html->url(array('admin' => true, 'action' => 'view', $user['User']['id']), true);?>';">
                     <?php echo h($user['User']['email']); ?>&nbsp;
                 </td>
-                <td ondblclick="document.location ='<?php echo $this->Html->url(array('admin' => true, 'action' => 'view', $user['User']['id']), true);?>';" class="quickSelect <?php echo $user['Role']['perm_auth'] ? 'bold' : 'grey'; ?>">
-                    <?php echo h($user['User']['authkey']); ?>&nbsp;
+                <td ondblclick="document.location ='<?php echo $this->Html->url(array('admin' => true, 'action' => 'view', $user['User']['id']), true);?>';" class="<?php echo $user['Role']['perm_auth'] ? 'bold' : 'grey'; ?>">
+                    <span class="privacy-value quickSelect" data-hidden-value="<?= h($user['User']['authkey']) ?>">****************************************</span>&nbsp;<i class="privacy-toggle fas fa-eye useCursorPointer" title="<?= __('Reveal hidden value') ?>"></i>
                 </td>
                 <td class="short" ondblclick="document.location ='<?php echo $this->Html->url(array('admin' => true, 'action' => 'view', $user['User']['id']), true);?>';">
                     <?php echo $user['User']['autoalert']? __('Yes') : __('No'); ?>

app/View/Elements/ajaxTags.ctp

@@ -67,7 +67,7 @@
             }
         }
         $aText = h($aText);
-        $span_scope = sprintf(
+        $span_scope = !empty($hide_global_scope) ? '' : sprintf(
             '<span class="%s" title="%s" aria-label="%s"><i class="fas fa-%s"></i></span>',
             'black-white tag',
             !empty($tag['local']) ? __('Local tag') : __('Global tag'),
@@ -113,16 +113,15 @@
                 )
             );
         }
-        $tagData .= '<span class="tag-container nowrap  ">' . $span_scope . $span_tag . $span_delete . '</span> ';
+        $tagData .= '<span class="tag-container nowrap">' . $span_scope . $span_tag . $span_delete . '</span> ';
     }
     $buttonData = array();
     if ($full) {
         $buttonData[] = sprintf(
-            '<button id="%s" title="%s" role ="button" tabindex="0" aria-label="%s" class="%s" style="%s" onClick="%s">%s</button>',
-            'addTagButton',
+            '<button title="%s" role="button" tabindex="0" aria-label="%s" class="%s" style="%s" onClick="%s">%s</button>',
             __('Add a tag'),
             __('Add a tag'),
-            'btn btn-inverse noPrint',
+            'addTagButton btn btn-inverse noPrint',
             'line-height:10px; padding: 2px;',
             sprintf(
                 "popoverPopup(this, '%s%s', '%s', '%s');",
@@ -136,11 +135,10 @@
     }
     if ($host_org_editor || $full) {
         $buttonData[] = sprintf(
-            '<button id="%s" title="%s" role ="button" tabindex="0" aria-label="%s" class="%s" style="%s" onClick="%s">%s</button>',
-            'addLocalTagButton',
+            '<button title="%s" role="button" tabindex="0" aria-label="%s" class="%s" style="%s" onClick="%s">%s</button>',
             __('Add a local tag'),
             __('Add a local tag'),
-            'btn btn-inverse noPrint',
+            'addLocalTagButton btn btn-inverse noPrint',
             'line-height:10px; padding: 2px;',
             sprintf(
                 "popoverPopup(this, '%s%s', '%s', '%s')",

app/View/Elements/dashboard/Widgets/Button.ctp

@@ -0,0 +1,27 @@
+<div>
+        <?php
+        /*
+        * A simple button to add a link to a specific section
+        *
+        * Expected input:
+        * { url: <relative url>, text: <text to be displayed on the button>}
+        *
+        * Example:
+        * {url: "/events/index", text: "To the list of events"}
+        *
+        */
+        echo '<a href="'.$baseurl.h($data['url']).'">';
+        echo '<button class="btn btn-primary widget-button">';
+        echo h($data['text']);
+        echo '</button></a>';
+        ?>
+</div>
+
+<style widget-scoped>
+    .widget-button {
+        height: 100%;
+        width: 100%;
+        text-align: center;
+        font-size: large;
+    }
+</style>

app/View/Elements/eventattribute.ctp

@@ -13,29 +13,11 @@
     $all = false;
     if (isset($this->params->params['paging']['Event']['page'])) {
         if ($this->params->params['paging']['Event']['page'] == 0) $all = true;
-        $page = $this->params->params['paging']['Event']['page'];
+        $page = $this->params->params['paging']['Event']['page']; // $page is probably unused
     } else {
-        $page = 0;
+        $page = 0; // $page is probably unused
     }
     $fieldCount = 11;
-    if (!empty($event['Sighting'])) {
-        foreach ($sightingsData['data'] as $aid => $data) {
-            $sightingsData['data'][$aid]['html'] = '';
-            foreach ($data as $type => $typeData) {
-                $name = (($type != 'expiration') ? Inflector::pluralize($type) : $type);
-                $sightingsData['data'][$aid]['html'] .= '<span class=\'blue bold\'>' . ucfirst(h($name)) . '</span><br />';
-                foreach ($typeData['orgs'] as $org => $orgData) {
-                    $extra = (($org == $me['Organisation']['name']) ? " class=  'bold'" : "");
-                    if ($type == 'expiration') {
-                        $sightingsData['data'][$aid]['html'] .= '<span ' . $extra . '>' . h($org) . '</span>: <span class=\'orange bold\'>' . date('Y-m-d H:i:s', $orgData['date']) . '</span><br />';
-                    } else {
-                        $sightingsData['data'][$aid]['html'] .= '<span ' . $extra . '>' . h($org) . '</span>: <span class=\'' . (($type == 'sighting') ? 'green' : 'red') . ' bold\'>' . h($orgData['count']) . ' (' . date('Y-m-d H:i:s', $orgData['date']) . ')</span><br />';
-                    }
-                }
-                $sightingsData['data'][$aid]['html'] .= '<br />';
-            }
-        }
-    }
     $filtered = false;
     if(isset($passedArgsArray)){
         if (count($passedArgsArray) > 0) {
@@ -43,18 +25,22 @@
         }
     }
 ?>
+    <br />
     <div class="pagination">
         <ul>
         <?php
             $params = $this->request->named;
+            if (isset($params['focus'])) {
+                $focus = $params['focus'];
+            }
             unset($params['focus']);
             $url = array_merge(array('controller' => 'events', 'action' => 'viewEventAttributes', $event['Event']['id']), $params);
             $this->Paginator->options(array(
                 'url' => $url,
                 'update' => '#attributes_div',
                 'evalScripts' => true,
-                'before' => '$(".progress").show()',
-                'complete' => '$(".progress").hide()',
+                'before' => '$(".loading").show()',
+                'complete' => '$(".loading").hide()',
             ));
             echo $this->Paginator->prev('&laquo; ' . __('previous'), array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'prev disabled', 'escape' => false, 'disabledTag' => 'span'));
             echo $this->Paginator->numbers(array('modulus' => 60, 'separator' => '', 'tag' => 'li', 'currentClass' => 'red', 'currentTag' => 'span'));
@@ -71,7 +57,6 @@
         </li>
         </ul>
     </div>
-<br />
 <div id="edit_object_div">
     <?php
         $deleteSelectedUrl = $baseurl . '/attributes/deleteSelected/' . $event['Event']['id'];
@@ -117,7 +102,7 @@
             'target' => $target,
             'attributeFilter' => $attributeFilter,
             'urlHere' => $urlHere,
-            'filtered' =>$filtered,
+            'filtered' => $filtered,
             'mayModify' => $mayModify,
             'possibleAction' => $possibleAction
         ));
@@ -132,17 +117,13 @@
             <?php
                 endif;
             ?>
-            <th class="context hidden"><?php echo $this->Paginator->sort('id');?></th>
+            <th class="context hidden"><?php echo $this->Paginator->sort('id', 'ID');?></th>
             <th class="context hidden">UUID</th>
             <th class="context hidden"><?php echo __('First seen') ?> <i class="fas fa-arrow-right"></i> <?php echo __('Last seen') ?></th>
             <th><?php echo $this->Paginator->sort('timestamp', __('Date'), array('direction' => 'desc'));?></th>
-            <?php
-                if ($extended):
-            ?>
-                    <th class="event_id"><?php echo $this->Paginator->sort('event_id', __('Event'));?></th>
-            <?php
-                endif;
-            ?>
+            <?php if ($extended): ?>
+                <th class="event_id"><?php echo $this->Paginator->sort('event_id', __('Event'));?></th>
+            <?php endif; ?>
             <th><?php echo $this->Paginator->sort('Org.name', __('Org')); ?>
             <th><?php echo $this->Paginator->sort('category');?></th>
             <th><?php echo $this->Paginator->sort('type');?></th>
@@ -183,32 +164,20 @@
             <th class="actions"><?php echo __('Actions');?></th>
         </tr>
         <?php
-            $elements = array(
-                0 => 'attribute',
-                1 => 'proposal',
-                2 => 'proposal_delete',
-                3 => 'object'
-            );
-            $focusedRow = false;
             foreach ($event['objects'] as $k => $object) {
-                $insertBlank = false;
                 echo $this->element('/Events/View/row_' . $object['objectType'], array(
                     'object' => $object,
                     'k' => $k,
                     'mayModify' => $mayModify,
                     'mayChangeCorrelation' => $mayChangeCorrelation,
-                    'page' => $page,
                     'fieldCount' => $fieldCount,
                     'includeRelatedTags' => !empty($includeRelatedTags) ? 1 : 0,
                     'includeDecayingScore' => !empty($includeDecayingScore) ? 1 : 0,
                     'includeSightingdb' => !empty($includeSightingdb) ? 1 : 0
                 ));
-                if (!empty($focus) && ($object['objectType'] == 'object' || $object['objectType'] == 'attribute') && $object['uuid'] == $focus) {
-                    $focusedRow = $k;
-                }
                 if (
-                    ($object['objectType'] == 'attribute' && !empty($object['ShadowAttribute'])) ||
-                    $object['objectType'] == 'object'
+                    ($object['objectType'] === 'attribute' && !empty($object['ShadowAttribute'])) ||
+                    $object['objectType'] === 'object'
                 ):
         ?>
                     <tr class="blank_table_row"><td colspan="<?php echo $fieldCount; ?>"></td></tr>
@@ -218,9 +187,8 @@
         ?>
     </table>
 </div>
-    <?php if ($emptyEvent && (empty($attributeFilter) || $attributeFilter === 'all')): ?>
-        <div class="background-red bold">
-            <span>
+    <?php if ($emptyEvent && (empty($attributeFilter) || $attributeFilter === 'all') && !$filtered): ?>
+        <div class="background-red bold" style="padding: 2px 5px">
             <?php
                 if ($me['org_id'] != $event['Event']['orgc_id']) {
                     echo __('Attribute warning: This event doesn\'t have any attributes visible to you. Either the owner of the event decided to have
@@ -230,7 +198,6 @@ attributes or the appropriate distribution level. If you think there is a mistak
                     echo __('Attribute warning: This event doesn\'t contain any attribute. It\'s strongly advised to populate the event with attributes (indicators, observables or information) to provide a meaningful event');
                 }
             ?>
-            </span>
         </div>
     <?php endif;?>
     <div class="pagination">
@@ -240,8 +207,8 @@ attributes or the appropriate distribution level. If you think there is a mistak
                 'url' => $url,
                 'update' => '#attributes_div',
                 'evalScripts' => true,
-                'before' => '$(".progress").show()',
-                'complete' => '$(".progress").hide()',
+                'before' => '$(".loading").show()',
+                'complete' => '$(".loading").hide()',
             ));
             echo $this->Paginator->prev('&laquo; ' . __('previous'), array('tag' => 'li', 'escape' => false), null, array('tag' => 'li', 'class' => 'prev disabled', 'escape' => false, 'disabledTag' => 'span'));
             echo $this->Paginator->numbers(array('modulus' => 60, 'separator' => '', 'tag' => 'li', 'currentClass' => 'red', 'currentTag' => 'span'));
@@ -262,29 +229,23 @@ attributes or the appropriate distribution level. If you think there is a mistak
     var currentUri = "<?php echo isset($currentUri) ? h($currentUri) : $baseurl . '/events/viewEventAttributes/' . h($event['Event']['id']); ?>";
     var currentPopover = "";
     var ajaxResults = {"hover": [], "persistent": []};
-    var timer;
     var lastSelected = false;
     var deleted = <?php echo (!empty($deleted)) ? '1' : '0';?>;
     var includeRelatedTags = <?php echo (!empty($includeRelatedTags)) ? '1' : '0';?>;
-    $(document).ready(function() {
+    $(function() {
         $('.addGalaxy').click(function() {
             addGalaxyListener(this);
         });
         <?php
-            if ($focusedRow !== false):
+            if (isset($focus)):
         ?>
-            //window.location.hash = '.row_' + '<?php echo h($focusedRow); ?>';
-            //$.scrollTo('#row_' + '<?php echo h($k); ?>', 800, {easing:'elasout'});
-            //$('html,body').animate({scrollTop: $('#row_' + '<?php echo h($k); ?>').offset().top}, 'slow');
-                $('.row_' + '<?php echo h($focusedRow); ?>').focus();
+        focusObjectByUuid('<?= h($focus); ?>');
         <?php
             endif;
         ?>
         setContextFields();
         popoverStartup();
-        $('.select_attribute').removeAttr('checked');
-        $('.select_proposal').removeAttr('checked');
-        $('.select_attribute').click(function(e) {
+        $('.select_attribute').prop('checked', false).click(function(e) {
             if ($(this).is(':checked')) {
                 if (e.shiftKey) {
                     selectAllInbetween(lastSelected, this.id);
@@ -293,7 +254,7 @@ attributes or the appropriate distribution level. If you think there is a mistak
             }
             attributeListAnyAttributeCheckBoxesChecked();
         });
-        $('.select_proposal').click(function(e){
+        $('.select_proposal').prop('checked', false).click(function(e){
             if ($(this).is(':checked')) {
                 if (e.shiftKey) {
                     selectAllInbetween(lastSelected, this.id);
@@ -334,78 +295,10 @@ attributes or the appropriate distribution level. If you think there is a mistak
             url = "<?php echo $baseurl; ?>" + "/sightings/advanced/" + object_id + "/" + object_context;
             genericPopup(url, '#popover_box');
         });
-        $(".eventViewAttributeHover").mouseenter(function() {
-            if (currentPopover !== undefined && currentPopover !== '') {
-                $('#' + currentPopover).popover('destroy');
-            }
-            var type = $(this).attr('data-object-type');
-            var id = $(this).attr('data-object-id');
-
-            if (type + "_" + id in ajaxResults["hover"]) {
-                var element = $('#' + type + '_' + id + '_container');
-                element.popover({
-                    title: attributeHoverTitle(id, type),
-                    content: ajaxResults["hover"][type + "_" + id],
-                    placement: attributeHoverPlacement(element),
-                    html: true,
-                    trigger: 'manual',
-                    container: 'body'
-                }).popover('show');
-                currentPopover = type + '_' + id + '_container';
-            } else {
-              timer = setTimeout(function () {
-                  runHoverLookup(type, id)
-                },
-                500
-              );
-            }
-        }).mouseout(function() {
-            clearTimeout(timer);
-        });
     });
-    $('#attributesFilterField').bind("keydown", function(e) {
-        var eventid = $('#attributesFilterField').data("eventid");
-        if ((e.keyCode == 13 || e.keyCode == 10)) {
-            filterAttributes('value', eventid);
-        }
-    });
-    $('.hex-value-convert').click(function() {
-        var val = $(this).parent().children(':first-child').text();
-        if ($(this).parent().children(':first-child').attr('data-original-title') == 'Hexadecimal representation') {
-            var bin = [];
-            var temp;
-            val.split('').forEach(function(entry) {
-                temp = parseInt(entry, 16).toString(2);
-                bin.push(Array(5 - (temp.length)).join('0') + temp);
-            });
-            bin = bin.join(' ');
-            $(this).parent().children(':first-child').text(bin);
-            $(this).parent().children(':first-child').attr('data-original-title', 'Binary representation');
-            $(this).parent().children(':nth-child(2)').attr('data-original-title', 'Switch to hexadecimal representation');
-            $(this).parent().children(':nth-child(2)').attr('aria-label', 'Switch to hexadecimal representation');
-        } else {
-            val = val.split(' ');
-            hex = '';
-            val.forEach(function(entry) {
-                hex += parseInt(entry , 2).toString(16).toUpperCase();
-            });
-            $(this).parent().children(':first-child').text(hex);
-            $(this).parent().children(':first-child').attr('data-original-title', 'Hexadecimal representation');
-            $(this).parent().children(':nth-child(2)').attr('data-original-title', 'Switch to binary representation');
-            $(this).parent().children(':nth-child(2)').attr('aria-label', 'Switch to binary representation');
-        }
-    });
-    $('.searchFilterButton').click(function() {
+    $('.searchFilterButton, #quickFilterButton').click(function() {
         filterAttributes('value', '<?php echo h($event['Event']['id']); ?>');
     });
-    $('#quickFilterButton').click(function() {
-        filterAttributes('value', '<?php echo h($event['Event']['id']); ?>');
-    });
-    $('#quickFilterField').on('keypress', function (e) {
-        if(e.which === 13) {
-            filterAttributes('value', '<?php echo h($event['Event']['id']); ?>');
-        }
-    });
 </script>
 <?php
     echo $this->Js->writeBuffer();

app/View/Elements/eventattributecreation.ctp

@@ -1,207 +0,0 @@
-<?php
-    $mayModify = (($isAclModify && $event['Event']['user_id'] == $me['id'] && $event['Orgc']['id'] == $me['org_id']) || ($isAclModifyOrg && $event['Orgc']['id'] == $me['org_id']));
-    $mayPublish = ($isAclPublish && $event['Orgc']['id'] == $me['org_id']);
-?>
-
-<div class="attribute_creation">
-    <?php echo $this->Form->create('Attribute');?>
-        <fieldset>
-            <legend><?php echo __('Add Attribute'); ?></legend>
-            <div class="add_attribute_fields">
-                <?php
-                echo $this->Form->hidden('event_id');
-                echo $this->Form->input('category', array(
-                        'empty' => '(choose one)'
-                        ));
-                echo $this->Form->input('type', array(
-                        'empty' => '(first choose category)'
-                        ));
-
-                $initialDistribution = 3;
-                if (Configure::read('MISP.default_attribute_distribution') != null) {
-                    if (Configure::read('MISP.default_attribute_distribution') === 'event') {
-                        $initialDistribution = 5;
-                    } else {
-                        $initialDistribution = Configure::read('MISP.default_attribute_distribution');
-                    }
-                }
-                echo $this->Form->input('distribution', array(
-                    'options' => array($distributionLevels),
-                    'label' => __('Distribution'),
-                    'selected' => $initialDistribution,
-                ));
-
-                echo $this->Form->input('value', array(
-                        'type' => 'textarea',
-                        'error' => array('escape' => false),
-                        'div' => 'input clear',
-                        'class' => 'input-xxlarge'
-                ));
-                echo $this->Form->input('comment', array(
-                        'type' => 'text',
-                        'label' => __('Contextual Comment'),
-                        'error' => array('escape' => false),
-                        'div' => 'input clear',
-                        'class' => 'input-xxlarge'
-                ));
-                ?>
-                <div class="input clear"></div>
-                <?php
-                echo $this->Form->input('to_ids', array(
-                            'checked' => false,
-                            'data-content' => isset($attrDescriptions['signature']['formdesc']) ? $attrDescriptions['signature']['formdesc'] : $attrDescriptions['signature']['desc'],
-                            'label' => __('for Intrusion Detection System'),
-                ));
-                echo $this->Form->input('batch_import', array(
-                        'type' => 'checkbox',
-                        'data-content' => __('Create multiple attributes one per line'),
-                ));
-                // link an onchange event to the form elements
-                $this->Js->get('#AttributeCategory')->event('change', 'formCategoryChanged("#AttributeCategory")');
-                ?>
-            </div>
-        </fieldset>
-        <div class="overlay_spacing">
-            <table>
-                <tr>
-                <td style="vertical-align:top">
-                    <?php
-                    echo $this->Js->submit('Submit', array(
-                            'before'=>$this->Js->get('#loading')->effect('fadeIn'),
-                            'success'=>$this->Js->get('#loading')->effect('fadeOut'),
-                            'complete'=> $this->Js->request(
-                                    array('action' => 'view', $event['Event']['id'], 'attributesPage:' . $page),
-                                    array(
-                                            'update' => '#attributes_div',
-                                            'before' => '$(".loading").show();$("#gray_out").hide();$("#attribute_creation_div").hide();',
-                                            'complete' => '$(".loading").hide();',
-                                    )
-                            ),
-                            'class'=>'btn btn-primary',
-                            'url' => $baseurl . '/attributes/add/' . $event['Event']['id']
-                    ));
-                    ?>
-                </td>
-                <td style="width:540px;">
-                    <p style="color:red;font-weight:bold;display:none;text-align:center" id="warning-message"><?php echo __('Warning: You are about to share data that is of a classified nature (Attribution / targeting data). Make sure that you are authorised to share this.');?></p>
-                </td>
-                <td style="vertical-align:top;">
-                    <span class="btn btn-inverse" id="cancel_attribute_add"><?php echo __('Cancel');?></span>
-                </td>
-                </tr>
-            </table>
-        </div>
-    <?php
-        echo $this->Form->end();
-    ?>
-
-    <script type="text/javascript">
-    //
-    //Generate Category / Type filtering array
-    //
-    var category_type_mapping = new Array();
-    <?php
-    foreach ($categoryDefinitions as $category => $def) {
-        echo "category_type_mapping['" . addslashes($category) . "'] = {";
-        $first = true;
-        foreach ($def['types'] as $type) {
-            if ($first) $first = false;
-            else echo ', ';
-            echo "'" . addslashes($type) . "' : '" . addslashes($type) . "'";
-        }
-        echo "}; \n";
-    }
-    ?>
-    function formCategoryChanged(id) {
-        // fill in the types
-        var options = $('#AttributeType').prop('options');
-        $('option', $('#AttributeType')).remove();
-        $.each(category_type_mapping[$('#AttributeCategory').val()], function(val, text) {
-            options[options.length] = new Option(text, val);
-        });
-        // enable the form element
-        $('#AttributeType').prop('disabled', false);
-    }
-
-
-    //
-    // Generate tooltip information
-    //
-    var formInfoValues = new Array();
-    <?php
-    foreach ($typeDefinitions as $type => $def) {
-        $info = isset($def['formdesc']) ? $def['formdesc'] : $def['desc'];
-        echo "formInfoValues['" . addslashes($type) . "'] = \"" . addslashes($info) . "\";\n";  // as we output JS code we need to add slashes
-    }
-    foreach ($categoryDefinitions as $category => $def) {
-        $info = isset($def['formdesc']) ? $def['formdesc'] : $def['desc'];
-        echo "formInfoValues['" . addslashes($category) . "'] = \"" . addslashes($info) . "\";\n"; // as we output JS code we need to add slashes
-    }
-    foreach ($distributionDescriptions as $type => $def) {
-        $info = isset($def['formdesc']) ? $def['formdesc'] : $def['desc'];
-        echo "formInfoValues['" . addslashes($type) . "'] = \"" . addslashes($info) . "\";\n";  // as we output JS code we need to add slashes
-    }
-    ?>
-
-    $(document).ready(function() {
-
-        $("#AttributeType, #AttributeCategory, #Attribute, #AttributeDistribution").on('mouseover', function(e) {
-            var $e = $(e.target);
-            if ($e.is('option')) {
-                $('#'+e.currentTarget.id).popover('destroy');
-                $('#'+e.currentTarget.id).popover({
-                    trigger: 'focus',
-                    placement: 'right',
-                    container: 'body',
-                    content: formInfoValues[$e.val()],
-                }).popover('show');
-            }
-        });
-
-        $("input, label").on('mouseleave', function(e) {
-            $('#'+e.currentTarget.id).popover('destroy');
-        });
-
-        $("input, label").on('mouseover', function(e) {
-            var $e = $(e.target);
-            $('#'+e.currentTarget.id).popover('destroy');
-            $('#'+e.currentTarget.id).popover({
-                trigger: 'focus',
-                placement: 'right',
-                container: 'body',
-            }).popover('show');
-
-        });
-
-        // workaround for browsers like IE and Chrome that do now have an onmouseover on the 'options' of a select.
-        // disadvangate is that user needs to click on the item to see the tooltip.
-        // no solutions exist, except to generate the select completely using html.
-        $("#AttributeType, #AttributeCategory, #Attribute, #AttributeDistribution").on('change', function(e) {
-            if (this.id === "AttributeCategory") {
-                var select = document.getElementById("AttributeCategory");
-                if (select.value === 'Attribution' || select.value === 'Targeting data') {
-                    $("#warning-message").show();
-                } else {
-                    $("#warning-message").hide();
-                }
-            }
-            var $e = $(e.target);
-            $('#'+e.currentTarget.id).popover('destroy');
-            $('#'+e.currentTarget.id).popover({
-                trigger: 'focus',
-                placement: 'right',
-                container: 'body',
-                content: formInfoValues[$e.val()],
-            }).popover('show');
-        });
-
-        $('#cancel_attribute_add').click(function() {
-            $('#gray_out').hide();
-            $('#attribute_creation_div').hide();
-        });
-    });
-    </script>
-</div>
-<?php
-    echo $this->Js->writeBuffer();
-?>

app/View/Elements/eventattributetoolbar.ctp

@@ -229,7 +229,7 @@
                 'type' => 'search',
                 'fa-icon' => 'search',
                 'placeholder' => __('Enter value to search'),
-                'data' => '',
+                'value' => isset($this->passedArgs['searchFor']) ? $this->passedArgs['searchFor'] : null,
                 'cancel' => array(
                     'fa-icon' => 'times',
                     'title' => __('Remove filters'),
@@ -241,4 +241,3 @@
     );
     echo $this->element('/genericElements/ListTopBar/scaffold', array('data' => $data));
     echo $this->element('/Events/View/eventFilteringQueryBuilder');
-?>

app/View/Elements/galaxyQuickView.ctp

@@ -38,7 +38,7 @@
                     } else if($cluster_field['key'] == 'country') {
                         $value = array();
                         foreach ($cluster_field['value'] as $k => $v) {
-                            $value[] = '<div class="famfamfam-flag-' . strtolower(h($v)) . '" ></div>&nbsp;' . h($v);
+                            $value[] = $this->Icon->countryFlag($v) . '&nbsp;' . h($v);
                         }
                         $dataValue .= nl2br(implode("\n", $value));
                     } else {

app/View/Elements/galaxyQuickViewMini.ctp

@@ -45,7 +45,7 @@
                     } else if($cluster_field['key'] == 'country') {
                         $value = array();
                         foreach ($cluster_field['value'] as $k => $v) {
-                            $value[] = '<span class="famfamfam-flag-' . strtolower(h($v)) . '" ></span>&nbsp;' . h($v);
+                            $value[] = $this->Icon->countryFlag($v) . '&nbsp;' . h($v);
                         }
                         $value_contents = nl2br(implode("\n", $value));
                     } else {