Merge branch '2.4' into fix-duplicated-uuids

.travis.yml

@@ -25,17 +25,15 @@ before_install:
 install:
     - date
     - sudo apt-get -y update
-    # Travis lacks entropy.
-    - sudo apt-get -y install haveged
-    - sudo apt-get -y install python3 python3-venv python3-pip python3-dev python3-nose libxml2-dev libzmq3-dev zlib1g-dev apache2 curl php-mysql php-dev php-cli libapache2-mod-php libfuzzy-dev php-mbstring libonig4 php-json php-xml php-opcache php-readline php-redis php-gnupg php-gd
-    - sudo apt-get -y dist-upgrade
+    # Install haveged, because Travis lacks entropy.
+    - sudo apt-get -y install haveged python3 python3-venv python3-pip python3-dev python3-nose python3-redis python3-lxml python3-dateutil python3-msgpack libxml2-dev libzmq3-dev zlib1g-dev apache2 curl php-mysql php-dev php-cli libapache2-mod-php libfuzzy-dev php-mbstring libonig4 php-json php-xml php-opcache php-readline php-redis php-gnupg php-gd
     - sudo pip3 install --upgrade pip setuptools requests pyzmq
     - sudo pip3 install --upgrade -r requirements.txt
-    - sudo pip3 install poetry
+    - pip3 install --user poetry
     - phpenv rehash
     - sudo mkdir $HOME/.composer ; sudo chown $USER:www-data $HOME/.composer
     - pushd app
-    - sudo -H -u $USER php composer.phar install
+    - sudo -H -u $USER php composer.phar install --no-progress
     - sudo phpenmod redis
     - sudo phpenmod gnupg
     - popd
@@ -85,13 +83,13 @@ install:
     # /!\ VERY INSECURE BUT FASTER ON THE BUILD ENV OF TRAVIS
     - sudo cp -a /dev/urandom /dev/random
     - sudo gpg --no-tty --no-permission-warning --pinentry-mode=loopback --passphrase "travistest" --homedir `pwd`/.gnupg --gen-key --batch `pwd`/travis/gpg
-    - sudo chown $USER:www-data `pwd`/.gnupg
-    - sudo chmod 700 `pwd`/.gnupg
+    - sudo gpg --list-secret-keys --homedir `pwd`/.gnupg
     # change perms
     - sudo chown -R $USER:www-data `pwd`
     - sudo chmod +x /home/travis/build
     - sudo chmod +x /home/travis
     - sudo chmod +x /home
+    - sudo chmod -R 770 `pwd`/.gnupg
     # Get authkey
     - sudo usermod -a -G www-data $USER
     - sudo -E su $USER -c 'app/Console/cake userInit -q | sudo tee ./key.txt'
@@ -108,6 +106,7 @@ install:
     - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_port" 6379'
     - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_database" 13'
     - sudo -E su $USER -c 'app/Console/cake Admin setSetting "MISP.redis_password" ""'
+    - sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.email" "info@admin.test"'
     - sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.homedir" "`pwd`/.gnupg"'
     - sudo -E su $USER -c 'app/Console/cake Admin setSetting "GnuPG.password" "travistest"'
     - sudo -E su $USER -c 'app/Console/cake Admin updateGalaxies'
@@ -121,7 +120,7 @@ install:
     - sudo chmod -R 777 ./tests
     # Start workers
     - sudo chmod +x app/Console/worker/start.sh
-    - app/Console/worker/start.sh &
+    - sudo -E su $USER -c 'app/Console/worker/start.sh &'
     - sleep 10
     # Dirty install python stuff
     - virtualenv -p python3.6 ./venv
@@ -167,13 +166,11 @@ script:
     - popd
 
 after_failure:
-    - ls -Rl `pwd`/app
+    - ls -aRl `pwd`
     - curl http://misp.local
     - cat /etc/apache2/sites-available/misp.local.conf
+    - sudo tail -n +1 `pwd`/app/tmp/logs/*
     - sudo ls -l /var/log/apache2
-    - sudo cat `pwd`/app/tmp/logs/error.log
-    - sudo cat `pwd`/app/tmp/logs/exec-errors.log
-    - sudo cat `pwd`/app/tmp/logs/debug.log
     - sudo cat /var/log/apache2/error.log
     - sudo cat /var/log/apache2/misp.local_error.log
     - sudo cat /var/log/apache2/misp.local_access.log
@@ -188,6 +185,7 @@ notifications:
     on_start: never     # options: [always|never|change] default: always
 
 after_success:
+    - sudo tail -n +1 `pwd`/app/tmp/logs/*
     - coveralls
     - coverage report
     - coverage xml

INSTALL/workerstartsh.te

@@ -0,0 +1,18 @@
+module my-startsh 1.0;
+
+require {
+	type httpd_sys_script_exec_t;
+	type init_t;
+	type httpd_sys_rw_content_t;
+	class file { execute execute_no_trans open read };
+}
+
+#============= init_t ==============
+
+#!!!! This avc is allowed in the current policy
+allow init_t httpd_sys_rw_content_t:file execute;
+allow init_t httpd_sys_rw_content_t:file { open read };
+allow init_t httpd_sys_script_exec_t:file execute_no_trans;
+
+#!!!! This avc is allowed in the current policy
+allow init_t httpd_sys_script_exec_t:file { execute open read };

PyMISP

@@ -1 +1 @@
-Subproject commit b952876361882782128a7e95fcdced9deace14cc
+Subproject commit 918f841087cee950384a50ca7bcd22e769700ca2

VERSION.json

@@ -1 +1 @@
-{"major":2, "minor":4, "hotfix":129}
+{"major":2, "minor":4, "hotfix":130}

app/Config/bootstrap.default.php

@@ -108,6 +108,18 @@ if (!$relativePaths) {
 }
 
 /**
+ * Configure base URL for CakePHP
+ */
+if (Configure::read('MISP.baseurl')) {
+	$regex = "%^(?<fullBaseUrl>(?<proto>https?)://(?<host>(?:(?:\w|-)+\.)+[a-z]{2,5})(?::(?<port>[0-9]+))?)(?<base>/[a-z0-9_\-\.]+)?$%i";
+	if (preg_match($regex, Configure::read('MISP.baseurl'), $matches)) {
+		if (isset($matches['base'])) {
+			Configure::write('App.base', $matches['base']);
+			Configure::write('App.fullBaseUrl', $matches['fullBaseUrl']);
+		}
+	}
+}
+/*
  * Plugins need to be loaded manually, you can either load them one by one or all of them in a single call
  * Uncomment one of the lines below, as you need. make sure you read the documentation on CakePlugin to use more
  * advanced ways of loading plugins

app/Config/config.default.php

@@ -48,6 +48,7 @@ $config = array(
 			'password'          => '',
             'bodyonlyencrypted' => false,
             'sign'              => true,
+            'obscure_subject'   => false,
 		),
 	'SMIME'            =>
 		array(

app/Console/Command/AdminShell.php

@@ -2,7 +2,7 @@
 App::uses('AppShell', 'Console/Command');
 class AdminShell extends AppShell
 {
-    public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Whitelist', 'Server', 'Organisation', 'AdminSetting', 'Galaxy', 'Taxonomy', 'Warninglist', 'Noticelist', 'ObjectTemplate', 'Bruteforce', 'Role', 'Feed');
+    public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Allowedlist', 'Server', 'Organisation', 'AdminSetting', 'Galaxy', 'Taxonomy', 'Warninglist', 'Noticelist', 'ObjectTemplate', 'Bruteforce', 'Role', 'Feed');
 
     public $tasks = array('ConfigLoad');
 

app/Console/Command/EventShell.php

@@ -4,7 +4,7 @@ App::uses('File', 'Utility');
 require_once 'AppShell.php';
 class EventShell extends AppShell
 {
-    public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Whitelist', 'Server', 'Organisation');
+    public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Allowedlist', 'Server', 'Organisation');
     public $tasks = array('ConfigLoad');
 
     public function doPublish()
@@ -136,6 +136,9 @@ class EventShell extends AppShell
         $eventId = $this->args[2];
         $oldpublish = $this->args[3];
         $user = $this->User->getAuthUser($userId);
+        if (empty($user)) {
+            die("Invalid user ID '$userId' provided.");
+        }
         $result = $this->Event->sendAlertEmail($eventId, $user, $oldpublish, $processId);
         $job['Job']['progress'] = 100;
         $job['Job']['message'] = 'Emails sent.';
@@ -154,6 +157,9 @@ class EventShell extends AppShell
         $processId = $this->args[5];
         $this->Job->id = $processId;
         $user = $this->User->getAuthUser($userId);
+        if (empty($user)) {
+            die("Invalid user ID '$userId' provided.");
+        }
         $result = $this->Event->sendContactEmail($id, $message, $all, array('User' => $user), $isSiteAdmin);
         $this->Job->saveField('progress', '100');
         $this->Job->saveField('date_modified', date("Y-m-d H:i:s"));
@@ -238,6 +244,9 @@ class EventShell extends AppShell
         $jobId = $this->args[2];
         $userId = $this->args[3];
         $user = $this->User->getAuthUser($userId);
+        if (empty($user)) {
+            die("Invalid user ID '$userId' provided.");
+        }
         $job = $this->Job->read(null, $jobId);
         $this->Event->Behaviors->unload('SysLogLogable.SysLogLogable');
         $result = $this->Event->publish($id, $passAlong);
@@ -262,6 +271,9 @@ class EventShell extends AppShell
         $jobId = $this->args[2];
         $userId = $this->args[3];
         $user = $this->User->getAuthUser($userId);
+        if (empty($user)) {
+            die("Invalid user ID '$userId' provided.");
+        }
         $job = $this->Job->read(null, $jobId);
         $this->Event->Behaviors->unload('SysLogLogable.SysLogLogable');
         $result = $this->Event->publish_sightings($id, $passAlong);
@@ -324,7 +336,7 @@ class EventShell extends AppShell
         } else {
             $job['Job']['message'] = 'Enrichment finished, but no attributes added.';
         }
-        echo $job['Job']['message'] . PHP_EOL;
+	echo $job['Job']['message'] . PHP_EOL;
         $this->Job->save($job);
         $log = ClassRegistry::init('Log');
         $log->create();
@@ -345,7 +357,7 @@ class EventShell extends AppShell
             $inputData['attributes'],
             $inputData['id'],
             $inputData['default_comment'],
-            $inputData['force'],
+            $inputData['proposals'],
             $inputData['adhereToWarninglists'],
             $inputData['jobId']
         );

app/Console/Command/ServerShell.php

@@ -338,7 +338,7 @@ class ServerShell extends AppShell
             $message = __n(
                 '%s feed from %s cached. Failed: %s',
                 '%s feeds from %s cached. Failed: %s',
-                $result['successes'], $total, $result['fails']
+                $result['successes'], $result['successes'], $total, $result['fails']
             );
             if ($result['fails'] > 0) {
                 $message .= ' ' . __('See error logs for more details.');

app/Controller/AllowedlistsController.php

@@ -2,7 +2,7 @@
 
 App::uses('AppController', 'Controller');
 
-class WhitelistsController extends AppController
+class AllowedlistsController extends AppController
 {
     public $components = array(
         'Security',
@@ -12,7 +12,7 @@ class WhitelistsController extends AppController
     public $paginate = array(
             'limit' => 60,
             'order' => array(
-                    'Whitelist.name' => 'ASC'
+                    'Allowedlist.name' => 'ASC'
             )
     );
 
@@ -27,7 +27,7 @@ class WhitelistsController extends AppController
     public function admin_index()
     {
         if (!$this->userRole['perm_regexp_access']) {
-            $this->redirect(array('controller' => 'whitelists', 'action' => 'index', 'admin' => false));
+            $this->redirect(array('controller' => 'allowedlists', 'action' => 'index', 'admin' => false));
         }
         $this->AdminCrud->adminIndex();
     }
@@ -35,7 +35,7 @@ class WhitelistsController extends AppController
     public function admin_edit($id = null)
     {
         if (!$this->userRole['perm_regexp_access']) {
-            $this->redirect(array('controller' => 'whitelists', 'action' => 'index', 'admin' => false));
+            $this->redirect(array('controller' => 'allowedlists', 'action' => 'index', 'admin' => false));
         }
         $this->AdminCrud->adminEdit($id);
     }
@@ -43,7 +43,7 @@ class WhitelistsController extends AppController
     public function admin_delete($id = null)
     {
         if (!$this->userRole['perm_regexp_access']) {
-            $this->redirect(array('controller' => 'whitelists', 'action' => 'index', 'admin' => false));
+            $this->redirect(array('controller' => 'allowedlists', 'action' => 'index', 'admin' => false));
         }
         $this->AdminCrud->adminDelete($id);
     }

app/Controller/AppController.php

@@ -46,8 +46,8 @@ class AppController extends Controller
 
     public $helpers = array('Utility', 'OrgImg', 'FontAwesome', 'UserName', 'DataPathCollector');
 
-    private $__queryVersion = '108';
-    public $pyMispVersion = '2.4.128';
+    private $__queryVersion = '109';
+    public $pyMispVersion = '2.4.130';
     public $phpmin = '7.2';
     public $phprec = '7.4';
     public $pythonmin = '3.6';
@@ -1192,6 +1192,9 @@ class AppController extends Controller
         );
         $exception = false;
         $filters = $this->_harvestParameters($filterData, $exception, $this->_legacyParams);
+        if (empty($filters) && $this->request->is('get')) {
+            throw new InvalidArgumentException(__('Restsearch queries using GET and no parameters are not allowed. If you have passed parameters via a JSON body, make sure you use POST requests.'));
+        }
         if (empty($filters['returnFormat'])) {
             $filters['returnFormat'] = 'json';
         }
@@ -1231,4 +1234,54 @@ class AppController extends Controller
             return $this->RestResponse->viewData($final, $responseType, false, true, $filename, array('X-Result-Count' => $elementCounter, 'X-Export-Module-Used' => $returnFormat, 'X-Response-Format' => $responseType));
         }
     }
+
+    /**
+     * Returns true if user can modify given event.
+     *
+     * @param array $event
+     * @return bool
+     */
+    protected function __canModifyEvent(array $event)
+    {
+        if (!isset($event['Event'])) {
+            throw new InvalidArgumentException('Passed object does not contains Event.');
+        }
+
+        if ($this->userRole['perm_site_admin']) {
+            return true;
+        }
+        if ($this->userRole['perm_modify_org'] && $event['Event']['orgc_id'] == $this->Auth->user('org_id')) {
+            return true;
+        }
+        if ($this->userRole['perm_modify'] && $event['Event']['user_id'] == $this->Auth->user('id')) {
+            return true;
+        }
+        return false;
+    }
+
+    /**
+     * Returns true if user can add or remove tags for given event.
+     *
+     * @param array $event
+     * @param bool $isTagLocal
+     * @return bool
+     */
+    protected function __canModifyTag(array $event, $isTagLocal = false)
+    {
+        // Site admin can add any tag
+        if ($this->userRole['perm_site_admin']) {
+            return true;
+        }
+        // User must have tagger or sync permission
+        if (!$this->userRole['perm_tagger'] && !$this->userRole['perm_sync']) {
+            return false;
+        }
+        if ($this->__canModifyEvent($event)) {
+            return true; // full access
+        }
+        if ($isTagLocal && Configure::read('MISP.host_org_id') == $this->Auth->user('org_id')) {
+            return true;
+        }
+        return false;
+    }
 }

app/Controller/Component/ACLComponent.php

@@ -111,12 +111,37 @@ class ACLComponent extends Component
                     'requestAccess' => array(),
                     'view' => array()
             ),
-            'eventBlacklists' => array(
-                    'add' => array(),
-                    'delete' => array(),
-                    'edit' => array(),
-                    'index' => array(),
-                    'massDelete' => array()
+            'eventBlocklists' => array(
+                    'add' => [
+                        'AND' => [
+                            'host_org_user',
+                            'perm_add'
+                        ]
+                    ],
+                    'delete' => [
+                        'AND' => [
+                            'host_org_user',
+                            'perm_add'
+                        ]
+                    ],
+                    'edit' => [
+                        'AND' => [
+                            'host_org_user',
+                            'perm_add'
+                        ]
+                    ],
+                    'index' => [
+                        'AND' => [
+                            'host_org_user',
+                            'perm_add'
+                        ]
+                    ],
+                    'massDelete' => [
+                        'AND' => [
+                            'host_org_user',
+                            'perm_add'
+                        ]
+                    ]
             ),
             'eventDelegations' => array(
                     'acceptDelegation' => array('perm_add'),
@@ -324,7 +349,7 @@ class ACLComponent extends Component
             'objectTemplateElements' => array(
                 'viewElements' => array('*')
             ),
-            'orgBlacklists' => array(
+            'orgBlocklists' => array(
                     'add' => array(),
                     'delete' => array(),
                     'edit' => array(),
@@ -398,6 +423,7 @@ class ACLComponent extends Component
                     'getSubmoduleQuickUpdateForm' => array(),
                     'getWorkers' => array(),
                     'getVersion' => array('*'),
+                    'idTranslator' => array('*'),
                     'import' => array(),
                     'index' => array(),
                     'ondemandAction' => array(),
@@ -607,7 +633,7 @@ class ACLComponent extends Component
                     'update' => array(),
                     'view' => array('*')
             ),
-            'whitelists' => array(
+            'allowedlists' => array(
                     'admin_add' => array('perm_regexp_access'),
                     'admin_delete' => array('perm_regexp_access'),
                     'admin_edit' => array('perm_regexp_access'),
@@ -714,6 +740,7 @@ class ACLComponent extends Component
     {
         $controller = lcfirst(Inflector::camelize($controller));
         $action = strtolower($action);
+        $host_org_id = Configure::read('MISP.host_org_id');
         $aclList = $this->__aclList;
         foreach ($aclList as $k => $v) {
             $aclList[$k] = array_change_key_case($v);
@@ -731,21 +758,35 @@ class ACLComponent extends Component
             }
             if (isset($aclList[$controller][$action]['OR'])) {
                 foreach ($aclList[$controller][$action]['OR'] as $permission) {
-                    if ($user['Role'][$permission]) {
-                        return true;
+                    if ($permission === 'host_org_user') {
+                        if ((int)$user['org_id'] === (int)$host_org_id) {
+                            return true;
+                        }
+                    } else {
+                        if ($user['Role'][$permission]) {
+                            return true;
+                        }
                     }
                 }
             } elseif (isset($aclList[$controller][$action]['AND'])) {
                 $allConditionsMet = true;
                 foreach ($aclList[$controller][$action]['AND'] as $permission) {
-                    if (!$user['Role'][$permission]) {
-                        $allConditionsMet = false;
+                    if ($permission === 'host_org_user') {
+                        if ((int)$user['org_id'] !== (int)$host_org_id) {
+                            $allConditionsMet = false;
+                        }
+                    } else {
+                        if (!$user['Role'][$permission]) {
+                            $allConditionsMet = false;
+                        }
                     }
                 }
                 if ($allConditionsMet) {
                     return true;
                 }
-            } elseif ($user['Role'][$aclList[$controller][$action][0]]) {
+            } elseif ($aclList[$controller][$action][0] !== 'host_org_user' && $user['Role'][$aclList[$controller][$action][0]]) {
+                return true;
+            } elseif ($aclList[$controller][$action][0] === 'host_org_user' && (int)$user['org_id'] === (int)$host_org_id) {
                 return true;
             }
         }

app/Controller/Component/BlockListComponent.php

@@ -4,25 +4,28 @@
  * create, read, update and delete (CRUD)
  */
 
-class BlackListComponent extends Component
+class BlocklistComponent extends Component
 {
     public $settings = array();
     public $defaultModel = '';
 
+    public $components = array('RestResponse');
 
     public function index($rest = false, $filters = array())
     {
         if (!empty($filters)) {
             $this->controller->paginate['conditions'] = $filters;
         }
-        if ($this->controller->response->type() === 'application/json' || $this->controller->response->type() == 'application/xml' || $rest) {
-            $blackList = $this->controller->paginate();
-            $blacklist= array();
-            foreach ($blackList as $item) {
-                $blacklist[] = $item[$this->controller->defaultModel];
+        if ($rest) {
+            $data = $this->controller->{$this->controller->defaultModel}->find('all', array(
+                'recursive' => -1,
+                'conditions' => isset($this->controller->paginate['conditions']) ? $this->controller->paginate['conditions'] : []
+            ));
+            $blocklist = [];
+            foreach ($data as $item) {
+                $blocklist[] = $item[$this->controller->defaultModel];
             }
-            $this->controller->set($this->controller->defaultModel, $blacklist);
-            $this->controller->set('_serialize', $this->controller->defaultModel);
+            return $this->RestResponse->viewData($blocklist);
         } else {
             $this->controller->set('response', $this->controller->paginate());
         }
@@ -44,10 +47,16 @@ class BlackListComponent extends Component
             } else {
                 $data = $this->controller->request->data;
             }
+            if (!isset($data[$this->controller->defaultModel])) {
+                $data = [$this->controller->defaultModel => $data];
+            }
+            if (!isset($data[$this->controller->defaultModel])) {
+                throw new InvalidArgumentException(__('Pass a list of uuids via the "uuids" key in the request object.'));
+            }
             if (is_array($data[$this->controller->defaultModel]['uuids'])) {
                 $uuids = $data[$this->controller->defaultModel]['uuids'];
             } else {
-                $uuids = explode(PHP_EOL, $data[$this->controller->defaultModel]['uuids']);
+                $uuids = explode(PHP_EOL, trim($data[$this->controller->defaultModel]['uuids']));
             }
             $successes = array();
             $fails = array();
@@ -56,8 +65,8 @@ class BlackListComponent extends Component
                 if (strlen($uuid) == 36) {
                     $this->controller->{$this->controller->defaultModel}->create();
                     $object = array();
-                    foreach ($this->controller->{$this->controller->defaultModel}->blacklistFields as $f) {
-                        if (strpos($f, '_uuid')) {
+                    foreach ($this->controller->{$this->controller->defaultModel}->blocklistFields as $f) {
+                        if ($f === $this->controller->{$this->controller->defaultModel}->blocklistTarget . '_uuid') {
                             $object[$f] = $uuid;
                         } else {
                             $object[$f] = !empty($data[$this->controller->defaultModel][$f]) ? $data[$this->controller->defaultModel][$f] : '';
@@ -72,11 +81,16 @@ class BlackListComponent extends Component
                     $fails[] = $uuid;
                 }
             }
-            $message = sprintf(__('Done. Added %d new entries to the blacklist. %d entries could not be saved.'), count($successes), count($fails));
+            $message = sprintf(__('Done. Added %d new entries to the blocklist. %d entries could not be saved.'), count($successes), count($fails));
             if ($rest) {
-                $this->set('result', array('successes' => $successes, 'fails' => $fails));
-                $this->set('message', $message);
-                $this->set('_serialize', array('message', 'result'));
+                $result = [
+                    'result' => [
+                        'successes' => $successes,
+                        'fails' => $fails
+                    ],
+                    'message' => $message
+                ];
+                return $this->RestResponse->viewData($result);
             } else {
                 $this->controller->Session->setFlash($message);
                 $this->controller->redirect(array('action' => 'index'));
@@ -86,18 +100,22 @@ class BlackListComponent extends Component
 
     public function edit($rest = false, $id)
     {
-        if (strlen($id) == 36) {
-            $blockEntry = $this->controller->{$this->controller->defaultModel}->find('first', array('conditions' => array('uuid' => $id)));
+        if (Validation::uuid($id)) {
+            $blockEntry = $this->controller->{$this->controller->defaultModel}->find('first', [
+                'conditions' => array(
+                    $this->controller->{$this->controller->defaultModel}->blocklistTarget . '_uuid' => $id
+                )
+            ]);
         } else {
             $blockEntry = $this->controller->{$this->controller->defaultModel}->find('first', array('conditions' => array('id' => $id)));
         }
         if (empty($blockEntry)) {
-            throw new NotFoundException('Blacklist item not found.');
+            throw new NotFoundException('Blocklist item not found.');
         }
         $this->controller->set('blockEntry', $blockEntry);
         if ($this->controller->request->is('post')) {
             if ($rest) {
-                if ($this->response->type() === 'application/json') {
+                if ($this->controller->response->type() === 'application/json') {
                     $isJson = true;
                     $data = $this->controller->request->input('json_decode', true);
                 } else {
@@ -106,10 +124,13 @@ class BlackListComponent extends Component
                 if (isset($data['request'])) {
                     $data = $data['request'];
                 }
+                if (!isset($data[$this->controller->defaultModel])) {
+                    $data = [$this->controller->defaultModel => $data];
+                }
             } else {
                 $data = $this->controller->request->data;
             }
-            $fields = $this->controller->{$this->controller->defaultModel}->blacklistFields;
+            $fields = $this->controller->{$this->controller->defaultModel}->blocklistFields;
             foreach ($fields as $f) {
                 if ($f == 'uuid') {
                     continue;
@@ -120,17 +141,23 @@ class BlackListComponent extends Component
             }
             if ($this->controller->{$this->controller->defaultModel}->save($blockEntry)) {
                 if ($rest) {
-                    $this->controller->set('message', array('Blacklist item added.'));
-                    $this->controller->set('_serialize', array('message'));
+                    return $this->RestResponse->viewData(
+                        $this->controller->{$this->controller->defaultModel}->find('first', [
+                            'recursive' => -1,
+                            'conditions' => [
+                                'id' => $this->controller->{$this->controller->defaultModel}->id
+                            ]
+                        ])
+                    );
                 } else {
-                    $this->controller->Session->setFlash(__('Blacklist item added.'));
+                    $this->controller->Session->setFlash(__('Blocklist item added.'));
                     $this->controller->redirect(array('action' => 'index'));
                 }
             } else {
                 if ($rest) {
-                    throw new MethodNotAllowedException('Could not save the blacklist item.');
+                    throw new MethodNotAllowedException('Could not save the blocklist item.');
                 } else {
-                    $this->controller->Session->setFlash(__('Could not save the blacklist item'));
+                    $this->controller->Session->setFlash(__('Could not save the blocklist item'));
                     $this->controller->redirect(array('action' => 'index'));
                 }
             }
@@ -139,26 +166,31 @@ class BlackListComponent extends Component
 
     public function delete($rest = false, $id)
     {
-        if (strlen($id) == 36) {
-            $blockEntry = $this->controller->{$this->controller->defaultModel}->find('first', array(
-                'fields' => array('id'),
-                'conditions' => array('event_uuid' => $id),
-            ));
-            $id = $blockEntry[$this->controller->defaultModel]['id'];
-        }
-        if (!$this->controller->request->is('post') && !$rest) {
-            throw new MethodNotAllowedException();
-        }
-
-        $this->controller->{$this->controller->defaultModel}->id = $id;
-        if (!$this->controller->{$this->controller->defaultModel}->exists()) {
-            throw new NotFoundException(__('Invalid blacklist entry'));
-        }
-
-        if ($this->controller->{$this->controller->defaultModel}->delete()) {
-            $this->controller->Session->setFlash(__('Blacklist entry removed'));
+        if (Validation::uuid($id)) {
+            $blockEntry = $this->controller->{$this->controller->defaultModel}->find('first', [
+                'conditions' => array(
+                    $this->controller->{$this->controller->defaultModel}->blocklistTarget . '_uuid' => $id
+                )
+            ]);
         } else {
-            $this->controller->Session->setFlash(__('Could not remove the blacklist entry'));
+            $blockEntry = $this->controller->{$this->controller->defaultModel}->find('first', array('conditions' => array('id' => $id)));
+        }
+        if (empty($blockEntry)) {
+            throw new NotFoundException(__('Invalid blocklist entry'));
+        }
+
+        if ($this->controller->{$this->controller->defaultModel}->delete($blockEntry[$this->controller->defaultModel]['id'])) {
+            $message = __('Blocklist entry removed');
+            if ($rest) {
+                return $this->RestResponse->saveSuccessResponse($this->controller->defaultModel, 'delete', $id, false, $message);
+            }
+            $this->controller->Flash->success($message);
+        } else {
+            $message = __('Could not remove the blocklist entry');
+            if ($rest) {
+                return $this->RestResponse->saveFailResponse($this->controller->defaultModel, 'delete', $id, $message);
+            }
+            $this->controller->error($message);
         }
         $this->controller->redirect(array('action' => 'index'));
     }

app/Controller/Component/DeprecationComponent.php

@@ -36,7 +36,7 @@ class DeprecationComponent extends Component
                 'add' => false,
                 'populateEventFromTemplate' => false
             ),
-            'whitelists' => array(
+            'allowedlists' => array(
                 'admin_add' => false
             )
         );

app/Controller/Component/RestResponseComponent.php

@@ -294,7 +294,7 @@ class RestResponseComponent extends Component
                         $action = substr($action, 6);
                         $admin_routing = 'admin/';
                     }
-                    $url = '/' . $admin_routing . $controller . '/' . $action;
+                    $url = $this->baseurl . '/' . $admin_routing . $controller . '/' . $action;
                     $result[$url] = $data;
                 }
             }
@@ -337,7 +337,7 @@ class RestResponseComponent extends Component
                         }
                     }
                     $data['body'] = json_encode($data['body'], JSON_PRETTY_PRINT);
-                    $url = '/' . $admin_routing . $controller . '/' . $action;
+                    $url = $this->baseurl . '/' . $admin_routing . $controller . '/' . $action;
                     $data['url'] = $url;
                     if (!empty($data['params'])) {
                         foreach ($data['params'] as $param) {
@@ -1099,7 +1099,7 @@ class RestResponseComponent extends Component
                 'input' => 'select',
                 'type' => 'string',
                 'operators' => array('equal'),
-                'values' => array('Attribute', 'Event', 'EventBlacklist', 'EventTag', 'MispObject', 'Organisation', 'Post', 'Regexp', 'Role', 'Server', 'ShadowAttribute', 'SharingGroup', 'Tag', 'Task', 'Taxonomy', 'Template', 'Thread', 'User', 'Whitelist'),
+                'values' => array('Attribute', 'Event', 'EventBlocklist', 'EventTag', 'MispObject', 'Organisation', 'Post', 'Regexp', 'Role', 'Server', 'ShadowAttribute', 'SharingGroup', 'Tag', 'Task', 'Taxonomy', 'Template', 'Thread', 'User', 'Whitelist'),
             ),
             'model_id' => array(
                 'input' => 'number',

app/Controller/EventBlocklistsController.php

@@ -1,18 +1,15 @@
 <?php
 App::uses('AppController', 'Controller');
 
-class EventBlacklistsController extends AppController
+class EventBlocklistsController extends AppController
 {
-    public $components = array('Session', 'RequestHandler', 'BlackList');
+    public $components = array('Session', 'RequestHandler', 'BlockList');
 
     public function beforeFilter()
     {
         parent::beforeFilter();
-        if (!$this->_isSiteAdmin()) {
-            $this->redirect('/');
-        }
-        if (false === Configure::read('MISP.enableEventBlacklisting')) {
-            $this->Flash->info(__('Event Blacklisting is not currently enabled on this instance.'));
+        if (false === Configure::read('MISP.enableEventBlocklisting')) {
+            $this->Flash->info(__('Event Blocklisting is not currently enabled on this instance.'));
             $this->redirect('/');
         }
     }
@@ -21,7 +18,7 @@ class EventBlacklistsController extends AppController
             'limit' => 60,
             'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page.
             'order' => array(
-                    'EventBlacklist.created' => 'DESC'
+                    'EventBlocklist.created' => 'DESC'
             ),
     );
 
@@ -46,50 +43,50 @@ class EventBlacklistsController extends AppController
         }
         $this->set('passedArgs', json_encode($passedArgs));
         $this->set('passedArgsArray', $passedArgsArray);
-        $this->BlackList->index($this->_isRest(), $params);
+        return $this->BlockList->index($this->_isRest(), $params);
     }
 
     public function add()
     {
-        $this->BlackList->add($this->_isRest());
+        return $this->BlockList->add($this->_isRest());
     }
 
     public function edit($id)
     {
-        $this->BlackList->edit($this->_isRest(), $id);
+        return $this->BlockList->edit($this->_isRest(), $id);
     }
 
     public function delete($id)
     {
-        $this->BlackList->delete($this->_isRest(), $id);
+        return $this->BlockList->delete($this->_isRest(), $id);
     }
 
     public function massDelete()
     {
         if ($this->request->is('post') || $this->request->is('put')) {
-            if (!isset($this->request->data['EventBlacklist'])) {
-                $this->request->data = array('EventBlacklist' => $this->request->data);
+            if (!isset($this->request->data['EventBlocklist'])) {
+                $this->request->data = array('EventBlocklist' => $this->request->data);
             }
-            $ids = $this->request->data['EventBlacklist']['ids'];
+            $ids = $this->request->data['EventBlocklist']['ids'];
             $event_ids = json_decode($ids, true);
             if (empty($event_ids)) {
                 throw new NotFoundException(__('Invalid event IDs.'));
             }
-            $result = $this->EventBlacklist->deleteAll(array('EventBlacklist.id' => $event_ids));
+            $result = $this->EventBlocklist->deleteAll(array('EventBlocklist.id' => $event_ids));
             if ($result) {
                 if ($this->_isRest()) {
-                    return $this->RestResponse->saveSuccessResponse('EventBlacklist', 'Deleted', $ids, $this->response->type());
+                    return $this->RestResponse->saveSuccessResponse('EventBlocklist', 'Deleted', $ids, $this->response->type());
                 } else {
-                    $this->Flash->success('Blacklist entry removed');
-                    $this->redirect(array('controller' => 'eventBlacklists', 'action' => 'index'));
+                    $this->Flash->success('Blocklist entry removed');
+                    $this->redirect(array('controller' => 'eventBlocklists', 'action' => 'index'));
                 }
             } else {
-                $error = __('Failed to delete Event from EventBlacklist. Error: ') . PHP_EOL . h($result);
+                $error = __('Failed to delete Event from EventBlocklist. Error: ') . PHP_EOL . h($result);
                 if ($this->_isRest()) {
-                    return $this->RestResponse->saveFailResponse('EventBlacklist', 'Deleted', false, $error, $this->response->type());
+                    return $this->RestResponse->saveFailResponse('EventBlocklist', 'Deleted', false, $error, $this->response->type());
                 } else {
                     $this->Flash->error($error);
-                    $this->redirect(array('controller' => 'eventBlacklists', 'action' => 'index'));
+                    $this->redirect(array('controller' => 'eventBlocklists', 'action' => 'index'));
                 }
             }
         } else {

app/Controller/GalaxiesController.php

@@ -155,14 +155,14 @@ class GalaxiesController extends AppController
         $items = array(
             array(
                 'name' => __('All clusters'),
-                'value' => "/galaxies/selectCluster/" . h($target_id) . '/' . h($target_type) . '/0'. '/local:' . $local
+                'value' => $this->baseurl . "/galaxies/selectCluster/" . h($target_id) . '/' . h($target_type) . '/0'. '/local:' . $local
             )
         );
         foreach ($galaxies as $galaxy) {
             if (!isset($galaxy['Galaxy']['kill_chain_order'])) {
                 $items[] = array(
                     'name' => h($galaxy['Galaxy']['name']),
-                    'value' => "/galaxies/selectCluster/" . $target_id . '/' . $target_type . '/' . $galaxy['Galaxy']['id'] . '/local:' . $local,
+                    'value' => $this->baseurl . "/galaxies/selectCluster/" . $target_id . '/' . $target_type . '/' . $galaxy['Galaxy']['id'] . '/local:' . $local,
                     'template' => array(
                         'preIcon' => 'fa-' . $galaxy['Galaxy']['icon'],
                         'name' => $galaxy['Galaxy']['name'],
@@ -183,7 +183,7 @@ class GalaxiesController extends AppController
                     'isMatrix' => true
                 );
                 if ($galaxy['Galaxy']['id'] == $mitreAttackGalaxyId) {
-                    $param['img'] = "/img/mitre-attack-icon.ico";
+                    $param['img'] = $this->baseurl . "/img/mitre-attack-icon.ico";
                 }
                 $items[] = $param;
             }
@@ -205,12 +205,12 @@ class GalaxiesController extends AppController
         $items = array();
         $items[] = array(
             'name' => __('All namespaces'),
-            'value' => "/galaxies/selectGalaxy/" . $target_id . '/' . $target_type . '/0' . '/local:' . $local
+            'value' => $this->baseurl . "/galaxies/selectGalaxy/" . $target_id . '/' . $target_type . '/0' . '/local:' . $local
         );
         foreach ($namespaces as $namespace) {
             $items[] = array(
                 'name' => $namespace,
-                'value' => "/galaxies/selectGalaxy/" . $target_id . '/' . $target_type . '/' . $namespace . '/local:' . $local
+                'value' => $this->baseurl . "/galaxies/selectGalaxy/" . $target_id . '/' . $target_type . '/' . $namespace . '/local:' . $local
             );
         }
 

app/Controller/GalaxyClustersController.php

@@ -1,6 +1,9 @@
 <?php
 App::uses('AppController', 'Controller');
 
+/**
+ * @property GalaxyCluster $GalaxyCluster
+ */
 class GalaxyClustersController extends AppController
 {
     public $components = array('Session', 'RequestHandler');
@@ -13,17 +16,6 @@ class GalaxyClustersController extends AppController
                 'GalaxyCluster.value' => 'ASC'
             ),
             'contain' => array(
-                'Tag' => array(
-                    'fields' => array('Tag.id'),
-                    /*
-                    'EventTag' => array(
-                        'fields' => array('EventTag.event_id')
-                    ),
-                    'AttributeTag' => array(
-                        'fields' => array('AttributeTag.event_id', 'AttributeTag.attribute_id')
-                    )
-                    */
-                ),
                 'GalaxyElement' => array(
                     'conditions' => array('GalaxyElement.key' => 'synonyms'),
                     'fields' => array('value')
@@ -34,7 +26,6 @@ class GalaxyClustersController extends AppController
     public function index($id)
     {
         $filters = $this->IndexFilter->harvestParameters(array('context', 'searchall'));
-        $contextConditions = array();
         if (empty($filters['context'])) {
             $filters['context'] = 'all';
         }
@@ -42,33 +33,46 @@ class GalaxyClustersController extends AppController
         $this->set('searchall', isset($filters['searchall']) ? $filters['searchall'] : '');
         $this->paginate['conditions'] = array('GalaxyCluster.galaxy_id' => $id);
         if (isset($filters['searchall']) && strlen($filters['searchall']) > 0) {
+            $search = '%' . strtolower($filters['searchall']) . '%';
             $synonym_hits = $this->GalaxyCluster->GalaxyElement->find(
                 'list',
                 array(
                     'recursive' => -1,
                     'conditions' => array(
-                        'LOWER(GalaxyElement.value) LIKE' => '%' . strtolower($filters['searchall']) . '%',
-                        'GalaxyElement.key' => 'synonyms' ),
-                        'fields' => array(
-                            'GalaxyElement.galaxy_cluster_id')
-                        )
+                        'LOWER(GalaxyElement.value) LIKE' => $search,
+                        'GalaxyElement.key' => 'synonyms',
+                    ),
+                    'fields' => array(
+                        'GalaxyElement.galaxy_cluster_id',
+                    )
+                )
             );
-            $this->paginate['conditions'] =
-                array("AND" => array(
+            $this->paginate['conditions'] = array(
+                "AND" => array(
                     'OR' => array(
-                        "LOWER(GalaxyCluster.value) LIKE" => '%'. strtolower($filters['searchall']) .'%',
-                        "LOWER(GalaxyCluster.description) LIKE" => '%'. strtolower($filters['searchall']) .'%',
+                        "LOWER(GalaxyCluster.value) LIKE" => $search,
+                        "LOWER(GalaxyCluster.description) LIKE" => $search,
                         "GalaxyCluster.id" => array_values($synonym_hits)
                     ),
                     "GalaxyCluster.galaxy_id" => $id
-                    ));
-            $this->set('passedArgsArray', array('all'=>$filters['searchall']));
+                )
+            );
+            $this->set('passedArgsArray', array('all' => $filters['searchall']));
         }
         $clusters = $this->paginate();
-        $sgs = $this->GalaxyCluster->Tag->EventTag->Event->SharingGroup->fetchAllAuthorised($this->Auth->user());
         foreach ($clusters as $k => $cluster) {
-            if (!empty($cluster['Tag']['id'])) {
-                $clusters[$k]['GalaxyCluster']['event_count'] = $this->GalaxyCluster->Tag->EventTag->countForTag($cluster['Tag']['id'], $this->Auth->user(), $sgs);
+            $tag = $this->GalaxyCluster->Tag->find('first', array(
+                'conditions' => array(
+                    'LOWER(name)' => strtolower($cluster['GalaxyCluster']['tag_name']),
+                ),
+                'fields' => array('id'),
+                'recursive' => -1,
+                'contain' => array('EventTag.event_id')
+            ));
+
+            if ($tag) {
+                $clusters[$k]['GalaxyCluster']['event_count'] = $this->GalaxyCluster->Tag->EventTag->countForTag($tag, $this->Auth->user());
+                $clusters[$k]['Tag'] = $tag['Tag'];
             } else {
                 $clusters[$k]['GalaxyCluster']['event_count'] = 0;
             }
@@ -76,7 +80,6 @@ class GalaxyClustersController extends AppController
         $tagIds = array();
         $sightings = array();
         if (!empty($clusters)) {
-            $galaxyType = $clusters[0]['GalaxyCluster']['type'];
             foreach ($clusters as $k => $v) {
                 $clusters[$k]['event_ids'] = array();
                 if (!empty($v['Tag'])) {
@@ -138,17 +141,16 @@ class GalaxyClustersController extends AppController
             'conditions' => $conditions
         ));
         if (!empty($cluster)) {
-            $this->loadModel('Tag');
-            $tag = $this->Tag->find('first', array(
+            $tag = $this->GalaxyCluster->Tag->find('first', array(
                 'conditions' => array(
                     'LOWER(name)' => strtolower($cluster['GalaxyCluster']['tag_name']),
                 ),
                 'fields' => array('id'),
                 'recursive' => -1,
-                'contain' => array('EventTag.tag_id')
+                'contain' => array('EventTag.event_id')
             ));
             if (!empty($tag)) {
-                $cluster['GalaxyCluster']['tag_count'] = count($tag['EventTag']);
+                $cluster['GalaxyCluster']['tag_count'] = $this->GalaxyCluster->Tag->EventTag->countForTag($tag, $this->Auth->user());
                 $cluster['GalaxyCluster']['tag_id'] = $tag['Tag']['id'];
             }
         } else {

app/Controller/LogsController.php

@@ -404,7 +404,7 @@ class LogsController extends AppController
             $this->set('actions', $actions);
 
             // combobox for models
-            $models = array('Attribute', 'Event', 'EventBlacklist', 'EventTag', 'Feed', 'DecayingModel', 'MispObject', 'Organisation', 'Post', 'Regexp', 'Role', 'Server', 'ShadowAttribute', 'SharingGroup', 'Tag', 'Task', 'Taxonomy', 'Template', 'Thread', 'User', 'Whitelist');
+            $models = array('Attribute', 'Event', 'EventBlocklist', 'EventTag', 'Feed', 'DecayingModel', 'MispObject', 'Organisation', 'Post', 'Regexp', 'Role', 'Server', 'ShadowAttribute', 'SharingGroup', 'Tag', 'Task', 'Taxonomy', 'Template', 'Thread', 'User', 'Allowedlist');
             $models = array('' => 'ALL') + $this->_arrayToValuesIndexArray($models);
             $this->set('models', $models);
             $this->set('actionDefinitions', $this->{$this->defaultModel}->actionDefinitions);

app/Controller/ObjectReferencesController.php

@@ -41,12 +41,12 @@ class ObjectReferencesController extends AppController
             'recursive' => -1,
             'contain' => array(
                 'Event' => array(
-                    'fields' => array('Event.id', 'Event.orgc_id')
+                    'fields' => array('Event.id', 'Event.orgc_id', 'Event.user_id')
                 )
             )
         ));
-        if (empty($object) || (!$this->_isSiteAdmin() && $object['Event']['orgc_id'] != $this->Auth->user('org_id'))) {
-            throw new MethodNotAllowedException('Invalid object.');
+        if (empty($object) || !$this->__canModifyEvent($object)) {
+            throw new NotFoundException('Invalid object.');
         }
         $this->set('objectId', $objectId);
         if ($this->request->is('post')) {
@@ -60,7 +60,6 @@ class ObjectReferencesController extends AppController
                 $relationship_type = $this->request->data['ObjectReference']['relationship_type_select'];
             }
             $data = array(
-                'referenced_type' => $referenced_type,
                 'referenced_id' => $referenced_id,
                 'referenced_uuid' => $referenced_uuid,
                 'relationship_type' => $relationship_type,
@@ -168,10 +167,10 @@ class ObjectReferencesController extends AppController
             'contain' => array('Object' => array('Event'))
         ));
         if (empty($objectReference)) {
-            throw new MethodNotAllowedException('Invalid object reference.');
+            throw new NotFoundException(__('Invalid object reference.'));
         }
-        if (!$this->_isSiteAdmin() && $this->Auth->user('org_id') != $objectReference['Object']['Event']['orgc_id']) {
-            throw new MethodNotAllowedException('Invalid object reference.');
+        if (!$this->__canModifyEvent($objectReference['Object'])) {
+            throw new ForbiddenException(__('Invalid object reference.'));
         }
         if ($this->request->is('post') || $this->request->is('put') || $this->request->is('delete')) {
             $result = $this->ObjectReference->smartDelete($objectReference['ObjectReference']['id'], $hard);

app/Controller/ObjectTemplatesController.php

@@ -29,12 +29,12 @@ class ObjectTemplatesController extends AppController
         $items = array();
         $items[] = array(
             'name' => __('All Objects'),
-            'value' => "/ObjectTemplates/objectChoice/" . h($event_id) . "/" . "0"
+            'value' => $this->baseurl . "/ObjectTemplates/objectChoice/" . h($event_id) . "/" . "0"
         );
         foreach($metas as $meta) {
             $items[] = array(
                 'name' => $meta,
-                'value' => "/ObjectTemplates/objectChoice/" . h($event_id) . "/" . h($meta)
+                'value' => $this->baseurl . "/ObjectTemplates/objectChoice/" . h($event_id) . "/" . h($meta)
             );
         }
 

app/Controller/ObjectsController.php

@@ -2,6 +2,9 @@
 
 App::uses('AppController', 'Controller');
 
+/**
+ * @property MispObject $MispObject
+ */
 class ObjectsController extends AppController
 {
     public $uses = 'MispObject';
@@ -29,11 +32,6 @@ class ObjectsController extends AppController
             throw new MethodNotAllowedException(__('This action can only be reached via POST requests'));
         }
         $this->request->data = $this->MispObject->attributeCleanup($this->request->data);
-        $eventFindParams = array(
-            'recursive' => -1,
-            'fields' => array('Event.id', 'Event.uuid', 'Event.orgc_id'),
-            'conditions' => array('Event.id' => $event_id)
-        );
         $template = $this->MispObject->ObjectTemplate->find('first', array(
             'conditions' => array('ObjectTemplate.id' => $template_id),
             'recursive' => -1,
@@ -41,10 +39,13 @@ class ObjectsController extends AppController
                 'ObjectTemplateElement'
             )
         ));
-        $event = $this->MispObject->Event->find('first', $eventFindParams);
-        if (empty($event) || (!$this->_isSiteAdmin() && $event['Event']['orgc_id'] != $this->Auth->user('org_id'))) {
+        $event = $this->MispObject->Event->fetchSimpleEvent($this->Auth->user(), $event_id);
+        if (empty($event)) {
             throw new NotFoundException(__('Invalid event.'));
         }
+        if (!$this->__canModifyEvent($event)) {
+            throw new ForbiddenException(__('You do not have permission to do that.'));
+        }
         $sharing_groups = array();
         if ($this->request->data['Object']['distribution'] == 4) {
             $sharing_groups[$this->request->data['Object']['sharing_group_id']] = false;
@@ -55,33 +56,8 @@ class ObjectsController extends AppController
             }
         }
         if (!empty($sharing_groups)) {
-            $sgs = $this->MispObject->SharingGroup->find('all', array(
-                'conditions' => array('SharingGroup.id' => array_keys($sharing_groups)),
-                'recursive' => -1,
-                'fields' => array('SharingGroup.id', 'SharingGroup.name'),
-                'order' => false
-            ));
-            foreach ($sgs as $sg) {
-                $sharing_groups[$sg['SharingGroup']['id']] = $sg;
-            }
-            foreach ($sharing_groups as $k => $sg) {
-                if (empty($sg)) {
-                    throw new NotFoundException(__('Invalid sharing group.'));
-                }
-            }
-            $this->set('sharing_groups', $sharing_groups);
-        }
-        if ($this->request->data['Object']['distribution'] == 4) {
-            $sg = $this->MispObject->SharingGroup->find('first', array(
-                'conditions' => array('SharingGroup.id' => $this->request->data['Object']['sharing_group_id']),
-                'recursive' => -1,
-                'fields' => array('SharingGroup.id', 'SharingGroup.name'),
-                'order' => false
-            ));
-            if (empty($sg)) {
-                throw new NotFoundException(__('Invalid sharing group.'));
-            }
-            $this->set('sg', $sg);
+            $sgs = $this->MispObject->SharingGroup->fetchAllAuthorised($this->Auth->user(), 'name', false, array_keys($sharing_groups));
+            $this->set('sharing_groups', $sgs);
         }
         $multiple_template_elements = Hash::extract($template['ObjectTemplateElement'], sprintf('{n}[multiple=true]'));
         $multiple_attribute_allowed = array();
@@ -157,13 +133,8 @@ class ObjectsController extends AppController
     public function add($eventId, $templateId = false, $version = false)
     {
         if (!$this->userRole['perm_modify']) {
-            throw new MethodNotAllowedException(__('You don\'t have permissions to create objects.'));
+            throw new ForbiddenException(__('You don\'t have permissions to create objects.'));
         }
-        $eventFindParams = array(
-            'recursive' => -1,
-            'fields' => array('Event.id', 'Event.uuid', 'Event.orgc_id'),
-            'conditions' => array()
-        );
 
         if (!empty($templateId) && Validation::uuid($templateId)) {
             $conditions = array('ObjectTemplate.uuid' => $templateId);
@@ -189,16 +160,12 @@ class ObjectsController extends AppController
             }
         }
         // Find the event that is to be updated
-        if (Validation::uuid($eventId)) {
-            $eventFindParams['conditions']['Event.uuid'] = $eventId;
-        } elseif (is_numeric($eventId)) {
-            $eventFindParams['conditions']['Event.id'] = $eventId;
-        } else {
+        $event = $this->MispObject->Event->fetchSimpleEvent($this->Auth->user(), $eventId);
+        if (empty($event)) {
             throw new NotFoundException(__('Invalid event.'));
         }
-        $event = $this->MispObject->Event->find('first', $eventFindParams);
-        if (empty($event) || (!$this->_isSiteAdmin() && $event['Event']['orgc_id'] != $this->Auth->user('org_id'))) {
-            throw new NotFoundException(__('Invalid event.'));
+        if (!$this->__canModifyEvent($event)) {
+            throw new ForbiddenException(__('You do not have permission to do that.'));
         }
         $eventId = $event['Event']['id'];
         if (!$this->_isRest()) {
@@ -374,16 +341,17 @@ class ObjectsController extends AppController
 
     public function edit($id, $update_template_available=false, $onlyAddNewAttribute=false)
     {
-        $id = $this->Toolbox->findIdByUuid($this->MispObject, $id);
-        $object = $this->MispObject->fetchObjects($this->Auth->user(), array('conditions' => array('Object.id' => $id)));
+        $object = $this->MispObject->fetchObjects($this->Auth->user(), array(
+            'conditions' => $this->__objectIdToConditions($id),
+        ));
         if (empty($object)) {
             throw new NotFoundException(__('Invalid object.'));
         }
         $object = $object[0];
-        if ((!$this->_isSiteAdmin() && $object['Event']['orgc_id'] != $this->Auth->user('org_id'))) {
-            throw new MethodNotAllowedException(__('Insufficient permissions to edit this object.'));
+        $event = $this->MispObject->Event->fetchSimpleEvent($this->Auth->user(), $object['Event']['id']);
+        if (!$this->__canModifyEvent($event)) {
+            throw new ForbiddenException(__('Insufficient permissions to edit this object.'));
         }
-        $event = $this->MispObject->Event->fetchEvent($this->Auth->user(), array('eventid' => $object['Event']['id'], 'metadata' => 1))[0];
         if (!$this->_isRest()) {
             $this->MispObject->Event->insertLock($this->Auth->user(), $object['Event']['id']);
         }
@@ -448,10 +416,10 @@ class ObjectsController extends AppController
             }
             $savedObject = array();
             if (is_numeric($objectToSave)) {
-                $savedObject = $this->MispObject->fetchObjects($this->Auth->user(), array('conditions' => array('Object.id' => $id)));
+                $savedObject = $this->MispObject->fetchObjects($this->Auth->user(), array('conditions' => array('Object.id' => $object['Object']['id'])));
                 if (isset($this->request->data['deleted']) && $this->request->data['deleted']) {
                     $this->MispObject->deleteObject($savedObject[0], $hard=false, $unpublish=false);
-                    $savedObject = $this->MispObject->fetchObjects($this->Auth->user(), array('conditions' => array('Object.id' => $id))); // make sure the object is deleted
+                    $savedObject = $this->MispObject->fetchObjects($this->Auth->user(), array('conditions' => array('Object.id' => $object['Object']['id']))); // make sure the object is deleted
                 }
             }
             // we pre-validate the attributes before we create an object at this point
@@ -528,35 +496,19 @@ class ObjectsController extends AppController
     // ajax edit - post a single edited field and this method will attempt to save it and return a json with the validation errors if they occur.
     public function editField($id)
     {
-        if (Validation::uuid($id)) {
-            $this->MispObject->recursive = -1;
-            $temp = $this->MispObject->findByUuid($id);
-            if ($temp == null) {
-                throw new NotFoundException(__('Invalid object'));
-            }
-            $id = $temp['Object']['id'];
-        } elseif (!is_numeric($id)) {
-            throw new NotFoundException(__('Invalid event id.'));
-        }
         if ((!$this->request->is('post') && !$this->request->is('put'))) {
             throw new MethodNotAllowedException(__('This function can only be accessed via POST or PUT'));
         }
         $object = $this->MispObject->find('first', array(
-            'conditions' => array('Object.id' => $id),
+            'conditions' => $this->__objectIdToConditions($id),
             'contain' => 'Event',
             'recursive' => -1
         ));
         if (empty($object)) {
             return $this->RestResponse->saveFailResponse('Objects', 'edit', false, 'Invalid object');
         }
-        if (!$this->_isSiteAdmin()) {
-            if ($this->MispObject->data['Event']['orgc_id'] == $this->Auth->user('org_id')
-            && (($this->userRole['perm_modify'] && $this->MispObject->data['Event']['user_id'] != $this->Auth->user('id'))
-            || $this->userRole['perm_modify_org'])) {
-                // Allow the edit
-            } else {
-                return $this->RestResponse->saveFailResponse('Objects', 'edit', false, 'Invalid attribute');
-            }
+        if (!$this->__canModifyEvent($object)) {
+            return $this->RestResponse->saveFailResponse('Objects', 'edit', false, 'You do not have permission to do that.');
         }
         $validFields = array('comment', 'distribution', 'first_seen', 'last_seen');
         $changed = false;
@@ -589,18 +541,10 @@ class ObjectsController extends AppController
         $object['Object']['timestamp'] = $date->getTimestamp();
         $object = $this->MispObject->syncObjectAndAttributeSeen($object, $forcedSeenOnElements);
         if ($this->MispObject->save($object)) {
-            $event = $this->MispObject->Event->find('first', array(
-                'recursive' => -1,
-                'fields' => array('id', 'published', 'timestamp', 'info', 'uuid'),
-                'conditions' => array(
-                    'id' => $object['Object']['event_id'],
-            )));
-            $event['Event']['timestamp'] = $date->getTimestamp();
-            $event['Event']['published'] = 0;
+            $this->MispObject->Event->unpublishEvent($object['Event']['id']);
             if ($seen_changed) {
                 $this->MispObject->Attribute->saveAttributes($object['Attribute'], $this->Auth->user());
             }
-            $this->MispObject->Event->save($event, array('fieldList' => array('published', 'timestamp', 'info')));
             return $this->RestResponse->saveSuccessResponse('Objects', 'edit', $id, false, 'Field updated');
         } else {
             return $this->RestResponse->saveFailResponse('Objects', 'edit', false, $this->MispObject->validationErrors);
@@ -665,14 +609,8 @@ class ObjectsController extends AppController
             throw new NotFoundException(__('Invalid attribute'));
         }
         $object = $object[0];
-        if (!$this->_isSiteAdmin()) {
-            if ($object['Event']['orgc_id'] == $this->Auth->user('org_id')
-            && (($this->userRole['perm_modify'] && $object['Event']['user_id'] != $this->Auth->user('id'))
-                    || $this->userRole['perm_modify_org'])) {
-                // Allow the edit
-            } else {
-                throw new NotFoundException(__('Invalid object'));
-            }
+        if (!$this->__canModifyEvent($object)) {
+            throw new NotFoundException(__('Invalid object'));
         }
         $this->layout = 'ajax';
         if ($field == 'distribution') {
@@ -686,15 +624,8 @@ class ObjectsController extends AppController
     }
 
     // Construct a template with valid object attributes to add to an object
-    public function quickFetchTemplateWithValidObjectAttributes($id) {
-        $this->MispObject->id = $id;
-        if (!$this->MispObject->exists()) {
-            if ($this->request->is('ajax')) {
-                return $this->RestResponse->saveFailResponse('Objects', 'add', false, 'Invalid object', $this->response->type());
-            } else {
-                throw new NotFoundException(__('Invalid object'));
-            }
-        }
+    public function quickFetchTemplateWithValidObjectAttributes($id)
+    {
         $fields = array('template_uuid', 'template_version', 'id');
         $params = array(
             'conditions' => array('Object.id' => $id),
@@ -771,20 +702,20 @@ class ObjectsController extends AppController
      * GET: Returns a form allowing to add a valid object attribute to an object
      * POST/PUT: Add the attribute to the object
      */
-    public function quickAddAttributeForm($id, $fieldName = null) {
+    public function quickAddAttributeForm($id, $fieldName = null)
+    {
         if ($this->request->is('GET')) {
             if (!isset($fieldName)) {
                 throw new MethodNotAllowedException('No field requested.');
             }
-            $this->MispObject->id = $id;
-            if (!$this->MispObject->exists()) {
-                throw new NotFoundException(__('Invalid object'));
-            }
             $fields = array('template_uuid', 'template_version', 'id', 'event_id');
             $params = array(
                 'conditions' => array('Object.id' => $id),
                 'fields' => $fields,
                 'flatten' => 1,
+                'contain' => array(
+                    'Event'
+                )
             );
             // fetchObjects restrict access based on user
             $object = $this->MispObject->fetchObjects($this->Auth->user(), $params);
@@ -793,6 +724,9 @@ class ObjectsController extends AppController
             } else {
                 $object = $object[0];
             }
+            if (!$this->__canModifyEvent($object)) {
+                throw new ForbiddenException(__('You do not have permission to do that.'));
+            }
             $template = $this->MispObject->ObjectTemplate->find('first', array(
                 'conditions' => array(
                     'ObjectTemplate.uuid' => $object['Object']['template_uuid'],
@@ -807,7 +741,7 @@ class ObjectsController extends AppController
                 )
             ));
             if (empty($template)) {
-                throw new NotFoundException(__('Invalid object'));
+                throw new NotFoundException(__('Invalid template'));
             }
             if (empty($template['ObjectTemplateElement'])) {
                 throw new NotFoundException(__('Invalid fields') . ' `' . h($fieldName) . '`');
@@ -841,30 +775,29 @@ class ObjectsController extends AppController
 
     public function delete($id, $hard = false)
     {
-        $id = $this->Toolbox->findIdByUuid($this->MispObject, $id);
         if (!$this->userRole['perm_modify']) {
-            throw new MethodNotAllowedException(__('You don\'t have permissions to delete objects.'));
+            throw new ForbiddenException(__('You don\'t have permissions to delete objects.'));
         }
         $object = $this->MispObject->find('first', array(
             'recursive' => -1,
-            'fields' => array('Object.id', 'Object.event_id', 'Event.id', 'Event.uuid', 'Event.orgc_id'),
-            'conditions' => array('Object.id' => $id),
+            'fields' => array('Object.id', 'Object.event_id', 'Event.id', 'Event.uuid', 'Event.orgc_id', 'Event.user_id'),
+            'conditions' => $this->__objectIdToConditions($id),
             'contain' => array(
                 'Event'
             )
         ));
         if (empty($object)) {
-            throw new NotFoundException(__('Invalid event.'));
+            throw new NotFoundException(__('Invalid object.'));
+        }
+        if (!$this->__canModifyEvent($object)) {
+            throw new ForbiddenException(__('You do not have permission to do that.'));
         }
         $eventId = $object['Event']['id'];
-        if (!$this->_isSiteAdmin() && ($object['Event']['orgc_id'] != $this->Auth->user('org_id') || !$this->userRole['perm_modify'])) {
-            throw new UnauthorizedException(__('You do not have permission to do that.'));
-        }
         if (!$this->_isRest()) {
             $this->MispObject->Event->insertLock($this->Auth->user(), $eventId);
         }
         if ($this->request->is('post') || $this->request->is('delete')) {
-            if ($this->__delete($id, $hard)) {
+            if ($this->__delete($object['Object']['id'], $hard)) {
                 $message = 'Object deleted.';
                 if ($this->request->is('ajax')) {
                     return new CakeResponse(
@@ -883,12 +816,12 @@ class ObjectsController extends AppController
                     return $this->RestResponse->saveSuccessResponse(
                         'Objects',
                         'delete',
-                        $id,
+                        $eventId,
                         $this->response->type()
                     );
                 } else {
                     $this->Flash->success($message);
-                    $this->redirect(array('controller' => 'events', 'action' => 'view', $object['Event']['id']));
+                    $this->redirect(array('controller' => 'events', 'action' => 'view', $eventId));
                 }
             } else {
                 $message = 'Object could not be deleted.';
@@ -922,7 +855,7 @@ class ObjectsController extends AppController
             if ($this->request->is('ajax') && $this->request->is('get')) {
                 $this->set('hard', $hard);
                 $this->set('id', $id);
-                $this->set('event_id', $object['Event']['id']);
+                $this->set('event_id', $eventId);
                 $this->render('ajax/delete');
             }
         }
@@ -943,9 +876,10 @@ class ObjectsController extends AppController
 
     public function view($id)
     {
-        $id = $this->Toolbox->findIdByUuid($this->MispObject, $id);
         if ($this->_isRest()) {
-            $objects = $this->MispObject->fetchObjects($this->Auth->user(), array('conditions' => array('Object.id' => $id)));
+            $objects = $this->MispObject->fetchObjects($this->Auth->user(), array(
+                'conditions' => $this->__objectIdToConditions($id),
+            ));
             if (!empty($objects)) {
                 $object = $objects[0];
                 if (!empty($object['Event'])) {
@@ -1187,16 +1121,19 @@ class ObjectsController extends AppController
         $this->set('event_id', $event_id);
     }
 
-    function groupAttributesIntoObject($event_id, $selected_template, $selected_attribute_ids='[]')
+    public function groupAttributesIntoObject($event_id, $selected_template, $selected_attribute_ids='[]')
     {
         $event = $this->MispObject->Event->find('first', array(
             'recursive' => -1,
-            'fields' => array('Event.id', 'Event.uuid', 'Event.orgc_id', 'Event.publish_timestamp'),
+            'fields' => array('Event.id', 'Event.uuid', 'Event.orgc_id', 'Event.user_id', 'Event.publish_timestamp'),
             'conditions' => array('Event.id' => $event_id)
         ));
-        if (empty($event) || (!$this->_isSiteAdmin() && $event['Event']['orgc_id'] != $this->Auth->user('org_id'))) {
+        if (empty($event)) {
             throw new NotFoundException(__('Invalid event.'));
         }
+        if (!$this->__canModifyEvent($event)) {
+            throw new ForbiddenException(__('You do not have permission to do that.'));
+        }
         $hard_delete_attribute = $event['Event']['publish_timestamp'] == 0;
         if (!$this->request->is('ajax')) {
             throw new MethodNotAllowedException(__('This action can only be reached via AJAX.'));
@@ -1215,12 +1152,7 @@ class ObjectsController extends AppController
             $selected_attribute_ids = json_decode($this->request->data['Object']['selectedAttributeIds'], true);
             $selected_object_relation_mapping = json_decode($this->request->data['Object']['selectedObjectRelationMapping'], true);
             if ($distribution == 4) {
-                $sg = $this->MispObject->SharingGroup->find('first', array(
-                    'conditions' => array('SharingGroup.id' => $sharing_group_id),
-                    'recursive' => -1,
-                    'fields' => array('SharingGroup.id', 'SharingGroup.name'),
-                    'order' => false
-                ));
+                $sg = $this->MispObject->SharingGroup->fetchSG($sharing_group_id, $this->Auth->user());
                 if (empty($sg)) {
                     throw new NotFoundException(__('Invalid sharing group.'));
                 }
@@ -1309,4 +1241,15 @@ class ObjectsController extends AppController
         }
     }
 
+    private function __objectIdToConditions($id)
+    {
+        if (is_numeric($id)) {
+            $conditions = array('Object.id' => $id);
+        } elseif (Validation::uuid($id)) {
+            $conditions = array('Object.uuid' => $id);
+        } else {
+            throw new NotFoundException(__('Invalid object ID.'));
+        }
+        return $conditions;
+    }
 }

app/Controller/OrgBlocklistsController.php

@@ -1,9 +1,9 @@
 <?php
 App::uses('AppController', 'Controller');
 
-class OrgBlacklistsController extends AppController
+class OrgBlocklistsController extends AppController
 {
-    public $components = array('Session', 'RequestHandler', 'BlackList');
+    public $components = array('Session', 'RequestHandler', 'BlockList');
 
     public function beforeFilter()
     {
@@ -11,8 +11,8 @@ class OrgBlacklistsController extends AppController
         if (!$this->_isSiteAdmin()) {
             $this->redirect('/');
         }
-        if (Configure::check('MISP.enableOrgBlacklisting') && !Configure::read('MISP.enableOrgBlacklisting') !== false) {
-            $this->Flash->info(__('Organisation Blacklisting is not currently enabled on this instance.'));
+        if (Configure::check('MISP.enableOrgBlocklisting') && !Configure::read('MISP.enableOrgBlocklisting') !== false) {
+            $this->Flash->info(__('Organisation BlockListing is not currently enabled on this instance.'));
             $this->redirect('/');
         }
     }
@@ -21,27 +21,27 @@ class OrgBlacklistsController extends AppController
             'limit' => 60,
             'maxLimit' => 9999, // LATER we will bump here on a problem once we have more than 9999 events <- no we won't, this is the max a user van view/page.
             'order' => array(
-                    'OrgBlacklist.created' => 'DESC'
+                    'OrgBlocklist.created' => 'DESC'
             ),
     );
 
     public function index()
     {
-        $this->BlackList->index($this->_isRest());
+        return $this->BlockList->index($this->_isRest());
     }
 
     public function add()
     {
-        $this->BlackList->add($this->_isRest());
+        return $this->BlockList->add($this->_isRest());
     }
 
     public function edit($id)
     {
-        $this->BlackList->edit($this->_isRest(), $id);
+        return $this->BlockList->edit($this->_isRest(), $id);
     }
 
     public function delete($id)
     {
-        $this->BlackList->delete($this->_isRest(), $id);
+        return $this->BlockList->delete($this->_isRest(), $id);
     }
 }

app/Controller/PostsController.php

@@ -49,28 +49,21 @@ class PostsController extends AppController
         switch ($target_type) {
             case 'event':
                 $this->loadModel('Event');
-                $this->Event->recursive = -1;
-                $this->Event->read(null, $target_id);
-                $eventDiscussionTitle = __('Discussion about Event #') . $this->Event->data['Event']['id'] . ' (' . $this->Event->data['Event']['info'] . ')';
-                if (!$this->Event->exists()) {
+                $event = $this->Event->fetchSimpleEvent($this->Auth->user(), $target_id);
+                if (!$event) {
                     throw new NotFoundException(__('Invalid event'));
                 }
-                if (!$this->_isSiteAdmin()) {
-                    if ($this->Event->data['Event']['distribution'] == 0 && $this->Event->data['Event']['org_id'] != $this->Auth->user('org_id')) {
-                        throw new MethodNotAllowedException(__('You don\'t have permission to do that.'));
-                    }
-                }
-                $thread = $this->Thread->find('first', array('conditions' => array('event_id' => $target_id)));
+                $eventDiscussionTitle = __('Discussion about Event #%s (%s)', $event['Event']['id'], $event['Event']['info']);
+                $thread = $this->Thread->find('first', array('conditions' => array('event_id' => $event['Event']['id'])));
                 $title = $eventDiscussionTitle;
                 if (isset($thread['Thread']['id'])) {
                     $target_thread_id = $thread['Thread']['id'];
                 } else {
                     $target_thread_id = null;
                 }
-                $distribution = $this->Event->data['Event']['distribution'];
-                $sgid = $this->Event->data['Event']['sharing_group_id'];
-                $org = $this->Event->data['Event']['org_id'];
-                $event_id = $this->Event->data['Event']['id'];
+                $distribution = $event['Event']['distribution'];
+                $sgid = $event['Event']['sharing_group_id'];
+                $event_id = $event['Event']['id'];
             break;
             case 'thread':
                 $target_thread_id = $target_id;
@@ -208,7 +201,7 @@ class PostsController extends AppController
             throw new NotFoundException(__('Invalid post'));
         }
         if (!$this->_isSiteAdmin() && $this->Auth->user('id') != $post['Post']['user_id']) {
-            throw new MethodNotAllowedException(__('This is not your event.'));
+            throw new MethodNotAllowedException(__('This is not your post.'));
         }
 
         if ($this->request->is('post') || $this->request->is('put')) {

app/Controller/ServersController.php

@@ -1,6 +1,7 @@
 <?php
 App::uses('AppController', 'Controller');
 App::uses('Xml', 'Utility');
+App::uses('AttachmentTool', 'Tools');
 
 class ServersController extends AppController
 {
@@ -1022,9 +1023,9 @@ class ServersController extends AppController
                 $php_ini = php_ini_loaded_file();
                 $this->set('php_ini', $php_ini);
 
-                $malwareTool = new MalwareTool();
+                $attachmentTool = new AttachmentTool();
                 try {
-                    $advanced_attachments = $malwareTool->checkAdvancedExtractionStatus($this->Server->getPythonVersion());
+                    $advanced_attachments = $attachmentTool->checkAdvancedExtractionStatus($this->Server->getPythonVersion());
                 } catch (Exception $e) {
                     $this->log($e->getMessage(), LOG_NOTICE);
                     $advanced_attachments = false;
@@ -1265,6 +1266,90 @@ class ServersController extends AppController
         }
     }
 
+    public function idTranslator() {
+
+        // The id translation feature is limited to people from the host org
+        if (!$this->_isSiteAdmin() && $this->Auth->user('org_id') != Configure::read('MISP.host_org_id')) {
+            throw new MethodNotAllowedException(__('You don\'t have the required privileges to do that.'));
+        }
+
+        //We retrieve the list of remote servers that we can query
+        $options = array();
+        $options['conditions'] = array("pull" => true);
+        $servers = $this->Server->find('all', $options);
+
+        // We generate the list of servers for the dropdown
+        $displayServers = array();
+        foreach($servers as $s) {
+            $displayServers[] = array('name' => $s['Server']['name'],
+                                      'value' => $s['Server']['id']);
+        }
+        $this->set('servers', $displayServers);
+
+        if ($this->request->is('post')) {
+            $remote_events = array();
+            if(!empty($this->request->data['Event']['uuid']) &&  $this->request->data['Event']['local'] == "local") {
+                $local_event = $this->Event->fetchSimpleEvent($this->Auth->user(), $this->request->data['Event']['uuid']);
+            } else if (!empty($this->request->data['Event']['uuid']) && $this->request->data['Event']['local'] == "remote" && !empty($this->request->data['Server']['id'])) {
+                //We check on the remote server for any event with this id and try to find a match locally
+                $conditions = array('AND' => array('Server.id' => $this->request->data['Server']['id'], 'Server.pull' => true));
+                $remote_server = $this->Server->find('first', array('conditions' => $conditions));
+                if(!empty($remote_server)) {
+                    try {
+                        $remote_event = $this->Event->downloadEventFromServer($this->request->data['Event']['uuid'], $remote_server, null, true);
+                    } catch (Exception $e) {
+                        $error_msg = __("Issue while contacting the remote server to retrieve event information");
+                        $this->logException($error_msg, $e);
+                        $this->Flash->error($error_msg);
+                        return;
+                    }
+
+                    $local_event = $this->Event->fetchSimpleEvent($this->Auth->user(), $remote_event[0]['uuid']);
+                    //we record it to avoid re-querying the same server in the 2nd phase
+                    if(!empty($local_event)) {
+                        $remote_events[] = array(
+                            "server_id" => $remote_server['Server']['id'],
+                            "server_name" => $remote_server['Server']['name'],
+                            "url" => $remote_server['Server']['url']."/events/view/".$remote_event[0]['id'],
+                            "remote_id" => $remote_event[0]['id']
+                        );
+                    }
+                }
+            }
+            if(empty($local_event)) {
+                $this->Flash->error( __("This event could not be found or you don't have permissions to see it."));
+                return;
+            } else {
+                $this->Flash->success(__('The event has been found.'));
+            }
+
+            // In the second phase, we query all configured sync servers to get their info on the event
+            foreach($servers as $s) {
+                // We check if the server was not already contacted in phase 1
+                if(count($remote_events) > 0 && $remote_events[0]['server_id'] == $s['Server']['id']) {
+                    continue;
+                }
+
+                try {
+                    $remote_event = $this->Event->downloadEventFromServer($local_event['Event']['uuid'], $s, null, true);
+                    $remote_event_id = $remote_event[0]['id'];
+                } catch (Exception $e) {
+                    $this->logException("Couldn't download event from server", $e);
+                    $remote_event_id = null;
+                }
+                $remote_events[] = array(
+                    "server_id" => $s['Server']['id'],
+                    "server_name" => $s['Server']['name'],
+                    "url" => isset($remote_event_id) ? $s['Server']['url']."/events/view/".$remote_event_id : $s['Server']['url'],
+                    "remote_id" => isset($remote_event_id) ? $remote_event_id : false
+                );
+            }
+
+            $this->set('local_event', $local_event);
+            $this->set('remote_events', $remote_events);
+        }
+    }
+
     public function getSubmodulesStatus() {
         if (!$this->_isSiteAdmin()) {
             throw new MethodNotAllowedException();
@@ -1744,7 +1829,7 @@ class ServersController extends AppController
             'recommendBackup' => false,
             'exitOnError' => false,
             'requirements' => '',
-            'url' => '/'
+            'url' => $this->baseurl . '/'
         );
         foreach($actions as $id => $action) {
             foreach($default_fields as $field => $value) {

app/Controller/ShadowAttributesController.php

@@ -2,7 +2,11 @@
 App::uses('AppController', 'Controller');
 App::uses('Folder', 'Utility');
 App::uses('File', 'Utility');
+App::uses('AttachmentTool', 'Tools');
 
+/**
+ * @property ShadowAttribute $ShadowAttribute
+ */
 class ShadowAttributesController extends AppController
 {
     public $components = array('Acl', 'Security', 'RequestHandler', 'Email');
@@ -23,32 +27,16 @@ class ShadowAttributesController extends AppController
         // convert uuid to id if present in the url, and overwrite id field
         if (isset($this->params->query['uuid'])) {
             $params = array(
-                    'conditions' => array('ShadowAttribute.uuid' => $this->params->query['uuid']),
-                    'recursive' => 0,
-                    'fields' => 'ShadowAttribute.id'
-                    );
+                'conditions' => array('ShadowAttribute.uuid' => $this->params->query['uuid']),
+                'recursive' => 0,
+                'fields' => 'ShadowAttribute.id'
+            );
             $result = $this->ShadowAttribute->find('first', $params);
             if (isset($result['ShadowAttribute']) && isset($result['ShadowAttribute']['id'])) {
                 $id = $result['ShadowAttribute']['id'];
                 $this->params->addParams(array('pass' => array($id))); // FIXME find better way to change id variable if uuid is found. params->url and params->here is not modified accordingly now
             }
         }
-
-        // if not admin or own org, check private as well..
-        if (!$this->_isSiteAdmin()) {
-            $this->paginate = Set::merge($this->paginate, array(
-            'conditions' =>
-                    array('OR' =>
-                            array(
-                                'Event.org =' => $this->Auth->user('org_id'),
-                                'AND' => array(
-                                    'ShadowAttribute.org =' => $this->Auth->user('org_id'),
-                                    'Event.distribution >' => 0,
-                                    Configure::read('MISP.unpublishedprivate') ? array('Event.published =' => 1) : array(),
-                                ),
-                            )
-            )));
-        }
     }
 
     private function __accept($id)
@@ -70,7 +58,7 @@ class ShadowAttributesController extends AppController
         }
         $this->ShadowAttribute->publishKafkaNotification('shadow_attribute', $shadow, 'accept');
         $shadow = $shadow['ShadowAttribute'];
-        if ($this->ShadowAttribute->typeIsAttachment($shadow['type'])) {
+        if ($this->ShadowAttribute->typeIsAttachment($shadow['type']) && !$shadow['proposal_to_delete']) {
             $encodedFile = $this->ShadowAttribute->base64EncodeAttachment($shadow);
             $shadow['data'] = $encodedFile;
         }
@@ -80,18 +68,16 @@ class ShadowAttributesController extends AppController
             $this->Attribute->contain = 'Event';
             $activeAttribute = $this->Attribute->findByUuid($shadow['uuid']);
 
-            // Send those away that shouldn't be able to see this
-            if (!$this->_isSiteAdmin()) {
-                if ($activeAttribute['Event']['orgc_id'] != $this->Auth->user('org_id') || (!$this->userRole['perm_modify'])) {
-                    if ($this->_isRest()) {
-                        return array('false' => true, 'errors' => 'Proposal not found or you are not authorised to accept it.');
-                    } else {
-                        $this->Flash->error('You don\'t have permission to do that');
-                        $this->redirect(array('controller' => 'events', 'action' => 'view', $shadow['event_id']));
-                    }
+            // Send those away that shouldn't be able to edit this
+            if (!$this->__canModifyEvent($activeAttribute)) {
+                if ($this->_isRest()) {
+                    return array('false' => true, 'errors' => 'Proposal not found or you are not authorised to accept it.');
+                } else {
+                    $this->Flash->error('You don\'t have permission to do that');
+                    $this->redirect(array('controller' => 'events', 'action' => 'view', $shadow['event_id']));
                 }
             }
-            $date = new DateTime();
+
             if (isset($shadow['proposal_to_delete']) && $shadow['proposal_to_delete']) {
                 $this->Attribute->delete($activeAttribute['Attribute']['id']);
             } else {
@@ -100,6 +86,7 @@ class ShadowAttributesController extends AppController
                 foreach ($fieldsToUpdate as $f) {
                     $activeAttribute['Attribute'][$f] = $shadow[$f];
                 }
+                $date = new DateTime();
                 $activeAttribute['Attribute']['timestamp'] = $date->getTimestamp();
                 $this->Attribute->save($activeAttribute['Attribute']);
             }
@@ -133,11 +120,9 @@ class ShadowAttributesController extends AppController
             $this->Event->recursive = -1;
             $event = $this->Event->read(null, $shadow['event_id']);
 
-            if (!$this->_isSiteAdmin()) {
-                if (($event['Event']['orgc_id'] != $this->Auth->user('org_id')) || (!$this->userRole['perm_modify'])) {
-                    $this->Flash->error('You don\'t have permission to do that');
-                    $this->redirect(array('controller' => 'events', 'action' => 'index'));
-                }
+            if (!$this->__canModifyEvent($event)) {
+                $this->Flash->error('You don\'t have permission to do that');
+                $this->redirect(array('controller' => 'events', 'action' => 'index'));
             }
             if (!$this->_isRest()) {
                 $this->Event->insertLock($this->Auth->user(), $event['Event']['id']);
@@ -186,7 +171,7 @@ class ShadowAttributesController extends AppController
                 $response['check_publish'] = true;
                 $this->set('name', $response['success']);
                 $this->set('message', $response['success']);
-                $this->set('url', '/shadow_attributes/accept/' . $id);
+                $this->set('url', $this->baseurl . '/shadow_attributes/accept/' . $id);
                 $this->set('_serialize', array('name', 'message', 'url'));
             } else {
                 throw new MethodNotAllowedException($response['errors']);
@@ -203,6 +188,7 @@ class ShadowAttributesController extends AppController
                 'first',
                 array(
                     'recursive' => -1,
+                    'contain' => 'Event',
                     'conditions' => array(
                         'ShadowAttribute.id' => $id,
                         'deleted' => 0
@@ -212,33 +198,18 @@ class ShadowAttributesController extends AppController
         if (empty($sa)) {
             return false;
         }
-        $this->ShadowAttribute->publishKafkaNotification('shadow_attribute', $sa, 'discard');
-        $eventId = $sa['ShadowAttribute']['event_id'];
-        $this->loadModel('Event');
-        $this->Event->Behaviors->detach('SysLogLogable.SysLogLogable');
-        $this->Event->recursive = -1;
-        $this->Event->id = $eventId;
-        $this->Event->read();
-        // Send those away that shouldn't be able to see this
-        if (!$this->_isSiteAdmin()) {
-            if ((($this->Event->data['Event']['orgc_id'] != $this->Auth->user('org_id')) && ($this->Auth->user('org_id') != $sa['ShadowAttribute']['org_id'])) || (!$this->userRole['perm_modify'])) {
-                return false;
-            }
+        // Just auth of proposal or user that can edit event can discard proposal.
+        if (!$this->__canModifyEvent($sa) && $this->Auth->user('email') !== $sa['ShadowAttribute']['email']) {
+            return false;
         }
+        $this->ShadowAttribute->publishKafkaNotification('shadow_attribute', $sa, 'discard');
         if ($this->ShadowAttribute->setDeleted($id)) {
-            if ($this->Auth->user('org_id') == $this->Event->data['Event']['orgc_id']) {
-                $this->ShadowAttribute->setProposalLock($eventId, false);
+            if ($this->Auth->user('org_id') == $sa['Event']['orgc_id']) {
+                $this->ShadowAttribute->setProposalLock($sa['Event']['id'], false);
             }
+            $logTitle = "Proposal ({$sa['ShadowAttribute']['id']}) of {$sa['ShadowAttribute']['org_id']} discarded - {$sa['ShadowAttribute']['category']}/{$sa['ShadowAttribute']['type']} {$sa['ShadowAttribute']['value']}";
             $this->Log = ClassRegistry::init('Log');
-            $this->Log->create();
-            $this->Log->save(array(
-                        'org_id' => $this->Auth->user('org_id'),
-                        'model' => 'ShadowAttribute',
-                        'model_id' => $id,
-                        'email' => $this->Auth->user('email'),
-                        'action' => 'discard',
-                        'title' => 'Proposal (' . $sa['ShadowAttribute']['id'] . ') of ' . $sa['ShadowAttribute']['org_id'] . ' discarded - ' . $sa['ShadowAttribute']['category'] . '/' . $sa['ShadowAttribute']['type'] . ' ' . $sa['ShadowAttribute']['value'],
-                ));
+            $this->Log->createLogEntry($this->Auth->user(), 'discard', 'ShadowAttribute', $id, $logTitle);
             return true;
         }
         return false;
@@ -252,7 +223,7 @@ class ShadowAttributesController extends AppController
                 if ($this->_isRest()) {
                     $this->set('name', 'Proposal discarded.');
                     $this->set('message', 'Proposal discarded.');
-                    $this->set('url', '/shadow_attributes/discard/' . $id);
+                    $this->set('url', $this->baseurl . '/shadow_attributes/discard/' . $id);
                     $this->set('_serialize', array('name', 'message', 'url'));
                 } else {
                     $this->autoRender = false;
@@ -260,7 +231,7 @@ class ShadowAttributesController extends AppController
                 }
             } else {
                 if ($this->_isRest()) {
-                    throw new MethodNotAllowedException(__('Could not discard proposal.'));
+                    throw new InternalErrorException(__('Could not discard proposal.'));
                 } else {
                     $this->autoRender = false;
                     return new CakeResponse(array('body'=> json_encode(array('false' => true, 'errors' => 'Could not discard proposal.')), 'status'=>200, 'type' => 'json'));
@@ -290,11 +261,10 @@ class ShadowAttributesController extends AppController
         } else {
             $this->set('ajax', false);
         }
-        $event = $this->ShadowAttribute->Event->fetchEvent($this->Auth->user(), array('eventid' => $eventId));
+        $event = $this->ShadowAttribute->Event->fetchSimpleEvent($this->Auth->user(), $eventId);
         if (empty($event)) {
             throw new NotFoundException(__('Invalid Event'));
         }
-        $event = $event[0];
 
         if ($this->request->is('post')) {
             if (isset($this->request->data['request'])) {
@@ -317,9 +287,9 @@ class ShadowAttributesController extends AppController
             // TODO change behavior attachment options - this is bad ... it should rather by a messagebox or should be filtered out on the view level
             if (isset($this->request->data['ShadowAttribute']['type']) && $this->ShadowAttribute->typeIsAttachment($this->request->data['ShadowAttribute']['type']) && !$this->_isRest()) {
                 $this->Flash->error(__('Attribute has not been added: attachments are added by "Add attachment" button', true), 'default', array(), 'error');
-                $this->redirect(array('controller' => 'events', 'action' => 'view', $eventId));
+                $this->redirect(array('controller' => 'events', 'action' => 'view', $event['Event']['id']));
             }
-            $this->request->data['ShadowAttribute']['event_id'] = $eventId;
+            $this->request->data['ShadowAttribute']['event_id'] = $event['Event']['id'];
             //
             // multiple attributes in batch import
             //
@@ -373,7 +343,7 @@ class ShadowAttributesController extends AppController
                     if ($successes) {
                         // list the ones that succeeded
                         $emailResult = "";
-                        if (!$this->ShadowAttribute->sendProposalAlertEmail($eventId) === false) {
+                        if (!$this->ShadowAttribute->sendProposalAlertEmail($event['Event']['id']) === false) {
                             $emailResult = " but nobody from the owner organisation could be notified by e-mail.";
                         }
                         $this->Flash->success(__('The lines' . $successes . ' have been saved' . $emailResult, true));
@@ -387,7 +357,6 @@ class ShadowAttributesController extends AppController
                 //
                 // create the attribute
                 $this->ShadowAttribute->create();
-                $savedId = $this->ShadowAttribute->getID();
                 $this->request->data['ShadowAttribute']['email'] = $this->Auth->user('email');
                 $this->request->data['ShadowAttribute']['org_id'] = $this->Auth->user('org_id');
                 $this->request->data['ShadowAttribute']['event_uuid'] = $event['Event']['uuid'];
@@ -436,9 +405,9 @@ class ShadowAttributesController extends AppController
             }
         } else {
             // set the event_id in the form
-            $this->request->data['ShadowAttribute']['event_id'] = $eventId;
+            $this->request->data['ShadowAttribute']['event_id'] = $event['Event']['id'];
         }
-        $this->set('event_id', $eventId);
+        $this->set('event_id', $event['Event']['id']);
         // combobox for types
         $types = array_keys($this->ShadowAttribute->typeDefinitions);
         foreach ($types as $key => $value) {
@@ -464,36 +433,32 @@ class ShadowAttributesController extends AppController
         $this->set('categoryDefinitions', $this->ShadowAttribute->categoryDefinitions);
     }
 
-    public function download($id = null)
+    public function download($id)
     {
-        $this->ShadowAttribute->id = $id;
-        if (!$this->ShadowAttribute->exists()) {
-            throw new NotFoundException(__('Invalid Proposal'));
-        }
+        $conditions = $this->ShadowAttribute->buildConditions($this->Auth->user());
+        $conditions['ShadowAttribute.id'] = $id;
+        $conditions['ShadowAttribute.deleted'] = 0;
+
         $sa = $this->ShadowAttribute->find('first', array(
             'recursive' => -1,
-            'contain' => array('Event' => array('fields' => array('Event.org_id', 'Event.distribution', 'Event.id'))),
-            'conditions' => array('ShadowAttribute.id' => $id)
+            'contain' => ['Event', 'Attribute'], // required because of conditions
+            'conditions' => $conditions,
         ));
-        if (!$this->ShadowAttribute->Event->checkIfAuthorised($this->Auth->user(), $sa['Event']['id'])) {
-            throw new UnauthorizedException(__('You do not have the permission to view this event.'));
+        if (!$sa) {
+            throw new NotFoundException(__('Invalid Proposal'));
         }
         $this->__downloadAttachment($sa['ShadowAttribute']);
     }
 
-    private function __downloadAttachment($shadowAttribute)
+    private function __downloadAttachment(array $shadowAttribute)
     {
-        $attachments_dir = Configure::read('MISP.attachments_dir');
-        if (empty($attachments_dir)) {
-            $attachments_dir = $this->ShadowAttribute->getDefaultAttachments_dir();
-        }
-        $path = $attachments_dir . DS . 'shadow' . DS . $shadowAttribute['event_id'] . DS;
-        $file = $shadowAttribute['id'];
-        if ('attachment' == $shadowAttribute['type']) {
+        $file = $this->ShadowAttribute->getAttachmentFile($shadowAttribute);
+
+        if ('attachment' === $shadowAttribute['type']) {
             $filename = $shadowAttribute['value'];
             $fileExt = pathinfo($filename, PATHINFO_EXTENSION);
             $filename = substr($filename, 0, strlen($filename) - strlen($fileExt) - 1);
-        } elseif ('malware-sample' == $shadowAttribute['type']) {
+        } elseif ('malware-sample' === $shadowAttribute['type']) {
             $filenameHash = explode('|', $shadowAttribute['value']);
             $filename = substr($filenameHash[0], strrpos($filenameHash[0], '\\'));
             $fileExt = "zip";
@@ -502,17 +467,15 @@ class ShadowAttributesController extends AppController
         }
         $this->autoRender = false;
         $this->response->type($fileExt);
-        $this->response->file($path . $file, array('download' => true, 'name' => $filename . '.' . $fileExt));
+        $this->response->file($file->path, array('download' => true, 'name' => $filename . '.' . $fileExt));
     }
 
     public function add_attachment($eventId = null)
     {
-        $event = $this->ShadowAttribute->Event->fetchEvent($this->Auth->user(), array('eventid' => $eventId));
+        $event = $this->ShadowAttribute->Event->fetchSimpleEvent($this->Auth->user(), $eventId);
         if (empty($event)) {
             throw new NotFoundException(__('Invalid Event'));
         }
-        $event = $event[0];
-
         if ($this->request->is('post')) {
             // Check if there were problems with the file upload
             // only keep the last part of the filename, this should prevent directory attacks
@@ -526,7 +489,7 @@ class ShadowAttributesController extends AppController
                     throw new InternalErrorException(__('PHP says file was not uploaded. Are you attacking me?'));
                 }
             } else {
-                $this->Flash->error(__('There was a problem to upload the file.', true));
+                $this->Flash->error(__('There was a problem to upload the file.'));
                 $this->redirect(array('controller' => 'events', 'action' => 'view', $this->request->data['ShadowAttribute']['event_id']));
             }
 
@@ -536,7 +499,7 @@ class ShadowAttributesController extends AppController
             if ($this->request->data['ShadowAttribute']['malware']) {
                 $result = $this->ShadowAttribute->Event->Attribute->handleMaliciousBase64($this->request->data['ShadowAttribute']['event_id'], $filename, base64_encode($tmpfile->read()), array_keys($hashes));
                 if (!$result['success']) {
-                    $this->Flash->error(__('There was a problem to upload the file.', true), 'default', array(), 'error');
+                    $this->Flash->error(__('There was a problem to upload the file.'), 'default', array(), 'error');
                     $this->redirect(array('controller' => 'events', 'action' => 'view', $this->request->data['ShadowAttribute']['event_id']));
                 }
                 foreach ($hashes as $hash => $typeName) {
@@ -604,7 +567,7 @@ class ShadowAttributesController extends AppController
             $this->redirect(array('controller' => 'events', 'action' => 'view', $this->request->data['ShadowAttribute']['event_id']));
         } else {
             // set the event_id in the form
-            $this->request->data['ShadowAttribute']['event_id'] = $eventId;
+            $this->request->data['ShadowAttribute']['event_id'] = $event['Event']['id'];
         }
 
         // combobox for categories
@@ -648,14 +611,13 @@ class ShadowAttributesController extends AppController
     // if any of these fields is set, it will create a proposal
     public function edit($id = null)
     {
-        $id = $this->Toolbox->findIdByUuid($this->ShadowAttribute->Event->Attribute, $id);
         $existingAttribute = $this->ShadowAttribute->Event->Attribute->fetchAttributes($this->Auth->user(), array(
                 'contain' => array('Event' => array('fields' => array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.distribution', 'Event.uuid'))),
-                'conditions' => array('Attribute.id' => $id),
+                'conditions' => $this->__attributeIdToConditions($id),
                 'flatten' => 1
         ));
         if (empty($existingAttribute)) {
-            throw new MethodNotAllowedException(__('Invalid Attribute.'));
+            throw new NotFoundException(__('Invalid Attribute.'));
         }
         $existingAttribute = $existingAttribute[0];
 
@@ -742,7 +704,7 @@ class ShadowAttributesController extends AppController
                     foreach ($this->ShadowAttribute->validationErrors as $k => $v) {
                         $message .= '[' . $k . ']: ' . $v[0] . PHP_EOL;
                     }
-                    throw new NotFoundException(__('Could not save the proposal. Errors: %s', $message));
+                    throw new InternalErrorException(__('Could not save the proposal. Errors: %s', $message));
                 } else {
                     $this->Flash->error(__('The ShadowAttribute could not be saved. Please, try again.'));
                 }
@@ -799,17 +761,9 @@ class ShadowAttributesController extends AppController
 
     public function delete($id)
     {
-        if (is_numeric($id)) {
-            $conditions = ['Attribute.id' => $id];
-        } else if (Validation::uuid($id)) {
-            $conditions = ['Attribute.uuid' => $id];
-        } else {
-            throw new NotFoundException(__('Invalid attribute'));
-        }
-
         $existingAttribute = $this->ShadowAttribute->Event->Attribute->fetchAttributes(
             $this->Auth->user(),
-            array('conditions' => $conditions, 'flatten' => true)
+            array('conditions' => $this->__attributeIdToConditions($id), 'flatten' => true)
         );
         if ($this->request->is('post')) {
             if (empty($existingAttribute)) {
@@ -847,7 +801,7 @@ class ShadowAttributesController extends AppController
                 throw new NotFoundException(__('Invalid attribute'));
             }
             $existingAttribute = $existingAttribute[0];
-            $this->set('id', $id);
+            $this->set('id', $existingAttribute['Attribute']['id']);
             $this->set('event_id', $existingAttribute['Attribute']['event_id']);
             $this->render('ajax/deletionProposalConfirmationForm');
         }
@@ -855,40 +809,21 @@ class ShadowAttributesController extends AppController
 
     public function view($id)
     {
-        $distConditions = array();
-        if (!$this->_isSiteAdmin()) {
-            $distConditions = array(
-                    'OR' => array(
-                            'Event.distribution >' => 0,
-                            'Event.org_id' => $this->Auth->user('org_id'),
-                            'Event.orgc_id' => $this->Auth->user('org_id'),
-                    ),
-            );
-        }
+        $conditions = $this->ShadowAttribute->buildConditions($this->Auth->user());
+        $conditions['ShadowAttribute.id'] = $id;
+        $conditions['ShadowAttribute.deleted'] = 0;
+
         $sa = $this->ShadowAttribute->find('first', array(
-                'recursive' => -1,
-                'contain' => 'Event',
-                'fields' => array(
-                    'ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.first_seen', 'ShadowAttribute.last_seen',
-                    'Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.distribution', 'Event.uuid'
-                ),
-                'conditions' => array('AND' => array('ShadowAttribute.id' => $id, $distConditions, 'ShadowAttribute.deleted' => 0))
+            'recursive' => -1,
+            'contain' => ['Event', 'Attribute'], // required because of conditions
+            'fields' => array(
+                'ShadowAttribute.id', 'ShadowAttribute.old_id', 'ShadowAttribute.event_id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.uuid', 'ShadowAttribute.to_ids', 'ShadowAttribute.value', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.first_seen', 'ShadowAttribute.last_seen',
+            ),
+            'conditions' => $conditions,
         ));
         if (empty($sa)) {
             throw new NotFoundException(__('Invalid proposal.'));
         }
-        if (!$this->_isSiteAdmin()) {
-            if ($sa['ShadowAttribute']['old_id'] != 0 && $sa['Event']['org_id'] != $this->Auth->user('org_id') && $sa['Event']['orgc_id'] != $this->Auth->user('org_id')) {
-                $a = $this->ShadowAttribute->Event->Attribute->find('first', array(
-                    'recursive' => -1,
-                    'fields' => array('Attribute.id', 'Attribute.distribution'),
-                    'conditions' => array('Attribute.id' => $sa['ShadowAttribute']['old_id'], 'Attribute.distribution >' => 0)
-                ));
-                if (empty($a)) {
-                    throw new NotFoundException(__('Invalid proposal.'));
-                }
-            }
-        }
         $this->set('ShadowAttribute', $sa['ShadowAttribute']);
         $this->set('_serialize', array('ShadowAttribute'));
     }
@@ -902,7 +837,7 @@ class ShadowAttributesController extends AppController
             $all = 1;
         }
         $eventId = $this->Toolbox->findIdByUuid($this->ShadowAttribute->Event, $eventId, true);
-        if ($eventId && is_numeric($eventId)) {
+        if ($eventId) {
             $conditions['ShadowAttribute.event_id'] = $eventId;
         }
         $temp = $this->ShadowAttribute->buildConditions($this->Auth->user());
@@ -914,22 +849,23 @@ class ShadowAttributesController extends AppController
             $conditions['AND'][] = array('Event.orgc_id' =>$this->Auth->user('org_id'));
         }
         if (!empty($this->request['named']['searchall'])) {
+            $term = '%' . strtolower(trim($this->request['named']['searchall'])) . '%';
             $conditions['AND'][] = array('OR' => array(
-                'LOWER(ShadowAttribute.value1) LIKE' => '%' . strtolower(trim($this->request['named']['searchall'])) . '%',
-                'LOWER(ShadowAttribute.value2) LIKE' => '%' . strtolower(trim($this->request['named']['searchall'])) . '%',
-                'LOWER(ShadowAttribute.comment) LIKE' => '%' . strtolower(trim($this->request['named']['searchall'])) . '%',
-                'LOWER(Event.info) LIKE' => '%' . strtolower(trim($this->request['named']['searchall'])) . '%',
-                'LOWER(Org.name) LIKE' => '%' . strtolower(trim($this->request['named']['searchall'])) . '%',
-                'LOWER(Org.uuid) LIKE' => '%' . strtolower(trim($this->request['named']['searchall'])) . '%',
-                'LOWER(ShadowAttribute.uuid) LIKE' => '%' . strtolower(trim($this->request['named']['searchall'])) . '%',
-                'LOWER(Event.uuid) LIKE' => '%' . strtolower(trim($this->request['named']['searchall'])) . '%'
+                'LOWER(ShadowAttribute.value1) LIKE' => $term,
+                'LOWER(ShadowAttribute.value2) LIKE' => $term,
+                'LOWER(ShadowAttribute.comment) LIKE' => $term,
+                'LOWER(Event.info) LIKE' => $term,
+                'LOWER(Org.name) LIKE' => $term,
+                'LOWER(Org.uuid) LIKE' => $term,
+                'LOWER(ShadowAttribute.uuid) LIKE' => $term,
+                'LOWER(Event.uuid) LIKE' => $term,
             ));
         }
         if (isset($this->request['named']['deleted'])) {
             $conditions['AND'][] = array(
                 'ShadowAttribute.deleted' => $this->request['named']['deleted']
             );
-        }
+       }
         if (!empty($this->request['named']['timestamp'])) {
             $conditions['AND'][] = array(
                 'ShadowAttribute.timestamp >=' => $this->request['named']['timestamp']
@@ -944,7 +880,7 @@ class ShadowAttributesController extends AppController
             'contain' => array(
                     'Event' => array(
                             'fields' => array('id', 'org_id', 'info', 'orgc_id', 'uuid'),
-                            'Orgc' => array('fields' => array('Orgc.name'))
+                            'Orgc' => array('fields' => array('Orgc.name', 'Orgc.id', 'Orgc.uuid'))
                     ),
                     'Org' => array(
                         'fields' => array('name', 'uuid'),
@@ -1012,21 +948,24 @@ class ShadowAttributesController extends AppController
                     'recursive' => -1,
                     'fields' => array('id', 'orgc_id', 'user_id')
             ));
-            if ($event['Event']['orgc_id'] != $this->Auth->user('org_id') || (!$this->userRole['perm_modify_org'] && !($this->userRole['perm_modify'] && $event['Event']['user_id'] == $this->Auth->user('id')))) {
+            if (!$event) {
+                return new CakeResponse(array('body'=> json_encode(array('false' => true, 'errors' => 'Invalid event.')), 'status' => 200, 'type' => 'json'));
+            }
+            if (!$this->__canModifyEvent($event)) {
                 return new CakeResponse(array('body'=> json_encode(array('false' => true, 'errors' => 'You don\'t have permission to do that.')), 'status'=>200, 'type' => 'json'));
             }
         }
 
         // find all attributes from the ID list that also match the provided event ID.
-        $shadowAttributes = $this->ShadowAttribute->find('all', array(
+        $shadowAttributes = $this->ShadowAttribute->find('list', array(
                 'recursive' => -1,
                 'conditions' => array('id' => $ids, 'event_id' => $id),
-                'fields' => array('id', 'event_id')
+                'fields' => array('id')
         ));
         $successes = array();
-        foreach ($shadowAttributes as $a) {
-            if ($this->discard($a['ShadowAttribute']['id'])) {
-                $successes[] = $a['ShadowAttribute']['id'];
+        foreach ($shadowAttributes as $id) {
+            if ($this->__discard($id)) {
+                $successes[] = $id;
             }
         }
         $fails = array_diff($ids, $successes);
@@ -1053,22 +992,25 @@ class ShadowAttributesController extends AppController
                     'recursive' => -1,
                     'fields' => array('id', 'orgc_id', 'user_id')
             ));
-            if ($event['Event']['orgc_id'] != $this->Auth->user('org_id') || (!$this->userRole['perm_modify_org'] && !($this->userRole['perm_modify'] && $event['Event']['user_id'] == $this->Auth->user('id')))) {
+            if (!$event) {
+                return new CakeResponse(array('body'=> json_encode(array('false' => true, 'errors' => 'Invalid event.')), 'status' => 200, 'type' => 'json'));
+            }
+            if (!$this->__canModifyEvent($event)) {
                 return new CakeResponse(array('body'=> json_encode(array('false' => true, 'errors' => 'You don\'t have permission to do that.')), 'status'=>200, 'type' => 'json'));
             }
         }
 
         // find all attributes from the ID list that also match the provided event ID.
-        $shadowAttributes = $this->ShadowAttribute->find('all', array(
+        $shadowAttributes = $this->ShadowAttribute->find('list', array(
                 'recursive' => -1,
                 'conditions' => array('id' => $ids, 'event_id' => $id),
-                'fields' => array('id', 'event_id')
+                'fields' => array('id')
         ));
         $successes = array();
-        foreach ($shadowAttributes as $a) {
-            $response = $this->__accept($a['ShadowAttribute']['id']);
+        foreach ($shadowAttributes as $shadowAttributeId) {
+            $response = $this->__accept($shadowAttributeId);
             if (isset($response['saved'])) {
-                $successes[] = $a['ShadowAttribute']['id'];
+                $successes[] = $shadowAttributeId;
             }
         }
         $this->ShadowAttribute->Event->unpublishEvent($id, true);
@@ -1114,4 +1056,16 @@ class ShadowAttributesController extends AppController
             $this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
         }
     }
+
+    private function __attributeIdToConditions($id)
+    {
+        if (is_numeric($id)) {
+            $conditions = array('Attribute.id' => $id);
+        } elseif (Validation::uuid($id)) {
+            $conditions = array('Attribute.uuid' => $id);
+        } else {
+            throw new NotFoundException(__('Invalid attribute ID.'));
+        }
+        return $conditions;
+    }
 }

app/Controller/SightingsController.php

@@ -3,6 +3,7 @@ App::uses('AppController', 'Controller');
 
 /**
  * @property Sighting $Sighting
+ * @property Event $Event
  */
 class SightingsController extends AppController
 {
@@ -326,28 +327,28 @@ class SightingsController extends AppController
                 throw new MethodNotAllowedException('Invalid object.');
             }
             $eventIds = array();
-            foreach ($object as $k => $v) {
+            foreach ($object as $v) {
                 $eventIds[] = $v['Attribute']['event_id'];
             }
-            $events = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $eventIds, 'metadata' => true));
+            $events = $this->Event->fetchSimpleEvents($this->Auth->user(), ['conditions' => ['id' => $eventIds]]);
         } else {
             $attribute_id = false;
             // let's set the context to event here, since we reuse the variable later on for some additional lookups.
             // Passing $context = 'org' could have interesting results otherwise...
-            $context = 'event';
-            $events = $this->Event->fetchEvent($this->Auth->user(), array('eventid' => $id, 'metadata' => true));
+            $events = $this->Event->fetchSimpleEvents($this->Auth->user(), ['conditions' => ['id' => $id]]);
         }
         if (empty($events)) {
             throw new MethodNotAllowedException('Invalid object.');
         }
-        $results = array();
         $raw = array();
         foreach ($events as $event) {
             $raw = array_merge($raw, $this->Sighting->attachToEvent($event, $this->Auth->user(), $attribute_id));
         }
+        $results = array();
         foreach ($raw as $sighting) {
             $results[$sighting['type']][date('Ymd', $sighting['date_sighting'])][] = $sighting;
         }
+        unset($raw);
         $dataPoints = array();
         $startDate = date('Ymd');
         $range = date('Ymd', $this->Sighting->getMaximumRange());

app/Controller/TagCollectionsController.php

@@ -2,6 +2,9 @@
 
 App::uses('AppController', 'Controller');
 
+/**
+ * @property TagCollection $TagCollection
+ */
 class TagCollectionsController extends AppController
 {
     public $components = array(
@@ -245,6 +248,7 @@ class TagCollectionsController extends AppController
         if (!$this->request->is('post')) {
             $this->set('object_id', $id);
             $this->set('scope', 'TagCollection');
+            $this->set('local', false);
             $this->layout = false;
             $this->autoRender = false;
             $this->render('/Events/add_tag');
@@ -305,23 +309,26 @@ class TagCollectionsController extends AppController
                 }
             }
             $this->autoRender = false;
-            $error = false;
             $success = false;
             if (empty($tag_id_list)) {
                 $tag_id_list = array($tag_id);
             }
 
             foreach ($tag_id_list as $tag_id) {
-                $this->TagCollection->TagCollectionTag->Tag->id = $tag_id;
-                if (!$this->TagCollection->TagCollectionTag->Tag->exists()) {
-                    $error = __('Invalid Tag.');
-                    continue;
+                $conditions = ['Tag.id' => $tag_id];
+                if (!$this->_isSiteAdmin()) {
+                    $conditions['Tag.org_id'] = array('0', $this->Auth->user('org_id'));
+                    $conditions['Tag.user_id'] = array('0', $this->Auth->user('id'));
                 }
                 $tag = $this->TagCollection->TagCollectionTag->Tag->find('first', array(
-                    'conditions' => array('Tag.id' => $tag_id),
+                    'conditions' => $conditions,
                     'recursive' => -1,
                     'fields' => array('Tag.name')
                 ));
+                if (!$tag) {
+                    // Invalid Tag
+                    continue;
+                }
                 $found = $this->TagCollection->TagCollectionTag->find('first', array(
                     'conditions' => array(
                         'tag_collection_id' => $id,
@@ -330,7 +337,7 @@ class TagCollectionsController extends AppController
                     'recursive' => -1,
                 ));
                 if (!empty($found)) {
-                    $error = __('Tag is already attached to this event.');
+                    // Tag is already attached to this collection
                     continue;
                 }
                 $this->TagCollection->TagCollectionTag->create();
@@ -355,9 +362,32 @@ class TagCollectionsController extends AppController
     public function removeTag($id = false, $tag_id = false)
     {
         if (!$this->request->is('post')) {
+            $tagCollection = $this->TagCollection->find('first', array(
+                'recursive' => -1,
+                'conditions' => array(
+                    'TagCollection.id' => $id
+                ),
+            ));
+            if (!$tagCollection) {
+                throw new NotFoundException(__('Invalid tag collection.'));
+            }
+            $tagCollectionTag = $this->TagCollection->TagCollectionTag->find('first', [
+                'recursive' => -1,
+                'conditions' => [
+                    'tag_collection_id' => $id,
+                    'tag_id' => $tag_id,
+                ],
+                'contain' => ['Tag'],
+            ]);
+            if (!$tagCollectionTag) {
+                throw new NotFoundException(__('Invalid tag collection tag.'));
+            }
+
             $this->set('id', $id);
+            $this->set('tag', $tagCollectionTag);
             $this->set('tag_id', $tag_id);
             $this->set('model', 'tag_collection');
+            $this->set('model_name', $tagCollection['TagCollection']['name']);
             $this->layout = 'ajax';
             $this->render('/Attributes/ajax/tagRemoveConfirmation');
         } else {

app/Controller/TagsController.php

@@ -2,6 +2,9 @@
 
 App::uses('AppController', 'Controller');
 
+/**
+ * @property Tag $Tag
+ */
 class TagsController extends AppController
 {
     public $components = array('Security' ,'RequestHandler');
@@ -71,31 +74,31 @@ class TagsController extends AppController
         }
         if ($this->_isRest()) {
             unset($this->paginate['limit']);
-            unset($this->paginate['contain']['EventTag']);
-            unset($this->paginate['contain']['AttributeTag']);
             $paginated = $this->Tag->find('all', $this->paginate);
         } else {
             $paginated = $this->paginate();
         }
         $tagList = array();
         $csv = array();
-        $sgs = $this->Tag->EventTag->Event->SharingGroup->fetchAllAuthorised($this->Auth->user());
         foreach ($paginated as $k => $tag) {
             $tagList[] = $tag['Tag']['id'];
-            $paginated[$k]['Tag']['count'] = $this->Tag->EventTag->countForTag($tag['Tag']['id'], $this->Auth->user(), $sgs);
+            $paginated[$k]['Tag']['count'] = $this->Tag->EventTag->countForTag($tag, $this->Auth->user());
+            $paginated[$k]['Tag']['attribute_count'] = $this->Tag->AttributeTag->countForTag($tag, $this->Auth->user());
+
             if (!$this->_isRest()) {
                 $paginated[$k]['event_ids'] = array();
-                $paginated[$k]['attribute_ids'] = array();
                 foreach ($paginated[$k]['EventTag'] as $et) {
                     $paginated[$k]['event_ids'][] = $et['event_id'];
                 }
-                unset($paginated[$k]['EventTag']);
+                $paginated[$k]['attribute_ids'] = array();
                 foreach ($paginated[$k]['AttributeTag'] as $at) {
                     $paginated[$k]['attribute_ids'][] = $at['attribute_id'];
                 }
-                unset($paginated[$k]['AttributeTag']);
             }
-            $paginated[$k]['Tag']['attribute_count'] = $this->Tag->AttributeTag->countForTag($tag['Tag']['id'], $this->Auth->user(), $sgs);
+
+            unset($paginated[$k]['EventTag']);
+            unset($paginated[$k]['AttributeTag']);
+
             if (!empty($tag['FavouriteTag'])) {
                 foreach ($tag['FavouriteTag'] as $ft) {
                     if ($ft['user_id'] == $this->Auth->user('id')) {
@@ -364,7 +367,7 @@ class TagsController extends AppController
             if ($this->_isRest()) {
                 $this->set('name', 'Tag deleted.');
                 $this->set('message', 'Tag deleted.');
-                $this->set('url', '/tags/delete/' . $id);
+                $this->set('url', $this->baseurl . '/tags/delete/' . $id);
                 $this->set('_serialize', array('name', 'message', 'url'));
             }
             $this->Flash->success(__('Tag deleted'));
@@ -395,58 +398,13 @@ class TagsController extends AppController
             if (empty($tag['EventTag'])) {
                 $tag['Tag']['count'] = 0;
             } else {
-                $eventIDs = array();
-                foreach ($tag['EventTag'] as $eventTag) {
-                    $eventIDs[] = $eventTag['event_id'];
-                }
-                $conditions = array('Event.id' => $eventIDs);
-                if (!$this->_isSiteAdmin()) {
-                    $conditions = array_merge(
-                        $conditions,
-                        array('OR' => array(
-                                array('AND' => array(
-                                        array('Event.distribution >' => 0),
-                                        array('Event.published =' => 1)
-                                )),
-                                array('Event.orgc_id' => $this->Auth->user('org_id'))
-                        ))
-                );
-                }
-                $events = $this->Tag->EventTag->Event->find('all', array(
-                        'fields' => array('Event.id', 'Event.distribution', 'Event.orgc_id'),
-                        'conditions' => $conditions
-                ));
-                $tag['Tag']['count'] = count($events);
+                $tag['Tag']['count'] = $this->Tag->EventTag->countForTag($tag, $this->Auth->user());
             }
             unset($tag['EventTag']);
             if (empty($tag['AttributeTag'])) {
                 $tag['Tag']['attribute_count'] = 0;
             } else {
-                $attributeIDs = array();
-                foreach ($tag['AttributeTag'] as $attributeTag) {
-                    $attributeIDs[] = $attributeTag['attribute_id'];
-                }
-                $conditions = array('Attribute.id' => $attributeIDs);
-                if (!$this->_isSiteAdmin()) {
-                    $conditions = array_merge(
-                        $conditions,
-                        array('OR' => array(
-                            array('AND' => array(
-                                array('Attribute.deleted =' => 0),
-                                array('Attribute.distribution >' => 0),
-                                array('Event.distribution >' => 0),
-                                array('Event.published =' => 1)
-                            )),
-                            array('Event.orgc_id' => $this->Auth->user('org_id'))
-                        ))
-                    );
-                }
-                $attributes = $this->Tag->AttributeTag->Attribute->find('all', array(
-                    'fields'     => array('Attribute.id', 'Attribute.deleted', 'Attribute.distribution', 'Event.id', 'Event.distribution', 'Event.orgc_id'),
-                    'contain'    => array('Event' => array('fields' => array('id', 'distribution', 'orgc_id'))),
-                    'conditions' => $conditions
-                ));
-                $tag['Tag']['attribute_count'] = count($attributes);
+                $tag['Tag']['attribute_count'] = $this->Tag->AttributeTag->countForTag($tag, $this->Auth->user());;
             }
             unset($tag['AttributeTag']);
             $this->set('Tag', $tag['Tag']);
@@ -458,37 +416,26 @@ class TagsController extends AppController
 
     public function showEventTag($id)
     {
-        $this->loadModel('EventTag');
         $this->loadModel('Taxonomy');
-        if (!$this->EventTag->Event->checkIfAuthorised($this->Auth->user(), $id)) {
-            throw new MethodNotAllowedException('Invalid event.');
+
+        $event = $this->Tag->EventTag->Event->fetchSimpleEvent($this->Auth->user(), $id, [
+            'fields' => ['Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.user_id'],
+            'contain' => [
+                'EventTag' => array(
+                    'Tag' => array('order' => false),
+                    'order' => false
+                )
+            ],
+        ]);
+        if (!$event) {
+            throw new NotFoundException(__('Invalid event.'));
         }
-        $this->loadModel('GalaxyCluster');
-        $cluster_names = $this->GalaxyCluster->find('list', array(
-            'fields' => array('GalaxyCluster.tag_name'),
-            'group' => array('GalaxyCluster.id', 'GalaxyCluster.tag_name')
-        ));
-        $this->helpers[] = 'TextColour';
-        $conditions = array(
-                'event_id' => $id,
-                'Tag.name !=' => $cluster_names
-        );
-        $tags = $this->EventTag->find('all', array(
-                'conditions' => $conditions,
-                'contain' => array('Tag'),
-                'fields' => array('Tag.id', 'Tag.colour', 'Tag.name', 'EventTag.local'),
-        ));
-        foreach ($tags as $k => $tag) {
-            $tags[$k]['local'] = $tag['EventTag']['local'];
-        }
-        $this->set('tags', $tags);
-        $event = $this->Tag->EventTag->Event->find('first', array(
-                'recursive' => -1,
-                'fields' => array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.user_id'),
-                'conditions' => array('Event.id' => $id)
-        ));
-        $this->set('required_taxonomies', $this->EventTag->Event->getRequiredTaxonomies());
-        $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($tags);
+        // Remove galaxy tags
+        $event = $this->Tag->EventTag->Event->massageTags($event, 'Event', false, true);
+
+        $this->set('tags', $event['EventTag']);
+        $this->set('required_taxonomies', $this->Tag->EventTag->Event->getRequiredTaxonomies());
+        $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($event['EventTag']);
         $this->set('tagConflicts', $tagConflicts);
         $this->set('event', $event);
         $this->layout = 'ajax';
@@ -498,38 +445,26 @@ class TagsController extends AppController
     public function showAttributeTag($id)
     {
         $this->helpers[] = 'TextColour';
-        $this->loadModel('AttributeTag');
+        $this->loadModel('Attribute');
         $this->loadModel('Taxonomy');
 
-        $this->Tag->AttributeTag->Attribute->id = $id;
-        if (!$this->Tag->AttributeTag->Attribute->exists()) {
+        $attributes = $this->Attribute->fetchAttributes($this->Auth->user(), [
+            'conditions' => ['Attribute.id' => $id],
+            'includeAllTags' => true,
+            'flatten' => true,
+            'contain' => array(
+                'Event',
+            ),
+        ]);
+        if (empty($attributes)) {
             throw new NotFoundException(__('Invalid attribute'));
         }
-        $this->Tag->AttributeTag->Attribute->read();
-        $eventId = $this->Tag->AttributeTag->Attribute->data['Attribute']['event_id'];
+        $attribute = $attributes[0];
+        // Remove galaxy tags
+        $attribute = $this->Tag->EventTag->Event->massageTags($attribute, 'Attribute', false, true);
+        $attributeTags = $attribute['AttributeTag'];
 
-        $conditions = array('attribute_id' => $id);
-        $attributeTags = $this->AttributeTag->find('all', array(
-            'conditions' => $conditions,
-            'contain' => array('Tag'),
-            'fields' => array('Tag.id', 'Tag.colour', 'Tag.name', 'AttributeTag.local'),
-        ));
-        foreach ($attributeTags as $k => $at) {
-            $attributeTags[$k]['local'] = $at['AttributeTag']['local'];
-        }
-        $this->loadModel('GalaxyCluster');
-        $cluster_names = $this->GalaxyCluster->find('list', array('fields' => array('GalaxyCluster.tag_name'), 'group' => array('GalaxyCluster.tag_name', 'GalaxyCluster.id')));
-        foreach ($attributeTags as $k => $attributeTag) {
-            if (in_array($attributeTag['Tag']['name'], $cluster_names)) {
-                unset($attributeTags[$k]);
-            }
-        }
-        $event = $this->Tag->AttributeTag->Attribute->Event->find('first', array(
-            'recursive' => -1,
-            'fields' => array('Event.id', 'Event.orgc_id', 'Event.org_id', 'Event.user_id'),
-            'conditions' => array('Event.id' => $eventId)
-        ));
-        $this->set('event', $event);
+        $this->set('event', ['Event' => $attribute['Event']]);
         $this->set('attributeTags', $attributeTags);
         $this->set('attributeId', $id);
         $tagConflicts = $this->Taxonomy->checkIfTagInconsistencies($attributeTags);
@@ -597,22 +532,22 @@ class TagsController extends AppController
         if ($favourites) {
             $items[] = array(
                 'name' => __('Favourite Tags'),
-                'value' => "/tags/selectTag/" . h($id) . "/favourites/" . h($scope) . $localFlag
+                'value' => $this->baseurl . "/tags/selectTag/" . h($id) . "/favourites/" . h($scope) . $localFlag
             );
         }
         if ($scope !== 'tag_collection') {
             $items[] = array(
                 'name' => __('Tag Collections'),
-                'value' => "/tags/selectTag/" . h($id) . "/collections/" . h($scope) . $localFlag
+                'value' => $this->baseurl . "/tags/selectTag/" . h($id) . "/collections/" . h($scope) . $localFlag
             );
         }
         $items[] = array(
             'name' => __('Custom Tags'),
-            'value' => "/tags/selectTag/" . h($id) . "/0/" . h($scope) . $localFlag
+            'value' => $this->baseurl . "/tags/selectTag/" . h($id) . "/0/" . h($scope) . $localFlag
         );
         $items[] = array(
             'name' => __('All Tags'),
-            'value' => "/tags/selectTag/" . h($id) . "/all/" . h($scope) . $localFlag
+            'value' => $this->baseurl . "/tags/selectTag/" . h($id) . "/all/" . h($scope) . $localFlag
         );
 
         $this->loadModel('Taxonomy');
@@ -620,7 +555,7 @@ class TagsController extends AppController
         foreach ($options as $k => $option) {
             $items[] = array(
                 'name' => __('Taxonomy Library') . ":" . h($option),
-                'value' => "/tags/selectTag/" . h($id) . "/" . h($k) . "/" . h($scope . $localFlag)
+                'value' => $this->baseurl . "/tags/selectTag/" . h($id) . "/" . h($k) . "/" . h($scope . $localFlag)
             );
         }
         $this->set('items', $items);
@@ -639,24 +574,10 @@ class TagsController extends AppController
         }
         $this->loadModel('Taxonomy');
         $expanded = array();
-        $banned_tags = $this->Tag->find('list', array(
-                'conditions' => array(
-                        'NOT' => array(
-                                'Tag.org_id' => array(
-                                        0,
-                                        $this->Auth->user('org_id')
-                                ),
-                                'Tag.user_id' => array(
-                                        0,
-                                        $this->Auth->user('id')
-                                )
-                        )
-                ),
-                'fields' => array('Tag.id')
-        ));
         $this->set('taxonomy_id', $taxonomy_id);
         if ($taxonomy_id === 'collections') {
             $this->loadModel('TagCollection');
+            // This method removes banned and hidden tags
             $tagCollections = $this->TagCollection->fetchTagCollection($this->Auth->user());
             $tags = array();
             $inludedTagListString = array();
@@ -666,12 +587,8 @@ class TagsController extends AppController
                 $expanded[$tagCollection['TagCollection']['id']] = empty($tagCollection['TagCollection']['description']) ? $tagCollection['TagCollection']['name'] : $tagCollection['TagCollection']['description'];
                 if (!empty($tagCollection['TagCollectionTag'])) {
                     $tagList = array();
-                    foreach ($tagCollection['TagCollectionTag'] as $k => $tce) {
-                        if (in_array($tce['tag_id'], $banned_tags)) {
-                            unset($tagCollection['TagCollectionTag'][$k]);
-                        } else {
-                            $tagList[] = $tce['Tag']['name'];
-                        }
+                    foreach ($tagCollection['TagCollectionTag'] as $tce) {
+                        $tagList[] = $tce['Tag']['name'];
                         $tagCollection['TagCollectionTag'] = array_values($tagCollection['TagCollectionTag']);
                     }
                     $tagList = implode(', ', $tagList);
@@ -681,7 +598,7 @@ class TagsController extends AppController
             }
         } else {
             if ($taxonomy_id === '0') {
-                $temp = $this->Taxonomy->getAllTaxonomyTags(true, false, true);
+                $temp = $this->Taxonomy->getAllTaxonomyTags(true, $this->Auth->user(), true);
                 $tags = array();
                 foreach ($temp as $tag) {
                     $tags[$tag['Tag']['id']] = $tag['Tag'];
@@ -690,7 +607,12 @@ class TagsController extends AppController
                 $expanded = $tags;
             } elseif ($taxonomy_id === 'favourites') {
                 $tags = array();
-                $conditions = array('FavouriteTag.user_id' => $this->Auth->user('id'));
+                $conditions = array(
+                    'FavouriteTag.user_id' => $this->Auth->user('id'),
+                    'Tag.org_id' => array(0, $this->Auth->user('org_id')),
+                    'Tag.user_id' => array(0, $this->Auth->user('id')),
+                    'Tag.hide_tag' => 0,
+                );
                 $favTags = $this->Tag->FavouriteTag->find('all', array(
                     'conditions' => $conditions,
                     'recursive' => -1,
@@ -702,9 +624,10 @@ class TagsController extends AppController
                     $expanded = $tags;
                 }
             } elseif ($taxonomy_id === 'all') {
+                $conditions = [];
                 if (!$this->_isSiteAdmin()) {
-                    $conditions = array('Tag.org_id' => array(0, $this->Auth->user('org_id')));
-                    $conditions = array('Tag.user_id' => array(0, $this->Auth->user('id')));
+                    $conditions[] = array('Tag.org_id' => array(0, $this->Auth->user('org_id')));
+                    $conditions[] = array('Tag.user_id' => array(0, $this->Auth->user('id')));
                 }
                 $conditions['Tag.hide_tag'] = 0;
                 $allTags = $this->Tag->find('all', array(
@@ -714,13 +637,9 @@ class TagsController extends AppController
                     'fields' => array('Tag.id', 'Tag.name', 'Tag.colour')
                 ));
                 $tags = array();
-                foreach ($allTags as $k => $tag) {
-                    $temp = explode(':', $tag['Tag']['name']);
-                    if (count($temp) > 1) {
-                        if ($temp[0] !== 'misp-galaxy') {
-                            $tags[$tag['Tag']['id']] = $tag['Tag'];
-                        }
-                    } else {
+                foreach ($allTags as $tag) {
+                    $isGalaxyTag = strpos($tag['Tag']['name'], 'misp-galaxy:') === 0;
+                    if (!$isGalaxyTag) {
                         $tags[$tag['Tag']['id']] = $tag['Tag'];
                     }
                 }
@@ -737,21 +656,38 @@ class TagsController extends AppController
                         }
                     }
                 }
-            }
-            // Unset all tags that this user cannot use for tagging, determined by the org restriction on tags
-            if (!$this->_isSiteAdmin()) {
-                foreach ($banned_tags as $banned_tag) {
-                    unset($tags[$banned_tag]);
-                    unset($expanded[$banned_tag]);
+
+                // Unset all tags that this user cannot use for tagging, determined by the org restriction on tags
+                if (!$this->_isSiteAdmin()) {
+                    $banned_tags = $this->Tag->find('list', array(
+                        'conditions' => array(
+                            'NOT' => array(
+                                'Tag.org_id' => array(
+                                    0,
+                                    $this->Auth->user('org_id')
+                                ),
+                                'Tag.user_id' => array(
+                                    0,
+                                    $this->Auth->user('id')
+                                )
+                            )
+                        ),
+                        'fields' => array('Tag.id')
+                    ));
+                    foreach ($banned_tags as $banned_tag) {
+                        unset($tags[$banned_tag]);
+                        unset($expanded[$banned_tag]);
+                    }
                 }
-            }
-            $hidden_tags = $this->Tag->find('list', array(
+
+                $hidden_tags = $this->Tag->find('list', array(
                     'conditions' => array('Tag.hide_tag' => 1),
                     'fields' => array('Tag.id')
-            ));
-            foreach ($hidden_tags as $hidden_tag) {
-                unset($tags[$hidden_tag]);
-                unset($expanded[$hidden_tag]);
+                ));
+                foreach ($hidden_tags as $hidden_tag) {
+                    unset($tags[$hidden_tag]);
+                    unset($expanded[$hidden_tag]);
+                }
             }
         }
 

app/Controller/UsersController.php

@@ -1,6 +1,9 @@
 <?php
 App::uses('AppController', 'Controller');
 
+/**
+ * @property User $User
+ */
 class UsersController extends AppController
 {
     public $newkey;
@@ -1101,7 +1104,7 @@ class UsersController extends AppController
         if ($this->request->is('post') || $this->request->is('put')) {
             $this->Bruteforce = ClassRegistry::init('Bruteforce');
             if (!empty($this->request->data['User']['email'])) {
-                if ($this->Bruteforce->isBlacklisted($_SERVER['REMOTE_ADDR'], $this->request->data['User']['email'])) {
+                if ($this->Bruteforce->isBlocklisted($_SERVER['REMOTE_ADDR'], $this->request->data['User']['email'])) {
                     $expire = Configure::check('SecureAuth.expire') ? Configure::read('SecureAuth.expire') : 300;
                     throw new ForbiddenException('You have reached the maximum number of login attempts. Please wait ' . Configure::read('SecureAuth.expire') . ' seconds and try again.');
                 }
@@ -1678,71 +1681,73 @@ class UsersController extends AppController
 
     public function email_otp()
     {
-      $user = $this->Session->read('email_otp_user');
-      if(empty($user)) {
-        $this->redirect('login');
-      }
-      $redis = $this->User->setupRedis();
-      $user_id = $user['id'];
-
-      if ($this->request->is('post') && isset($this->request->data['User']['otp'])) {
-        $stored_otp = $redis->get('misp:otp:'.$user_id);
-        if (!empty($stored_otp) && $this->request->data['User']['otp'] == $stored_otp) {
-            // we invalidate the previously generated OTP
-            $redis->delete('misp:otp:'.$user_id);
-            // We login the user with CakePHP
-            $this->Auth->login($user);
-            $this->_postlogin();
-        } else {
-            $this->Flash->error(__("The OTP is incorrect or has expired"));
+        $user = $this->Session->read('email_otp_user');
+        if (empty($user)) {
+            $this->redirect('login');
         }
-      } else {
-        // GET Request
+        $redis = $this->User->setupRedisWithException();
+        $user_id = $user['id'];
 
-        // We check for exceptions
-        $exception_list = Configure::read('Security.email_otp_exceptions');
-        if (!empty($exception_list)) {
-          $exceptions = explode(",", $exception_list);
-          foreach ($exceptions as &$exception) {
-            if ($user['email'] == trim($exception)) {
-              // We login the user with CakePHP
-              $this->Auth->login($user);
-              $this->_postlogin();
+        if ($this->request->is('post') && isset($this->request->data['User']['otp'])) {
+            $stored_otp = $redis->get('misp:otp:' . $user_id);
+            if (!empty($stored_otp) && $this->request->data['User']['otp'] == $stored_otp) {
+                // we invalidate the previously generated OTP
+                $redis->del('misp:otp:' . $user_id);
+                // We login the user with CakePHP
+                $this->Auth->login($user);
+                $this->_postlogin();
+            } else {
+                $this->Flash->error(__("The OTP is incorrect or has expired"));
             }
-          }
-        }
-        $this->loadModel('Server');
-
-        // Generating the OTP
-        $digits = !empty(Configure::read('Security.email_otp_length')) ? Configure::read('Security.email_otp_length') : $this->Server->serverSettings['Security']['email_otp_length']['value'];
-        $otp = "";
-        for ($i=0; $i<$digits; $i++) {
-          $otp.= random_int(0,9);
-        }
-        // We use Redis to cache the OTP
-        $redis->set('misp:otp:'.$user_id, $otp);
-        $validity = !empty(Configure::read('Security.email_otp_validity')) ? Configure::read('Security.email_otp_validity') : $this->Server->serverSettings['Security']['email_otp_validity']['value'];
-        $redis->expire('misp:otp:'.$user_id, (int) $validity * 60);
-
-        // Email construction
-        $body = !empty(Configure::read('Security.email_otp_text')) ? Configure::read('Security.email_otp_text') : $this->Server->serverSettings['Security']['email_otp_text']['value'];
-        $body = str_replace('$misp', Configure::read('MISP.baseurl'), $body);
-        $body = str_replace('$org', Configure::read('MISP.org'), $body);
-        $body = str_replace('$contact', Configure::read('MISP.contact'), $body);
-        $body = str_replace('$validity', $validity, $body);
-        $body = str_replace('$otp', $otp, $body);
-        $body = str_replace('$ip', $this->__getClientIP(), $body);
-        $body = str_replace('$username', $user['email'], $body);
-        $result = $this->User->sendEmail(array('User' => $user), $body, false, "[MISP] Email OTP");
-
-        if ( $result ) {
-          $this->Flash->success(__("An email containing a OTP has been sent."));
         } else {
-          $this->Flash->error(__("The email couldn't be sent, please reach out to your administrator."));
-        }
-      }
-    }
+            // GET Request
 
+            // We check for exceptions
+            $exception_list = Configure::read('Security.email_otp_exceptions');
+            if (!empty($exception_list)) {
+                $exceptions = explode(",", $exception_list);
+                foreach ($exceptions as $exception) {
+                    if ($user['email'] === trim($exception)) {
+                        // We login the user with CakePHP
+                        $this->Auth->login($user);
+                        $this->_postlogin();
+                    }
+                }
+            }
+            $this->loadModel('Server');
+
+            // Generating the OTP
+            $digits = Configure::read('Security.email_otp_length') ?: $this->Server->serverSettings['Security']['email_otp_length']['value'];
+            $otp = "";
+            for ($i = 0; $i < $digits; $i++) {
+                $otp .= random_int(0, 9);
+            }
+            // We use Redis to cache the OTP
+            $redis->set('misp:otp:' . $user_id, $otp);
+            $validity = Configure::read('Security.email_otp_validity') ?: $this->Server->serverSettings['Security']['email_otp_validity']['value'];
+            $redis->expire('misp:otp:' . $user_id, (int)$validity * 60);
+
+            // Email construction
+            $body = Configure::read('Security.email_otp_text') ?: $this->Server->serverSettings['Security']['email_otp_text']['value'];
+            $body = str_replace('$misp', Configure::read('MISP.baseurl'), $body);
+            $body = str_replace('$org', Configure::read('MISP.org'), $body);
+            $body = str_replace('$contact', Configure::read('MISP.contact'), $body);
+            $body = str_replace('$validity', $validity, $body);
+            $body = str_replace('$otp', $otp, $body);
+            $body = str_replace('$ip', $this->__getClientIP(), $body);
+            $body = str_replace('$username', $user['email'], $body);
+
+            // Fetch user that contains also PGP or S/MIME keys for e-mail encryption
+            $userForSendMail = $this->User->getUserById($user_id);
+            $result = $this->User->sendEmail($userForSendMail, $body, false, "[MISP] Email OTP");
+
+            if ($result) {
+                $this->Flash->success(__("An email containing a OTP has been sent."));
+            } else {
+                $this->Flash->error(__("The email couldn't be sent, please reach out to your administrator."));
+            }
+        }
+    }
 
     /**
     * Helper function to determine the IP of a client (proxy aware)

app/Lib/Dashboard/AchievementsWidget.php

@@ -0,0 +1,205 @@
+<?php
+class AchievementsWidget
+{
+    /*
+    * Note: for this widget to display as expected, you need all icons to be accessible in your webroot. (img/custom)
+    * Icons used:
+    * - misp_event.png --> https://user-images.githubusercontent.com/1073662/87687773-6eff6d80-c786-11ea-9dcf-009a158a276c.png
+    * - misp_object.png --> https://user-images.githubusercontent.com/1073662/87687775-6f980400-c786-11ea-985c-b3c15c01d63e.png
+    * - tlp_green.png --> https://raw.githubusercontent.com/MISP/intelligence-icons/master/square_png/48/tlp_green.png
+    * - attack.png --> https://raw.githubusercontent.com/mitre-attack/attack-website/master/attack-theme/static/images/attack-logo.png
+    * - taxonomy.png --> https://raw.githubusercontent.com/MISP/intelligence-icons/master/square_png/48/taxonomy.png
+    * - galaxy.png --> https://raw.githubusercontent.com/MISP/intelligence-icons/master/square_png/48/galaxy.png
+    *
+    */
+
+    public $render = 'Achievements';
+    public $title = 'Achievements of my organization';
+    public $description = 'Earn badges and improve your usage of MISP.';
+    public $width = 4;
+    public $height = 10;
+    public $cacheLifetime = false;
+    public $autoRefreshDelay = false;
+    public $params = array(
+            'past_days' => 'The past number of days considered to look for criteria satisfaction (default 180)'
+    );
+    public $placeholder =
+'{
+    "past_days": "180"
+}';
+
+
+    /*
+    * To add a new badge:    add a new item to the list below and write a function check_<name>.
+    *
+    * The check function returns true if the badge is granted
+    */
+
+    private $badges;
+    private $unlocked_badges;
+
+    public function __construct(){
+        $this->badges = array(
+           "events" => array(
+               "icon" => "/img/custom/misp_event.png",
+               "title" => __("MISP is all about sharing relevant data with each other. Start by creating your first event."),
+               "help_page" => "https://www.circl.lu/doc/misp/using-the-system/#creating-an-event"
+           ),
+           "tags" => array(
+               "icon" => "/img/custom/tlp_green.png",
+               "title" => __("By adding tags to your events, they can be categorized more easily."),
+               "help_page" => "https://www.circl.lu/doc/misp/using-the-system/#tagging"
+           ),
+           "objects" => array(
+               "icon" => "/img/custom/misp_object.png",
+               "title" => __("To enhance the structure of your events, use MISP Objects."),
+               "help_page" => "https://github.com/MISP/misp-objects/blob/main/README.md"
+           ),
+           "taxonomies" => array(
+               "icon" => "/img/custom/taxonomy.png",
+               "title" => __("Make sure to speak the same language as your counterparts by using taxonomies for your tags."),
+               "help_page" => "https://www.circl.lu/doc/misp/taxonomy/"
+           ),
+           "galaxies" => array (
+               "icon" => "/img/custom/galaxy.png",
+               "title" => __("Go above and beyond tags and taxonomies, and start using galaxies."),
+               "help_page" => "https://www.circl.lu/doc/misp/galaxy/"
+           ),
+           "attack" => array(
+               "icon" => "/img/custom/attack.png",
+               "title" => __("Add the TTPs following the MITRE ATT&CK framework to make your events even more interesting."),
+               "help_page" => "https://www.misp-project.org/2018/06/27/MISP.2.4.93.released.html"
+           )
+      );
+
+        // The title is modified if the badge is unlocked
+        $this->unlocked_badges = array(
+            "objects" => __("The data you share has now a better structure thanks to the MISP Objects you used."),
+            "events" => __("Congratulations, you have shared your first event!"),
+            "tags" => __("You have been using tags, good job!"),
+            "taxonomies" => __("Taxonomies have been used in your events."),
+            "galaxies" => __("Galaxies have no secrets for you in this Threat Sharing universe."),
+            "attack" => __("MISP & MITRE ATT&CK is a great combo.")
+        );
+    }
+
+
+
+
+
+    private function check_taxonomies($org_id) {
+        return $this->lookup_tag_name_value($org_id, '%:%');
+    }
+
+    private function check_galaxies($org_id) {
+        return $this->lookup_tag_name_value($org_id, 'misp-galaxy:%');
+    }
+
+    private function check_attack($org_id) {
+        return $this->lookup_tag_name_value($org_id, 'misp-galaxy:mitre%');
+    }
+
+
+    private function check_tags($org_id) {
+        $options['joins'] = array(
+                array('table' => 'event_tags',
+                        'alias' => 'EventTag',
+                        'type' => 'INNER',
+                        'conditions' => array(
+                                'EventTag.event_id = Event.id',
+                        )
+                )
+        );
+        $options['fields'] = 'Event.id';
+        $options['limit'] = 1;
+        $conditions = array('Event.orgc_id' => $org_id, 'Event.published' => 1, 'Event.timestamp >=' => $this->start_timestamp);
+        $options['conditions'] = array('AND' => $conditions);
+        $events = $this->Event->find('all', $options);
+        return count($events) > 0;
+    }
+
+    private function check_events($org_id) {
+        $conditions = array('Event.orgc_id' => $org_id, 'Event.published' => 1, 'Event.timestamp >=' => $this->start_timestamp);
+        $events = $this->Event->find('all', array('limit' => 1, 'conditions' => array('AND' => $conditions)));
+        return count($events) > 0;
+    }
+
+    private function check_objects($org_id) {
+        $options['joins'] = array(
+                array('table' => 'objects',
+                        'alias' => 'Objects',
+                        'type' => 'INNER',
+                        'conditions' => array(
+                                'Objects.event_id = Event.id',
+                        )
+                )
+        );
+        $options['fields'] = 'Event.id';
+        $options['limit'] = 1;
+        $conditions = array('Event.orgc_id' => $org_id, 'Event.published' => 1, 'Event.timestamp >=' => $this->start_timestamp);
+        $options['conditions'] = array('AND' => $conditions);
+        $events = $this->Event->find('all', $options);
+        return count($events) > 0;
+    }
+
+    private function lookup_tag_name_value($org_id, $value) {
+        $options['joins'] = array(
+                array('table' => 'event_tags',
+                        'alias' => 'EventTag',
+                        'type' => 'INNER',
+                        'conditions' => array(
+                                'EventTag.event_id = Event.id',
+                        )
+                ),
+                array('table' => 'tags',
+                            'alias' => 'Tag',
+                            'type' => 'INNER',
+                            'conditions' => array(
+                                'Tag.id = EventTag.tag_id'
+                            )
+                )
+        );
+        $options['fields'] = 'Event.id';
+        $options['limit'] = 1;
+        $conditions = array('Event.orgc_id' => $org_id, 'Event.published' => 1, 'Event.timestamp >=' => $this->start_timestamp, 'Tag.name LIKE' => $value);
+        $options['conditions'] = array('AND' => $conditions);
+        $events = $this->Event->find('all', $options);
+        return count($events) > 0;
+    }
+
+    public function handler($user, $options = array())
+    {
+        $this->Org = ClassRegistry::init('Organisation');
+        $this->Event = ClassRegistry::init('Event');
+
+        $days = 180;
+        if(!empty($options['past_days'])) {
+            $days = (int) $options['past_days'];
+        }
+        $this->start_timestamp = $this->Event->resolveTimeDelta($days.'d');
+
+        $org_id = $user['Organisation']['id'];
+        $locked = array();
+        $unlocked = array();
+
+        // We look through each badge and evaluate the condition
+        foreach ($this->badges as $key => $item) {
+            $fun = 'check_'.$key;
+            if($this->$fun($org_id)) { // Condition for badge is met
+                //we replace the text to the unlocked one
+                if(isset($this->unlocked_badges[$key])) {
+                    $item['title'] = $this->unlocked_badges[$key];
+                }
+                $unlocked[] = $item;
+            } else {
+                $locked[] = $item;
+            }
+        }
+
+        $result = array();
+        $result['locked'] = $locked;
+        $result['unlocked'] = $unlocked;
+        return $result;
+    }
+}
+?>

app/Lib/Dashboard/OrgsContributorsGeneric.php

@@ -7,12 +7,12 @@ class OrgsContributorsGeneric
     public $cacheLifetime = 3600;
     public $autoRefreshDelay = false;
     public $params = array (
-        'blacklist_orgs' => 'A list of organisation names to filter out',
+        'blocklist_orgs' => 'A list of organisation names to filter out',
         'timeframe' => 'Number of days considered for the query (30 by default)'
     );
     public $placeholder =
 '{
-    "blacklist_orgs": ["Orgs to filter"],
+    "blocklist_orgs": ["Orgs to filter"],
     "timeframe": "30"
 }';
 
@@ -35,7 +35,7 @@ class OrgsContributorsGeneric
         $orgs = $this->Org->find('all', array( 'conditions' => array('Organisation.local' => 1)));
         $result = array();
         foreach($orgs as $org) {
-            if(!empty($options['blacklist_orgs']) && in_array($org['Organisation']['name'], $options['blacklist_orgs'])) {
+            if(!empty($options['blocklist_orgs']) && in_array($org['Organisation']['name'], $options['blocklist_orgs'])) {
                 continue;
             }
             if ($this->filter($user, $org, $start_timestamp)) {

app/Lib/Dashboard/SharingGraphWidget.php

@@ -15,13 +15,13 @@ class SharingGraphWidget
     public $cacheLifetime = 10;
     public $autoRefreshDelay = false;
     public $params = array (
-        'blacklist_orgs' => 'A list of organisation names to filter out',
+        'blocklist_orgs' => 'A list of organisation names to filter out',
         'months' => 'Number of past months to consider for the graph'
     );
 
     public $placeholder =
 '{
-    "blacklist_orgs": ["Orgs to filter"],
+    "blocklist_orgs": ["Orgs to filter"],
     "months": "6"
 }';
 
@@ -117,8 +117,8 @@ class SharingGraphWidget
         $ghost_orgs = array(); // track orgs without any contribution
         // We start by putting all orgs_id in there:
         foreach($orgs as $org) {
-            // We check for blacklisted orgs
-            if(!empty($options['blacklist_orgs']) && in_array($org['Organisation']['name'], $options['blacklist_orgs'])) {
+            // We check for blocklisted orgs
+            if(!empty($options['blocklist_orgs']) && in_array($org['Organisation']['name'], $options['blocklist_orgs'])) {
                 unset($orgs[$offset]);
             } else {
                 $ghost_orgs[$org['Organisation']['name']] = true;

app/Lib/Export/CountExport.php

@@ -0,0 +1,36 @@
+<?php
+// You can count on me. Raiders roll.
+class CountExport
+{
+    public $additional_params = array(
+        'flatten' => 1
+    );
+    private $__count = 0;
+    public $non_restrictive_export = true;
+
+    public function handler($data, $options = array())
+    {
+        if ($options['scope'] === 'Attribute') {
+            $this->__count++;
+        }
+        if ($options['scope'] === 'Event') {
+            $this->__count++;
+        }
+        return '';
+    }
+
+    public function header($options = array())
+    {
+        return '';
+    }
+
+    public function footer()
+    {
+        return $this->__count;
+    }
+
+    public function separator()
+    {
+        return "";
+    }
+}

app/Lib/Network/Email/SmimeTransport.php

@@ -1,74 +0,0 @@
-<?php
-/**
- * Send mail using mail() function
- * with SMIME headers
- */
-
-/**
- * Send mail using mail() function
- *
- * @package       Cake.Network.Email
- */
-class SmimeTransport extends AbstractTransport
-{
-
-    /**
-     * Send mail
-     *
-     * @param CakeEmail $email CakeEmail
-     * @return array
-     * @throws SocketException When mail cannot be sent.
-     */
-    public function send(CakeEmail $email)
-    {
-        $eol = PHP_EOL;
-        if (isset($this->_config['eol'])) {
-            $eol = $this->_config['eol'];
-        }
-        $headers = $email->getHeaders(array('from', 'sender', 'replyTo', 'readReceipt', 'returnPath', 'to', 'cc', 'bcc'));
-        // DIRTY HEADERS
-        $headers = array_merge($headers, array('Content-Transfer-Encoding' => 'base64', 'Content-Type' => 'application/pkcs7-mime; name=smime.p7m; smime-type=enveloped-data', 'Content-Disposition' => 'attachment; filename=smime.p7m', 'Content-Description' => 'S/MIME Encrypted Message'));
-        $to = $headers['To'];
-        unset($headers['To']);
-        foreach ($headers as $key => $header) {
-            $headers[$key] = str_replace(array("\r", "\n"), '', $header);
-        }
-        $headers = $this->_headersToString($headers, $eol);
-        $subject = str_replace(array("\r", "\n"), '', $email->subject());
-        $to = str_replace(array("\r", "\n"), '', $to);
-
-        $message = implode($eol, $email->message());
-
-        $params = isset($this->_config['additionalParameters']) ? $this->_config['additionalParameters'] : null;
-        $this->_mail($to, $subject, $message, $headers, $params);
-        return array('headers' => $headers, 'message' => $message);
-    }
-
-    /**
-     * Wraps internal function mail() and throws exception instead of errors if anything goes wrong
-     *
-     * @param string $to email's recipient
-     * @param string $subject email's subject
-     * @param string $message email's body
-     * @param string $headers email's custom headers
-     * @param string $params additional params for sending email, will be ignored when in safe_mode
-     * @throws SocketException if mail could not be sent
-     * @return void
-     */
-    protected function _mail($to, $subject, $message, $headers, $params = null)
-    {
-        if (ini_get('safe_mode')) {
-            //@codingStandardsIgnoreStart
-            if (!@mail($to, $subject, $message, $headers)) {
-                $error = error_get_last();
-                $msg = 'Could not send email: ' . isset($error['message']) ? $error['message'] : 'unknown';
-                throw new SocketException($msg);
-            }
-        } elseif (!@mail($to, $subject, $message, $headers, $params)) {
-            $error = error_get_last();
-            $msg = 'Could not send email: ' . isset($error['message']) ? $error['message'] : 'unknown';
-            //@codingStandardsIgnoreEnd
-            throw new SocketException($msg);
-        }
-    }
-}

app/Lib/Tools/AWSS3Client.php

@@ -49,6 +49,14 @@ class AWSS3Client
         return $s3;
     }
 
+    public function exist($key)
+    {
+        return $this->__client->doesObjectExist([
+            'Bucket' => $this->__settings['bucket_name'],
+            'Key' => $key,
+        ]);
+    }
+
     public function upload($key, $data)
     {
         $this->__client->putObject([

app/Lib/Tools/AttachmentTool.php

@@ -0,0 +1,564 @@
+<?php
+App::uses('AWSS3Client', 'Tools');
+
+class AttachmentTool
+{
+    const ZIP_PASSWORD = 'infected';
+    const ADVANCED_EXTRACTION_SCRIPT_PATH = APP . 'files/scripts/generate_file_objects.py';
+
+    /** @var AWSS3Client */
+    private $s3client;
+
+    /**
+     * @param int $eventId
+     * @param int $attributeId
+     * @param string $path_suffix
+     * @return bool
+     * @throws Exception
+     */
+    public function exists($eventId, $attributeId, $path_suffix = '')
+    {
+        return $this->_exists(false, $eventId, $attributeId, $path_suffix);
+    }
+
+    /**
+     * @param int $eventId
+     * @param int $attributeId
+     * @param string $path_suffix
+     * @return bool
+     * @throws Exception
+     */
+    public function shadowExists($eventId, $attributeId, $path_suffix = '')
+    {
+        return $this->_exists(true, $eventId, $attributeId, $path_suffix);
+    }
+
+
+    /**
+     * @param bool $shadow
+     * @param int $eventId
+     * @param int $attributeId
+     * @param string $path_suffix
+     * @return bool
+     * @throws Exception
+     */
+    protected function _exists($shadow, $eventId, $attributeId, $path_suffix = '')
+    {
+        if ($this->attachmentDirIsS3()) {
+            $s3 = $this->loadS3Client();
+            $path = $this->getPath($shadow, $eventId, $attributeId, $path_suffix);
+            return $s3->exist($path);
+        } else {
+            try {
+                $this->_getFile($shadow, $eventId, $attributeId, $path_suffix);
+            } catch (NotFoundException $e) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    /**
+     * @param int $eventId
+     * @param int $attributeId
+     * @param string $path_suffix
+     * @return string
+     * @throws Exception
+     */
+    public function getContent($eventId, $attributeId, $path_suffix = '')
+    {
+        return $this->_getContent(false, $eventId, $attributeId, $path_suffix);
+    }
+
+    /**
+     * @param int $eventId
+     * @param int $attributeId
+     * @param string $path_suffix
+     * @return string
+     * @throws Exception
+     */
+    public function getShadowContent($eventId, $attributeId, $path_suffix = '')
+    {
+        return $this->_getContent(true, $eventId, $attributeId, $path_suffix);
+    }
+
+    /**
+     * @param bool $shadow
+     * @param int $eventId
+     * @param int $attributeId
+     * @param string $path_suffix
+     * @return string
+     * @throws Exception
+     */
+    protected function _getContent($shadow, $eventId, $attributeId, $path_suffix = '')
+    {
+        if ($this->attachmentDirIsS3()) {
+            $s3 = $this->loadS3Client();
+            $path = $this->getPath($shadow, $eventId, $attributeId, $path_suffix);
+            return $s3->download($path);
+        } else {
+            $file = $this->_getFile($shadow, $eventId, $attributeId, $path_suffix);
+            $result = $file->read();
+            if ($result === false) {
+                throw new Exception("Could not read file '{$file->path}'.");
+            }
+            return $result;
+        }
+    }
+
+    /**
+     * @param int $eventId
+     * @param int $attributeId
+     * @param string $pathSuffix
+     * @return File
+     * @throws Exception
+     */
+    public function getFile($eventId, $attributeId, $pathSuffix = '')
+    {
+        return $this->_getFile(false, $eventId, $attributeId, $pathSuffix);
+    }
+
+    /**
+     * @param int $eventId
+     * @param int $attributeId
+     * @param string $pathSuffix
+     * @return File
+     * @throws Exception
+     */
+    public function getShadowFile($eventId, $attributeId, $pathSuffix = '')
+    {
+        return $this->_getFile(true, $eventId, $attributeId, $pathSuffix);
+    }
+
+    /**
+     * @param bool $shadow
+     * @param int $eventId
+     * @param int $attributeId
+     * @param string $pathSuffix
+     * @return File
+     * @throws Exception
+     */
+    protected function _getFile($shadow, $eventId, $attributeId, $pathSuffix = '')
+    {
+        $path = $this->getPath($shadow, $eventId, $attributeId, $pathSuffix);
+
+        if ($this->attachmentDirIsS3()) {
+            $s3 = $this->loadS3Client();
+            $content = $s3->download($path);
+
+            $file = new File($this->tempFileName());
+            if (!$file->write($content)) {
+                throw new Exception("Could not write temporary file '{$file->path}'.");
+            }
+
+        } else {
+            $filepath = $this->attachmentDir() . DS . $path;
+            $file = new File($filepath);
+            if (!$file->exists()) {
+                throw new NotFoundException("File '$filepath' does not exists.");
+            }
+        }
+
+        return $file;
+    }
+
+    /**
+     * @param int $eventId
+     * @param int $attributeId
+     * @param string $data
+     * @param string $pathSuffix
+     * @return bool
+     * @throws Exception
+     */
+    public function save($eventId, $attributeId, $data, $pathSuffix = '')
+    {
+        return $this->_save(false, $eventId, $attributeId, $data, $pathSuffix);
+    }
+
+    /**
+     * @param int $eventId
+     * @param int $attributeId
+     * @param string $data
+     * @param string $pathSuffix
+     * @return bool
+     * @throws Exception
+     */
+    public function saveShadow($eventId, $attributeId, $data, $pathSuffix = '')
+    {
+        return $this->_save(true, $eventId, $attributeId, $data, $pathSuffix);
+    }
+
+    /**
+     * @param bool $shadow
+     * @param int $eventId
+     * @param int $attributeId
+     * @param string $data
+     * @param string $pathSuffix
+     * @return bool
+     * @throws Exception
+     */
+    protected function _save($shadow, $eventId, $attributeId, $data, $pathSuffix = '')
+    {
+        $path = $this->getPath($shadow, $eventId, $attributeId, $pathSuffix);
+
+        if ($this->attachmentDirIsS3()) {
+            $s3 = $this->loadS3Client();
+            $s3->upload($path, $data);
+
+        } else {
+            $path = $this->attachmentDir() . DS . $path;
+            $file = new File($path, true);
+            if (!$file->write($data)) {
+                throw new Exception("Could not save attachment to file '$path'.");
+            }
+        }
+
+        return true;
+    }
+
+    /**
+     * @param int $eventId
+     * @param int $attributeId
+     * @param string $pathSuffix
+     * @return bool
+     * @throws Exception
+     */
+    public function delete($eventId, $attributeId, $pathSuffix = '')
+    {
+        return $this->_delete(false, $eventId, $attributeId, $pathSuffix);
+    }
+
+    /**
+     * @param int $eventId
+     * @param int $attributeId
+     * @param string $pathSuffix
+     * @return bool
+     * @throws Exception
+     */
+    public function deleteShadow($eventId, $attributeId, $pathSuffix = '')
+    {
+        return $this->_delete(true, $eventId, $attributeId, $pathSuffix);
+    }
+
+    /**
+     * @param bool $shadow
+     * @param int $eventId
+     * @param int $attributeId
+     * @param string $pathSuffix
+     * @return bool Return true if file was deleted, `false` if file doesn't exists.
+     * @throws Exception
+     */
+    protected function _delete($shadow, $eventId, $attributeId, $pathSuffix = '')
+    {
+        if ($this->attachmentDirIsS3()) {
+            $s3 = $this->loadS3Client();
+            $path = $this->getPath($shadow, $eventId, $attributeId, $pathSuffix);
+            $s3->delete($path);
+        } else {
+            try {
+                $file = $this->_getFile($shadow, $eventId, $attributeId, $pathSuffix);
+            } catch (NotFoundException $e) {
+                return false;
+            }
+
+            if (!$file->delete()) {
+                throw new Exception(__('Delete of file attachment failed. Please report to administrator.'));
+            }
+        }
+
+        return true;
+    }
+
+    /**
+     * Deletes all attributes and shadow attributes files.
+     *
+     * @param int $eventId
+     * @return bool
+     * @throws Exception
+     */
+    public function deleteAll($eventId)
+    {
+        if ($this->attachmentDirIsS3()) {
+            $s3 = $this->loadS3Client();
+            $s3->deleteDirectory($eventId);
+        } else {
+            $dirPath = $this->attachmentDir();
+
+            foreach (array($dirPath, $dirPath . DS . 'shadow') as $dirPath) {
+                $folder = new Folder($dirPath . DS . $eventId);
+                if ($folder->pwd() && !$folder->delete()) {
+                    throw new Exception("Delete of directory '{$folder->pwd()}' failed: " . implode(', ', $folder->errors()));
+                }
+            }
+        }
+
+        return true;
+    }
+
+    /**
+     * @param string $originalFilename
+     * @param string $content
+     * @param string $md5
+     * @return string Content of zipped file
+     * @throws Exception
+     */
+    public function encrypt($originalFilename, $content, $md5)
+    {
+        if (method_exists("ZipArchive", "setEncryptionName")) {
+            // When PHP zip extension is installed and supports creating encrypted archives.
+            return $this->encryptByExtension($originalFilename, $content, $md5);
+        } else {
+            return $this->encryptByCommand($originalFilename, $content, $md5);
+        }
+    }
+
+    /**
+     * @param string $originalFilename
+     * @param string $content
+     * @param string $md5
+     * @return string Content of zipped file
+     * @throws Exception
+     */
+    private function encryptByCommand($originalFilename, $content, $md5)
+    {
+        $tempDir = $this->tempDir();
+
+        $contentsFile = new File($tempDir . DS . $md5, true);
+        if (!$contentsFile->write($content)) {
+            throw new Exception("Could not write content to file '{$contentsFile->path}'.");
+        }
+        $contentsFile->close();
+
+        $fileNameFile = new File($tempDir . DS . $md5 . '.filename.txt', true);
+        if (!$fileNameFile->write($originalFilename)) {
+            throw new Exception("Could not write original file name to file '{$fileNameFile->path}'.");
+        }
+        $fileNameFile->close();
+
+        $zipFile = new File($tempDir . DS . $md5 . '.zip');
+
+        $exec = [
+            'zip',
+            '-j', // junk (don't record) directory names
+            '-P', // use standard encryption
+            self::ZIP_PASSWORD,
+            escapeshellarg($zipFile->path),
+            escapeshellarg($contentsFile->path),
+            escapeshellarg($fileNameFile->path),
+        ];
+
+        try {
+            $this->execute($exec);
+            $zipContent = $zipFile->read();
+            if ($zipContent === false) {
+                throw new Exception("Could not read content of newly created ZIP file.");
+            }
+
+            return $zipContent;
+
+        } catch (Exception $e) {
+            throw new Exception("Could not create encrypted ZIP file '{$zipFile->path}'.", 0, $e);
+
+        } finally {
+            $fileNameFile->delete();
+            $contentsFile->delete();
+            $zipFile->delete();
+        }
+    }
+
+    /**
+     * @param string $originalFilename
+     * @param string $content
+     * @param string $md5
+     * @return string Content of zipped file
+     * @throws Exception
+     */
+    private function encryptByExtension($originalFilename, $content, $md5)
+    {
+        $zipFilePath = $this->tempFileName();
+
+        $zip = new ZipArchive();
+        $result = $zip->open($zipFilePath, ZipArchive::CREATE);
+        if ($result === true) {
+            $zip->setPassword(self::ZIP_PASSWORD);
+
+            $zip->addFromString($md5, $content);
+            $zip->setEncryptionName($md5, ZipArchive::EM_AES_128);
+
+            $zip->addFromString("$md5.filename.txt", $originalFilename);
+            $zip->setEncryptionName("$md5.filename.txt", ZipArchive::EM_AES_128);
+
+            $zip->close();
+        } else {
+            throw new Exception("Could not create encrypted ZIP file '$zipFilePath'. Error code: $result");
+        }
+
+        $zipFile = new File($zipFilePath);
+        $zipContent = $zipFile->read();
+        if ($zipContent === false) {
+            throw new Exception("Could not read content of newly created ZIP file.");
+        }
+        $zipFile->delete();
+
+        return $zipContent;
+    }
+
+    /**
+     * @param string $content
+     * @param array $hashTypes
+     * @return array
+     * @throws InvalidArgumentException
+     */
+    public function computeHashes($content, array $hashTypes = array())
+    {
+        $validHashes = array('md5', 'sha1', 'sha256');
+        $hashes = [];
+        foreach ($hashTypes as $hashType) {
+            if (!in_array($hashType, $validHashes)) {
+                throw new InvalidArgumentException("Hash type '$hashType' is not valid hash type.");
+            }
+            $hashes[$hashType] = hash($hashType, $content);
+        }
+        return $hashes;
+    }
+
+    /**
+     * @param string $pythonBin
+     * @param string $filePath
+     * @return array
+     * @throws Exception
+     */
+    public function advancedExtraction($pythonBin, $filePath)
+    {
+        return $this->executeAndParseJsonOutput([
+            $pythonBin,
+            self::ADVANCED_EXTRACTION_SCRIPT_PATH,
+            '-p',
+            escapeshellarg($filePath),
+        ]);
+    }
+
+    /**
+     * @param string $pythonBin
+     * @return array
+     * @throws Exception
+     */
+    public function checkAdvancedExtractionStatus($pythonBin)
+    {
+        return $this->executeAndParseJsonOutput([$pythonBin, self::ADVANCED_EXTRACTION_SCRIPT_PATH, '-c']);
+    }
+
+    private function tempFileName()
+    {
+        $randomName = (new RandomTool())->random_str(false, 12);
+        return $this->tempDir() . DS . $randomName;
+    }
+
+    /**
+     * @return string
+     */
+    private function tempDir()
+    {
+        return Configure::read('MISP.tmpdir') ?: sys_get_temp_dir();
+    }
+
+    /**
+     * @return string
+     */
+    private function attachmentDir()
+    {
+        return Configure::read('MISP.attachments_dir') ?: (APP . 'files');
+    }
+
+    /**
+     * Naive way to detect if we're working in S3
+     * @return bool
+     */
+    private function attachmentDirIsS3()
+    {
+        return substr(Configure::read('MISP.attachments_dir'), 0, 2) === "s3";
+    }
+
+    /**
+     * @return AWSS3Client
+     */
+    private function loadS3Client()
+    {
+        if ($this->s3client) {
+            return $this->s3client;
+        }
+
+        $client = new AWSS3Client();
+        $client->initTool();
+        $this->s3client = $client;
+        return $client;
+    }
+
+    /**
+     * @param bool $shadow
+     * @param int $eventId
+     * @param int $attributeId
+     * @param string $pathSuffix
+     * @return string
+     */
+    private function getPath($shadow, $eventId, $attributeId, $pathSuffix)
+    {
+        $path = $shadow ? ('shadow' . DS) : '';
+        return $path . $eventId . DS . $attributeId . $pathSuffix;
+    }
+
+    /**
+     * @param array $command
+     * @return array
+     * @throws Exception
+     */
+    private function executeAndParseJsonOutput(array $command)
+    {
+        $output = $this->execute($command);
+
+        $json = json_decode($output, true);
+        if ($json === null) {
+            throw new Exception("Command output is not valid JSON: " . json_last_error_msg());
+        }
+        return $json;
+    }
+
+    /**
+     * This method is much more complicated than just `exec`, but it also provide stderr output, so Exceptions
+     * can be much more specific.
+     *
+     * @param array $command
+     * @return string
+     * @throws Exception
+     */
+    private function execute(array $command)
+    {
+        $descriptorspec = [
+            1 => ["pipe", "w"], // stdout
+            2 => ["pipe", "w"], // stderr
+        ];
+
+        $command = implode(' ', $command);
+        $process = proc_open($command, $descriptorspec, $pipes);
+        if (!$process) {
+            throw new Exception("Command '$command' could be started.");
+        }
+
+        $stdout = stream_get_contents($pipes[1]);
+        if ($stdout === false) {
+            throw new Exception("Could not get STDOUT of command.");
+        }
+        fclose($pipes[1]);
+
+        $stderr = stream_get_contents($pipes[2]);
+        fclose($pipes[2]);
+
+        $returnCode = proc_close($process);
+        if ($returnCode !== 0) {
+            throw new Exception("Command '$command' return error code $returnCode. STDERR: '$stderr', STDOUT: '$stdout'");
+        }
+
+        return $stdout;
+    }
+}

app/Lib/Tools/CryptGpgExtended.php

@@ -0,0 +1,64 @@
+<?php
+class CryptGpgExtended extends Crypt_GPG
+{
+    /**
+     * Export the smallest public key possible from the keyring.
+     *
+     * This removes all signatures except the most recent self-signature on each user ID. This option is the same as
+     * running the --edit-key command "minimize" before export except that the local copy of the key is not modified.
+     *
+     * The exported key remains on the keyring. To delete the public key, use
+     * {@link Crypt_GPG::deletePublicKey()}.
+     *
+     * If more than one key fingerprint is available for the specified
+     * <kbd>$keyId</kbd> (for example, if you use a non-unique uid) only the
+     * first public key is exported.
+     *
+     * @param string  $keyId either the full uid of the public key, the email
+     *                       part of the uid of the public key or the key id of
+     *                       the public key. For example,
+     *                       "Test User (example) <test@example.com>",
+     *                       "test@example.com" or a hexadecimal string.
+     * @param boolean $armor optional. If true, ASCII armored data is returned;
+     *                       otherwise, binary data is returned. Defaults to
+     *                       true.
+     *
+     * @return string the public key data.
+     *
+     * @throws Crypt_GPG_KeyNotFoundException if a public key with the given
+     *         <kbd>$keyId</kbd> is not found.
+     *
+     * @throws Crypt_GPG_Exception if an unknown or unexpected error occurs.
+     *         Use the <kbd>debug</kbd> option and file a bug report if these
+     *         exceptions occur.
+     */
+    public function exportPublicKeyMinimal($keyId, $armor = true)
+    {
+        $fingerprint = $this->getFingerprint($keyId);
+
+        if ($fingerprint === null) {
+            throw new Crypt_GPG_KeyNotFoundException(
+                'Key not found: ' . $keyId,
+                self::ERROR_KEY_NOT_FOUND,
+                $keyId
+            );
+        }
+
+        $keyData   = '';
+        $operation = '--export';
+        $operation .= ' ' . escapeshellarg($fingerprint);
+
+        $arguments = array('--export-options', 'export-minimal');
+        if ($armor) {
+            $arguments[] = '--armor';
+        }
+
+        $this->engine->reset();
+        $this->engine->setPins($this->passphrases);
+        $this->engine->setOutput($keyData);
+        $this->engine->setOperation($operation, $arguments);
+        $this->engine->run();
+
+        return $keyData;
+    }
+}

app/Lib/Tools/GpgTool.php

@@ -1,6 +1,36 @@
 <?php
 class GpgTool
 {
+    /**
+     * @return CryptGpgExtended
+     * @throws Exception
+     */
+    public function initializeGpg()
+    {
+        if (!class_exists('Crypt_GPG')) {
+            // 'Crypt_GPG' class cannot be autoloaded, try to require from include_path.
+            if (!stream_resolve_include_path('Crypt/GPG.php')) {
+                throw new Exception("Crypt_GPG is not installed.");
+            }
+            require_once 'Crypt/GPG.php';
+        }
+
+        require_once __DIR__ . '/CryptGpgExtended.php';
+
+        $homedir = Configure::read('GnuPG.homedir');
+        if ($homedir === null) {
+            throw new Exception("Configuration option 'GnuPG.homedir' is not set, Crypt_GPG cannot be initialized.");
+        }
+
+        $options = array(
+            'homedir' => $homedir,
+            'gpgconf' => Configure::read('GnuPG.gpgconf'),
+            'binary' => Configure::read('GnuPG.binary') ?: '/usr/bin/gpg',
+        );
+
+        return new CryptGpgExtended($options);
+    }
+
     /**
      * @param string $search
      * @return array

app/Lib/Tools/MalwareTool.php

@@ -1,227 +0,0 @@
-<?php
-class MalwareTool
-{
-    const ZIP_PASSWORD = 'infected';
-    const ADVANCED_EXTRACTION_SCRIPT_PATH = APP . 'files/scripts/generate_file_objects.py';
-
-    /**
-     * @param string $originalFilename
-     * @param string $content
-     * @param string $md5
-     * @return string Content of zipped file
-     * @throws Exception
-     */
-    public function encrypt($originalFilename, $content, $md5)
-    {
-        if (method_exists("ZipArchive", "setEncryptionName")) {
-            // When PHP zip extension is installed and supports creating encrypted archives.
-            return $this->encryptByExtension($originalFilename, $content, $md5);
-        } else {
-            return $this->encryptByCommand($originalFilename, $content, $md5);
-        }
-    }
-
-    /**
-     * @param string $originalFilename
-     * @param string $content
-     * @param string $md5
-     * @return string Content of zipped file
-     * @throws Exception
-     */
-    private function encryptByCommand($originalFilename, $content, $md5)
-    {
-        $tempDir = $this->tempDir();
-
-        $contentsFile = new File($tempDir . DS . $md5, true);
-        if (!$contentsFile->write($content)) {
-            throw new Exception("Could not write content to file '{$contentsFile->path}'.");
-        }
-        $contentsFile->close();
-
-        $fileNameFile = new File($tempDir . DS . $md5 . '.filename.txt', true);
-        if (!$fileNameFile->write($originalFilename)) {
-            throw new Exception("Could not write original file name to file '{$fileNameFile->path}'.");
-        }
-        $fileNameFile->close();
-
-        $zipFile = new File($tempDir . DS . $md5 . '.zip');
-
-        $exec = [
-            'zip',
-            '-j', // junk (don't record) directory names
-            '-P', // use standard encryption
-            self::ZIP_PASSWORD,
-            escapeshellarg($zipFile->path),
-            escapeshellarg($contentsFile->path),
-            escapeshellarg($fileNameFile->path),
-        ];
-
-        try {
-            $this->execute($exec);
-            $zipContent = $zipFile->read();
-            if ($zipContent === false) {
-                throw new Exception("Could not read content of newly created ZIP file.");
-            }
-
-            return $zipContent;
-
-        } catch (Exception $e) {
-            throw new Exception("Could not create encrypted ZIP file '{$zipFile->path}'.", 0, $e);
-
-        } finally {
-            $fileNameFile->delete();
-            $contentsFile->delete();
-            $zipFile->delete();
-        }
-    }
-
-    /**
-     * @param string $originalFilename
-     * @param string $content
-     * @param string $md5
-     * @return string Content of zipped file
-     * @throws Exception
-     */
-    private function encryptByExtension($originalFilename, $content, $md5)
-    {
-        $zipFilePath = $this->tempFileName();
-
-        $zip = new ZipArchive();
-        $result = $zip->open($zipFilePath, ZipArchive::CREATE);
-        if ($result === true) {
-            $zip->setPassword(self::ZIP_PASSWORD);
-
-            $zip->addFromString($md5, $content);
-            $zip->setEncryptionName($md5, ZipArchive::EM_AES_128);
-
-            $zip->addFromString("$md5.filename.txt", $originalFilename);
-            $zip->setEncryptionName("$md5.filename.txt", ZipArchive::EM_AES_128);
-
-            $zip->close();
-        } else {
-            throw new Exception("Could not create encrypted ZIP file '$zipFilePath'. Error code: $result");
-        }
-
-        $zipFile = new File($zipFilePath);
-        $zipContent = $zipFile->read();
-        if ($zipContent === false) {
-            throw new Exception("Could not read content of newly created ZIP file.");
-        }
-        $zipFile->delete();
-
-        return $zipContent;
-    }
-
-    /**
-     * @param string $content
-     * @param array $hashTypes
-     * @return array
-     * @throws InvalidArgumentException
-     */
-    public function computeHashes($content, array $hashTypes = array())
-    {
-        $validHashes = array('md5', 'sha1', 'sha256');
-        $hashes = [];
-        foreach ($hashTypes as $hashType) {
-            if (!in_array($hashType, $validHashes)) {
-                throw new InvalidArgumentException("Hash type '$hashType' is not valid hash type.");
-            }
-            $hashes[$hashType] = hash($hashType, $content);
-        }
-        return $hashes;
-    }
-
-    /**
-     * @param string $pythonBin
-     * @param string $filePath
-     * @return array
-     * @throws Exception
-     */
-    public function advancedExtraction($pythonBin, $filePath)
-    {
-        return $this->executeAndParseJsonOutput([
-            $pythonBin,
-            self::ADVANCED_EXTRACTION_SCRIPT_PATH,
-            '-p',
-            escapeshellarg($filePath),
-        ]);
-    }
-
-    /**
-     * @param string $pythonBin
-     * @return array
-     * @throws Exception
-     */
-    public function checkAdvancedExtractionStatus($pythonBin)
-    {
-        return $this->executeAndParseJsonOutput([$pythonBin, self::ADVANCED_EXTRACTION_SCRIPT_PATH, '-c']);
-    }
-
-    private function tempFileName()
-    {
-        $randomName = (new RandomTool())->random_str(false, 12);
-        return $this->tempDir() . DS . $randomName;
-    }
-
-    /**
-     * @return string
-     */
-    private function tempDir()
-    {
-        return Configure::read('MISP.tmpdir') ?: sys_get_temp_dir();
-    }
-
-    /**
-     * @param array $command
-     * @return array
-     * @throws Exception
-     */
-    private function executeAndParseJsonOutput(array $command)
-    {
-        $output = $this->execute($command);
-
-        $json = json_decode($output, true);
-        if ($json === null) {
-            throw new Exception("Command output is not valid JSON: " . json_last_error_msg());
-        }
-        return $json;
-    }
-
-    /**
-     * This method is much more complicated than just `exec`, but it also provide stderr output, so Exceptions
-     * can be much more specific.
-     *
-     * @param array $command
-     * @return string
-     * @throws Exception
-     */
-    private function execute(array $command)
-    {
-        $descriptorspec = [
-            1 => ["pipe", "w"], // stdout
-            2 => ["pipe", "w"], // stderr
-        ];
-
-        $command = implode(' ', $command);
-        $process = proc_open($command, $descriptorspec, $pipes);
-        if (!$process) {
-            throw new Exception("Command '$command' could be started.");
-        }
-
-        $stdout = stream_get_contents($pipes[1]);
-        if ($stdout === false) {
-            throw new Exception("Could not get STDOUT of command.");
-        }
-        fclose($pipes[1]);
-
-        $stderr = stream_get_contents($pipes[2]);
-        fclose($pipes[2]);
-
-        $returnCode = proc_close($process);
-        if ($returnCode !== 0) {
-            throw new Exception("Command '$command' return error code $returnCode. STDERR: '$stderr', STDOUT: '$stdout'");
-        }
-
-        return $stdout;
-    }
-}

app/Lib/Tools/SendEmail.php

@@ -0,0 +1,866 @@
+<?php
+App::uses('CakeEmail', 'Network/Email');
+
+class SendEmailException extends Exception {}
+
+/**
+ * Class CakeEmailExtended
+ *
+ * Extends `CakeEmail` to implement RFC 4880 and 3156.
+ *
+ * @see https://dkg.fifthhorseman.net/notes/inline-pgp-harmful/
+ * @see https://www.dalesandro.net/create-self-signed-smime-certificates/
+ */
+class CakeEmailExtended extends CakeEmail
+{
+    /**
+     * @var MimeMultipart|MessagePart
+     */
+    private $body;
+
+    /**
+     * @param array $include
+     * @return array
+     */
+    public function getHeaders($include = array())
+    {
+        $headers = parent::getHeaders($include);
+
+        if ($this->body instanceof MimeMultipart) {
+            $headers['Content-Type'] = $this->body->getContentType();
+        } else if ($this->body instanceof MessagePart) {
+            $headers = array_merge($headers, $this->body->getHeaders());
+        } else {
+            $headers['Content-Type'] = 'multipart/mixed; boundary="' . $this->boundary() . '"';
+        }
+
+        return $headers;
+    }
+
+    /**
+     * @return string|null
+     */
+   public function boundary()
+   {
+       if ($this->body instanceof MimeMultipart) {
+           return $this->body->boundary();
+       }
+
+       return $this->_boundary;
+   }
+
+    /**
+     * @param string|null|MimeMultipart|MessagePart $message
+     * @return string|null|MimeMultipart|MessagePart|CakeEmailExtended
+     */
+    public function body($message = null)
+    {
+        if ($message === null) {
+            return $this->body;
+        }
+        $this->body = $message;
+        return $this;
+    }
+
+    /**
+     * @return array
+     */
+    public function render()
+    {
+        if ($this->body instanceof MimeMultipart) {
+            return $this->body->render();
+        } else if ($this->body instanceof MessagePart) {
+            return $this->body->render(false);
+        }
+
+        return  $this->_render($this->_wrap($this->body));
+    }
+
+    // This is hack how to force CakeEmail to always generate multipart message.
+    protected function _renderTemplates($content)
+    {
+        $this->_boundary = md5(uniqid());
+        $output = parent::_renderTemplates($content);
+        $output[''] = '';
+        return $output;
+    }
+
+    protected function _render($content)
+    {
+        if ($this->body instanceof MimeMultipart) {
+            return $this->body->render();
+        } else if ($this->body instanceof MessagePart) {
+            return $this->body->render(false);
+        }
+
+        return parent::_render($content);
+    }
+
+    public function send($content = null)
+    {
+        if ($content !== null) {
+            throw new InvalidArgumentException("Content must be null for CakeEmailExtended.");
+        }
+        return parent::send($this->body);
+    }
+
+    public function __toString()
+    {
+        return implode("\n", $this->render());
+    }
+}
+
+class MimeMultipart
+{
+    /**
+     * @var MessagePart[]
+     */
+    private $parts = array();
+
+    /**
+     * @var string
+     */
+    private $subtype;
+
+    /**
+     * @var string
+     */
+    private $boundary;
+
+    /**
+     * @var array
+     */
+    private $additionalTypes;
+
+    /**
+     * @param string $subtype
+     * @param array $additionalTypes
+     */
+    public function __construct($subtype = 'mixed', $additionalTypes = array())
+    {
+        $this->subtype = $subtype;
+        $this->boundary = md5(uniqid());
+        $this->additionalTypes = $additionalTypes;
+    }
+
+    /**
+     * @return string
+     */
+    public function getContentType()
+    {
+        $contentType = array_merge(array('multipart/' . $this->subtype), $this->additionalTypes);
+        $contentType[] = 'boundary="' . $this->boundary . '"';
+        return implode('; ', $contentType);
+    }
+
+    public function boundary()
+    {
+        return $this->boundary;
+    }
+
+    public function addPart(MessagePart $part)
+    {
+        $this->parts[] = $part;
+    }
+
+    /**
+     * @return array
+     */
+    public function render()
+    {
+        $msg = array('--' . $this->boundary);
+        foreach ($this->parts as $part) {
+            $msg = array_merge($msg, $part->render());
+            $msg[] = '--' . $this->boundary;
+        }
+        $msg[count($msg) - 1] .= '--'; // last boundary
+        return $msg;
+    }
+
+    public function __toString()
+    {
+        return implode("\n", $this->render());
+    }
+}
+
+class MessagePart
+{
+    /**
+     * @var array
+     */
+    private $headers = array();
+
+    /**
+     * @var array
+     */
+    private $payload;
+
+    /**
+     * @param string $name
+     * @param string|array $value
+     */
+    public function addHeader($name, $value)
+    {
+        if (is_array($value)) {
+            $value = implode('; ', $value);
+        }
+
+        $this->headers[$name] = $value;
+    }
+
+    /**
+     * @return array
+     */
+    public function getHeaders()
+    {
+        return $this->headers;
+    }
+
+    /**
+     * @param array|string $payload
+     */
+    public function setPayload($payload)
+    {
+        if (is_string($payload)) {
+            $payload = explode("\n", $payload);
+        }
+
+        $this->payload = $payload;
+    }
+
+    /**
+     * @param bool $withHeaders
+     * @return array
+     */
+    public function render($withHeaders = true)
+    {
+        $msg = array();
+        if ($withHeaders) {
+            foreach ($this->headers as $name => $value) {
+                $msg[] = "$name: $value";
+            }
+            $msg[] = '';
+        }
+        return array_merge($msg, $this->payload);
+    }
+
+    public function __toString()
+    {
+        return implode("\n", $this->render());
+    }
+}
+
+class SendEmail
+{
+    /**
+     * @var CryptGpgExtended
+     */
+    private $gpg;
+
+    /**
+     * @param CryptGpgExtended|null $gpg
+     */
+    public function __construct(CryptGpgExtended $gpg = null)
+    {
+        if ($gpg) {
+            $gpg->clearDecryptKeys()
+                ->clearEncryptKeys()
+                ->clearSignKeys()
+                ->clearPassphrases();
+            $this->gpg = $gpg;
+        }
+    }
+
+    /**
+     * @param array $params
+     * @return array|bool
+     * @throws Crypt_GPG_Exception
+     * @throws SendEmailException
+     */
+    public function sendExternal(array $params)
+    {
+        foreach (array('body', 'reply-to', 'to', 'subject', 'text') as $requiredParam) {
+            if (!isset($params[$requiredParam])) {
+                throw new InvalidArgumentException("Param '$requiredParam' is required, but not provided.");
+            }
+        }
+
+        $params['body'] = str_replace('\n', PHP_EOL, $params['body']); // TODO: Why this?
+
+        $attachments = array();
+        if (!empty($params['requestor_gpgkey'])) {
+            $attachments['gpgkey.asc'] = array(
+                'data' => $params['requestor_gpgkey']
+            );
+        }
+
+        if (!empty($params['attachments'])) {
+            foreach ($params['attachments'] as $key => $value) {
+                $attachments[$key] = array('data' => $value);
+            }
+        }
+
+        $email = new CakeEmailExtended();
+        $email->replyTo($params['reply-to']);
+        $email->from(Configure::read('MISP.email'));
+        $email->returnPath(Configure::read('MISP.email'));
+        $email->to($params['to']);
+        $email->subject($params['subject']);
+        $email->emailFormat('text');
+        $email->body($params['body']);
+        $email->attachments($attachments);
+
+        $mock = false;
+        if (!empty(Configure::read('MISP.disable_emailing')) || !empty($params['mock'])) {
+            $email->transport('Debug');
+            $mock = true;
+        }
+
+        if (!empty($params['gpgkey'])) {
+            if (!$this->gpg) {
+                throw new SendEmailException("GPG encryption is enabled, but GPG is not configured.");
+            }
+            
+            try {
+                $fingerprint = $this->importAndValidateGpgPublicKey($params['gpgkey']);
+            } catch (Crypt_GPG_NoDataException $e) {
+                throw new SendEmailException("The message could not be encrypted because the provided key is invalid.", 0, $e);
+            }
+            if (!$fingerprint) {
+                throw new SendEmailException("The message could not be encrypted because the provided key is either expired or cannot be used for encryption.");
+            }
+            try {
+                $this->gpg->addEncryptKey($fingerprint);
+                $this->encryptByGpg($email);
+            } catch (Exception $e) {
+                throw new SendEmailException("The message could not be encrypted.", 0, $e);
+            }
+        }
+
+        try {
+            $result = $email->send();
+        } catch (Exception $e) {
+            throw new SendEmailException("The message could not be sent.", 0, $e);
+        }
+
+        if ($result && !$mock) {
+            return true;
+        }
+        return $result;
+    }
+
+    /**
+     * @param array $user
+     * @param string $subject
+     * @param string $body
+     * @param string|null $bodyWithoutEncryption
+     * @param array $replyToUser
+     * @return bool True if e-mail is encrypted, false if not.
+     * @throws Crypt_GPG_BadPassphraseException
+     * @throws Crypt_GPG_Exception
+     * @throws SendEmailException
+     */
+    public function sendToUser(array $user, $subject, $body, $bodyWithoutEncryption = null, array $replyToUser = array())
+    {
+        if (Configure::read('MISP.disable_emailing')) {
+            throw new SendEmailException('Emailing is currently disabled on this instance.');
+        }
+
+        if (!isset($user['User'])) {
+            throw new InvalidArgumentException("Invalid user model provided.");
+        }
+
+        // Check if the e-mail can be encrypted
+        $canEncryptGpg = isset($user['User']['gpgkey']) && !empty($user['User']['gpgkey']);
+        $canEncryptSmime = isset($user['User']['certif_public']) && !empty($user['User']['certif_public']) && Configure::read('SMIME.enabled');
+
+        if (Configure::read('GnuPG.onlyencrypted') && !$canEncryptGpg && !$canEncryptSmime) {
+            throw new SendEmailException('Encrypted messages are enforced and the message could not be encrypted for this user as no valid encryption key was found.');
+        }
+
+        // If 'bodyonlyencrypted' is enabled and the user has no encryption key, use the alternate body (if it exists)
+        if (Configure::read('GnuPG.bodyonlyencrypted') && !$canEncryptSmime && !$canEncryptGpg && $bodyWithoutEncryption) {
+            $body = $bodyWithoutEncryption;
+        }
+
+        $body = str_replace('\n', PHP_EOL, $body); // TODO: Why this?
+
+        $email = $this->create($user, $subject, $body, array(), $replyToUser);
+
+        $signed = false;
+        if (Configure::read('GnuPG.sign')) {
+            if (!$this->gpg) {
+                throw new SendEmailException("GPG signing is enabled, but GPG is not initialized. Check debug log why GPG could not be initialized.");
+            }
+
+            try {
+                $gnupgEmail = Configure::read('GnuPG.email');
+                if (empty($gnupgEmail)) {
+                    throw new Exception("Email signing is enabled but variable 'GnuPG.email' is not set.");
+                }
+
+                $this->gpg->addSignKey($gnupgEmail, Configure::read('GnuPG.password'));
+                $this->signByGpg($email, $replyToUser);
+                $email->addHeaders(array('Autocrypt' => $this->generateAutocrypt($gnupgEmail)));
+                $this->gpg->clearSignKeys();
+
+                $signed = true;
+            } catch (Exception $e) {
+                throw new SendEmailException("The message could not be signed.", 0, $e);
+            }
+        }
+
+        $encrypted = false;
+        if ($canEncryptGpg) {
+            if (!$this->gpg) {
+                throw new SendEmailException("GPG signing is enabled, but GPG is not initialized. Check debug log why GPG could not be initialized.");
+            }
+            
+            try {
+                $fingerprint = $this->importAndValidateGpgPublicKey($user['User']['gpgkey']);
+            } catch (Crypt_GPG_NoDataException $e) {
+                throw new SendEmailException("The message could not be encrypted because the provided key is invalid.", 0, $e);
+            }
+
+            if (!$fingerprint) {
+                throw new SendEmailException("The message could not be encrypted because the provided key is either expired or cannot be used for encryption.");
+            }
+
+            try {
+                $this->gpg->addEncryptKey($fingerprint);
+                $this->encryptByGpg($email);
+                $this->gpg->clearEncryptKeys();
+
+                if ($signed && Configure::read('GnuPG.obscure_subject')) {
+                    // If message is signed, we can remove subject from unencrypted part of email and replace with '...',
+                    // because subject is also part of signed data. Three dots are used according to
+                    // 'draft-autocrypt-lamps-protected-headers-01' standard. This behaviour must be enabled by
+                    // 'GnuPG.obscure_subject' setting.
+                    $email->subject('...');
+                }
+
+                $encrypted = true;
+            } catch (Exception $e) {
+                throw new SendEmailException('The message could not be encrypted.', 0, $e);
+            }
+        }
+
+        if (!$canEncryptGpg && $canEncryptSmime) {
+            $this->signBySmime($email);
+            $this->encryptBySmime($email, $user['User']['certif_public']);
+            $encrypted = true;
+        }
+
+        try {
+            $email->send();
+            return $encrypted;
+        } catch (Exception $e) {
+            throw new SendEmailException('The message could not be sent.', 0, $e);
+        }
+    }
+
+    /**
+     * Test if S/MIME certificate is valid for email encrypting.
+     *
+     * @param string $certificate
+     * @return bool
+     * @throws Exception
+     */
+    public function testSmimeCertificate($certificate)
+    {
+        try {
+            // Try to encrypt empty message
+            $this->encryptTextBySmime($certificate, '');
+        } catch (SendEmailException $e) {
+            throw new Exception('This certificate cannot be used to encrypt email.', 0, $e);
+        }
+
+        $parsed = openssl_x509_parse($certificate);
+
+        if (!$parsed) {
+            throw new Exception('Could not parse certificate');
+        }
+
+        // Purpose '5' should be 'smimeencrypt'
+        if (!($parsed['purposes'][5][0] === 1 && $parsed['purposes'][5][2] === 'smimeencrypt')) {
+            throw new Exception('This certificate cannot be used to encrypt email.');
+        }
+
+        $now = new DateTime();
+        $validToTime = new DateTime("@{$parsed['validTo_time_t']}");
+        if ($validToTime <= $now) {
+            throw new Exception('This certificate is expired.');
+        }
+
+        return true;
+    }
+
+    /**
+     * @param array $user User model
+     * @param string $subject
+     * @param string $body
+     * @param array $attachments
+     * @param array $replyToUser User model
+     * @return CakeEmailExtended
+     */
+    private function create(array $user, $subject, $body, array $attachments = array(), array $replyToUser = array())
+    {
+        $email = new CakeEmailExtended();
+
+        // We must generate message ID by own, because CakeEmail returns different message ID for every call of
+        // getHeaders() method.
+        $email->messageId($this->generateMessageId($email));
+        // The same problem is with 'Date' header, that we need to protect by GPG signature.
+        $email->addHeaders(array('Date' => date(DATE_RFC2822)));
+
+        // If the e-mail is sent on behalf of a user, then we want the target user to be able to respond to the sender.
+        // For this reason we should also attach the public key of the sender along with the message (if applicable).
+        if ($replyToUser) {
+            $email->replyTo($replyToUser['User']['email']);
+            if (!empty($replyToUser['User']['gpgkey'])) {
+                $attachments['gpgkey.asc'] = $replyToUser['User']['gpgkey'];
+            } elseif (!empty($replyToUser['User']['certif_public'])) {
+                $attachments[$replyToUser['User']['email'] . '.pem'] = $replyToUser['User']['certif_public'];
+            }
+        } else if (Configure::read('MISP.email_reply_to')) {
+            $email->replyTo(Configure::read('MISP.email_reply_to'));
+        }
+
+        $email->from(Configure::read('MISP.email'));
+        $email->returnPath(Configure::read('MISP.email')); // TODO?
+        $email->to($user['User']['email']);
+        $email->subject($subject);
+        $email->emailFormat('text');
+        $email->body($body);
+
+        foreach ($attachments as $key => $value) {
+            $attachments[$key] = array('data' => $value);
+        }
+        $email->attachments($attachments);
+
+        return $email;
+    }
+
+    /**
+     * @param CakeEmailExtended $email
+     * @param array $replyToUser
+     * @throws Crypt_GPG_BadPassphraseException
+     * @throws Crypt_GPG_Exception
+     * @throws Crypt_GPG_KeyNotFoundException
+     */
+    private function signByGpg(CakeEmailExtended $email, array $replyToUser = array())
+    {
+        $renderedEmail = $email->render();
+
+        $messagePart = new MessagePart();
+        $messagePart->addHeader('Content-Type', array(
+            'multipart/mixed',
+            'boundary="' . $email->boundary() . '"',
+            'protected-headers="v1"',
+        ));
+
+        // Protect User-Facing Headers according to https://tools.ietf.org/id/draft-autocrypt-lamps-protected-headers-01.html
+        $originalHeaders = $email->getHeaders(array('subject', 'from', 'to'));
+        $protectedHeaders = array('From', 'To', 'Date', 'Message-ID', 'Subject', 'Reply-To');
+        foreach ($protectedHeaders as $header) {
+            if (isset($originalHeaders[$header])) {
+                $messagePart->addHeader($header, $originalHeaders[$header]);
+            }
+        }
+
+        // If the e-mail is sent on behalf of a user and that user has assigned GPG key, we will send his public key
+        // in signed autocrypt header.
+        if ($replyToUser) {
+            if (!empty($replyToUser['User']['gpgkey'])) {
+                $autocrypt = $this->generateAutocrypt($replyToUser['User']['email'], $replyToUser['User']['gpgkey'], false);
+                $messagePart->addHeader('Autocrypt-Gossip', $autocrypt);
+            }
+        } else if (Configure::read('MISP.email_reply_to')) {
+            $autocrypt = $this->generateAutocrypt(Configure::read('MISP.email_reply_to'), null, false);
+            if ($autocrypt) {
+                $messagePart->addHeader('Autocrypt-Gossip', $autocrypt);
+            }
+        }
+
+        $messagePart->setPayload($renderedEmail);
+
+        // GPG message to sign must be delimited by <CR><LF>
+        $messageToSign = implode("\r\n", $messagePart->render());
+        $signature = $this->gpg->sign($messageToSign, Crypt_GPG::SIGN_MODE_DETACHED);
+        $signatureInfo = $this->gpg->getLastSignatureInfo();
+
+        $signaturePart = new MessagePart();
+        $signaturePart->addHeader('Content-Type', array('application/pgp-signature', 'name="signature.asc"'));
+        $signaturePart->addHeader('Content-Description', 'OpenPGP digital signature');
+        $signaturePart->addHeader('Content-Disposition', array('attachment', 'filename="signature.asc"'));
+        $signaturePart->setPayload($signature);
+
+        $output = new MimeMultipart('signed', array(
+            "micalg=pgp-{$signatureInfo->getHashAlgorithmName()}",
+            'protocol="application/pgp-signature"'
+        ));
+        $output->addPart($messagePart);
+        $output->addPart($signaturePart);
+
+        $email->body($output);
+    }
+
+    /**
+     * @param CakeEmailExtended $email
+     * @throws Crypt_GPG_Exception
+     * @throws Crypt_GPG_KeyNotFoundException
+     */
+    private function encryptByGpg(CakeEmailExtended $email)
+    {
+        $versionPart = new MessagePart();
+        $versionPart->addHeader('Content-Type', 'application/pgp-encrypted');
+        $versionPart->addHeader('Content-Description', 'PGP/MIME version identification');
+        $versionPart->setPayload("Version 1\n");
+
+        $rendered = $email->render();
+
+        $messagePart = new MessagePart();
+        $messagePart->addHeader('Content-Type', $email->getHeaders()['Content-Type']);
+        $messagePart->setPayload($rendered);
+        $rendered = $messagePart->render();
+
+        $messageToEncrypt = implode("\r\n", $rendered);
+        $encrypted = $this->gpg->encrypt($messageToEncrypt, true);
+
+        $encryptedPart = new MessagePart();
+        $encryptedPart->addHeader('Content-Type', array('application/octet-stream', 'name="encrypted.asc"'));
+        $encryptedPart->addHeader('Content-Description', 'OpenPGP encrypted message');
+        $encryptedPart->addHeader('Content-Disposition', array('inline', 'filename="encrypted.asc"'));
+        $encryptedPart->setPayload($encrypted);
+
+        $output = new MimeMultipart('encrypted', array('protocol="application/pgp-encrypted"'));
+        $output->addPart($versionPart);
+        $output->addPart($encryptedPart);
+
+        $email->body($output);
+    }
+
+    /**
+     * @param CakeEmailExtended $email
+     * @throws SendEmailException
+     */
+    private function signBySmime(CakeEmailExtended $email)
+    {
+        $renderedEmail = $email->render();
+
+        $messagePart = new MessagePart();
+        $messagePart->addHeader('Content-Type', array(
+            'multipart/mixed',
+            'boundary="' . $email->boundary() . '"',
+        ));
+        $messagePart->setPayload($renderedEmail);
+
+        $signaturePart = new MessagePart();
+        $signaturePart->addHeader('Content-Type', array('application/pkcs7-signature', 'name="smime.p7s"'));
+        $signaturePart->addHeader('Content-Transfer-Encoding', 'base64');
+        $signaturePart->addHeader('Content-Disposition', array('attachment', 'filename="smime.p7s"'));
+        $signaturePart->setPayload($this->signTextBySmime(implode("\r\n", $messagePart->render())));
+
+        $output = new MimeMultipart('signed', array('protocol="application/x-pkcs7-signature"', 'micalg="sha-256"'));
+        $output->addPart($messagePart);
+        $output->addPart($signaturePart);
+
+        $email->body($output);
+    }
+
+    /**
+     * @param CakeEmailExtended $email
+     * @param string $publicKey
+     * @throws SendEmailException
+     */
+    private function encryptBySmime(CakeEmailExtended $email, $publicKey)
+    {
+        $rendered = $email->render();
+
+        $messagePart = new MessagePart();
+        $messagePart->addHeader('Content-Type', $email->getHeaders()['Content-Type']);
+        $messagePart->setPayload($rendered);
+        $rendered = $messagePart->render();
+
+        $encrypted = $this->encryptTextBySmime($publicKey, implode("\r\n", $rendered));
+
+        $messagePart = new MessagePart();
+        $messagePart->addHeader('Content-Transfer-Encoding', 'base64');
+        $messagePart->addHeader('Content-Type', 'application/pkcs7-mime; name="smime.p7m"; smime-type="enveloped-data"');
+        $messagePart->addHeader('Content-Disposition', 'attachment; filename="smime.p7m"');
+        $messagePart->addHeader('Content-Description', 'S/MIME Encrypted Message');
+        $messagePart->setPayload($encrypted);
+
+        $email->body($messagePart);
+    }
+
+    /**
+     * @param string $body
+     * @return false|string
+     * @throws SendEmailException
+     */
+    private function signTextBySmime($body)
+    {
+        $certPublicSignPath = Configure::read('SMIME.cert_public_sign');
+        $keySignPath = Configure::read('SMIME.key_sign');
+        if (empty($certPublicSignPath)) {
+            throw new SendEmailException("Configuration value 'SMIME.cert_public_sign' is not defined.");
+        }
+        if (empty($keySignPath)) {
+            throw new SendEmailException("Configuration value 'SMIME.key_sign' is not defined.");
+        }
+        if (!is_readable($certPublicSignPath)) {
+            throw new SendEmailException("Certification file '$certPublicSignPath' is not readable.");
+        }
+        if (!is_readable($keySignPath)) {
+            throw new SendEmailException("Sign key file '$keySignPath' is not readable.");
+        }
+        $certPublicSign = openssl_x509_read(file_get_contents($certPublicSignPath));
+        if (!$certPublicSign) {
+            throw new SendEmailException("Certification file '$certPublicSignPath' is not valid X.509 file: " . openssl_error_string());
+        }
+        $keySign = openssl_pkey_get_private(file_get_contents($keySignPath), Configure::read('SMIME.password'));
+        if (!$keySign) {
+            throw new SendEmailException("Sign key file '$keySignPath' is not valid private key file: " . openssl_error_string());
+        }
+
+        list($inputFile, $outputFile) = $this->createInputOutputFiles($body);
+        $result = openssl_pkcs7_sign($inputFile->pwd(), $outputFile->pwd(), $certPublicSign, $keySign, array(), 0);
+        $inputFile->delete();
+
+        if ($result) {
+            $data = $outputFile->read();
+            $outputFile->delete();
+            $parts = explode("\n\n", $data);
+            return $parts[1] . "\n";
+
+        } else {
+            $outputFile->delete();
+            throw new SendEmailException('Failed while attempting to sign the S/MIME message: ' . openssl_error_string());
+        }
+    }
+
+    /**
+     * @param string $publicKey
+     * @param string $body
+     * @return string
+     * @throws SendEmailException
+     */
+    private function encryptTextBySmime($publicKey, $body)
+    {
+        $publicKey = openssl_x509_read($publicKey);
+        if (!$publicKey) {
+            throw new SendEmailException('Certification file is not valid X.509 file: ' . openssl_error_string());
+        }
+
+        list($inputFile, $outputFile) = $this->createInputOutputFiles($body);
+        $result = openssl_pkcs7_encrypt($inputFile->pwd(), $outputFile->pwd(), $publicKey, array(), 0, OPENSSL_CIPHER_AES_256_CBC);
+        $inputFile->delete();
+
+        if ($result) {
+            $encryptedBody = $outputFile->read();
+            $outputFile->delete();
+            $parts = explode("\n\n", $encryptedBody);
+            return $parts[1];
+        } else {
+            $outputFile->delete();
+            throw new SendEmailException('Could not encrypt the S/MIME message: ' . openssl_error_string());
+        }
+    }
+
+    /**
+     * @param string $content
+     * @return File[]
+     * @throws SendEmailException
+     */
+    private function createInputOutputFiles($content)
+    {
+        $dir = APP . 'tmp' . DS . 'SMIME';
+        if (!file_exists($dir)) {
+            if (!mkdir($dir, 0750, true)) {
+                throw new SendEmailException("The SMIME temp directory '$dir' is not writeable.");
+            }
+        }
+
+        App::uses('FileAccessTool', 'Tools');
+        $fileAccessTool = new FileAccessTool();
+        $inputFile = $fileAccessTool->createTempFile($dir, 'SMIME');
+        $fileAccessTool->writeToFile($inputFile, $content);
+
+        $outputFile = $fileAccessTool->createTempFile($dir, 'SMIME');
+        return array(new File($inputFile), new File($outputFile));
+    }
+
+    /**
+     * Check if public key is not expired and can encrypt.
+     *
+     * @param string $gpgKey
+     * @return string|bool Fingerprint if key is valid, false otherwise.
+     * @throws Crypt_GPG_BadPassphraseException
+     * @throws Crypt_GPG_Exception
+     * @throws Crypt_GPG_NoDataException
+     */
+    private function importAndValidateGpgPublicKey($gpgKey)
+    {
+        $keyImportOutput = $this->gpg->importKey($gpgKey);
+        $key = $this->gpg->getKeys($keyImportOutput['fingerprint']);
+        $subKeys = $key[0]->getSubKeys();
+        $currentTimestamp = time();
+
+        foreach ($subKeys as $subKey) {
+            $expiration = $subKey->getExpirationDate();
+            if (($expiration == 0 || $currentTimestamp < $expiration) && $subKey->canEncrypt()) {
+                // key is valid, return fingerprint
+                return $keyImportOutput['fingerprint'];
+            }
+        }
+
+        return false;
+    }
+
+    /**
+     * This method generates Message-ID (RFC 2392).
+     *
+     * @param CakeEmail $email
+     * @return string
+     */
+    private function generateMessageId(CakeEmail $email)
+    {
+        $uuid = str_replace('-', '', CakeText::uuid());
+        return "<$uuid@{$email->domain()}>";
+    }
+
+    /**
+     * Generates Autocrypt header.
+     *
+     * If $gpgKey is not provided, GPG will try to find correct key by given e-mail address. If no key found, `null` is
+     * returned.
+     *
+     * @see https://autocrypt.org/level1.html
+     * @param string $address
+     * @param string|null $gpgKey
+     * @param bool $preferEncrypt
+     * @return string|null
+     * @throws Crypt_GPG_Exception
+     */
+    private function generateAutocrypt($address, $gpgKey = null, $preferEncrypt = true)
+    {
+        if ($gpgKey) {
+            $keyImportOutput = $this->gpg->importKey($gpgKey);
+            $keyData = $this->gpg->exportPublicKeyMinimal($keyImportOutput['fingerprint'], false);
+        } else {
+            try {
+                $keyData = $this->gpg->exportPublicKeyMinimal($address, false);
+            } catch (Crypt_GPG_KeyNotFoundException $e) {
+                return null;
+            }
+        }
+
+        $parts = array("addr=$address");
+        if ($preferEncrypt) {
+            $parts[] = 'prefer-encrypt=mutual';
+        }
+        $parts[] = 'keydata=' . base64_encode($keyData);
+        return implode('; ', $parts);
+    }
+}

app/Model/Allowedlist.php

@@ -2,9 +2,9 @@
 
 App::uses('AppModel', 'Model');
 
-class Whitelist extends AppModel
+class Allowedlist extends AppModel
 {
-    public $useTable = 'whitelist';
+    public $useTable = 'allowedlist';
 
     public $displayField = 'name';
 
@@ -17,7 +17,7 @@ class Whitelist extends AppModel
             ),
     );
 
-    public $whitelistedItems = false;
+    public $allowedlistedItems = false;
 
     public $validate = array(
         'name' => array(
@@ -26,7 +26,7 @@ class Whitelist extends AppModel
             ),
             'userdefined' => array(
                 'rule' => array('validateValue'),
-                'message' => 'Name not in the right format. Whitelist entries have to be enclosed by a valid php delimiter (which can be most non-alphanumeric / non-whitespace character). Format: "/8.8.8.8/" Please double check the name.',             //'allowEmpty' => false,
+                'message' => 'Name not in the right format. Allowedlist entries have to be enclosed by a valid php delimiter (which can be most non-alphanumeric / non-whitespace character). Format: "/8.8.8.8/" Please double check the name.',             //'allowEmpty' => false,
                 //'allowEmpty' => false,
                 //'required' => true,
                 //'last' => false, // Stop validation after this rule
@@ -56,9 +56,9 @@ class Whitelist extends AppModel
     {
         $value = $fields['name'];
 
-        $whitelist = $this->find('all', array('recursive' => 0,'fields' => 'name'));
-        foreach ($whitelist as $whitelistItem) {
-            if ($value == $whitelistItem['Whitelist']['name']) {
+        $allowedlist = $this->find('all', array('recursive' => 0,'fields' => 'name'));
+        foreach ($allowedlist as $allowedlistItem) {
+            if ($value == $allowedlistItem['Allowedlist']['name']) {
                 return false;
             }
         }
@@ -68,28 +68,28 @@ class Whitelist extends AppModel
 
     public function getBlockedValues()
     {
-        if ($this->whitelistedItems === false) {
-            $Whitelists = $this->find('all', array('fields' => array('name')));
-            $this->whitelistedItems = array();
-            foreach ($Whitelists as $item) {
-                $this->whitelistedItems[] = $item['Whitelist']['name'];
+        if ($this->allowedlistedItems === false) {
+            $Allowedlists = $this->find('all', array('fields' => array('name')));
+            $this->allowedlistedItems = array();
+            foreach ($Allowedlists as $item) {
+                $this->allowedlistedItems[] = $item['Allowedlist']['name'];
             }
         }
-        return $this->whitelistedItems;
+        return $this->allowedlistedItems;
     }
 
-    public function removeWhitelistedFromArray($data, $isAttributeArray)
+    public function removeAllowedlistedFromArray($data, $isAttributeArray)
     {
-        // Let's get all of the values that will be blocked by the whitelist
-        $whitelists = $this->getBlockedValues();
-        // if we don't have any whitelist items in the db, don't loop through each attribute
-        if (!empty($whitelists)) {
+        // Let's get all of the values that will be blocked by the allowedlist
+        $allowedlists = $this->getBlockedValues();
+        // if we don't have any allowedlist items in the db, don't loop through each attribute
+        if (!empty($allowedlists)) {
             // if $isAttributeArray, we know that we have just an array of attributes
             if ($isAttributeArray) {
-                // loop through each attribute and unset the ones that are whitelisted
+                // loop through each attribute and unset the ones that are allowedlisted
                 foreach ($data as $k => $attribute) {
-                    // loop through each whitelist item and run a preg match against the attribute value. If it matches, unset the attribute
-                    foreach ($whitelists as $wlitem) {
+                    // loop through each allowedlist item and run a preg match against the attribute value. If it matches, unset the attribute
+                    foreach ($allowedlists as $wlitem) {
                         if (preg_match($wlitem, $attribute['Attribute']['value'])) {
                             unset($data[$k]);
                         }
@@ -100,10 +100,10 @@ class Whitelist extends AppModel
                 // if !$isAttributeArray, we know that we have an array of events that we need to parse through
                 foreach ($data as $ke => $event) {
                     if (isset($event['Attribute'])) {
-                        // loop through each attribute and unset the ones that are whitelisted
+                        // loop through each attribute and unset the ones that are allowedlisted
                         foreach ($event['Attribute'] as $k => $attribute) {
-                            // loop through each whitelist item and run a preg match against the attribute value. If it matches, unset the attribute
-                            foreach ($whitelists as $wlitem) {
+                            // loop through each allowedlist item and run a preg match against the attribute value. If it matches, unset the attribute
+                            foreach ($allowedlists as $wlitem) {
                                 if (preg_match($wlitem, $attribute['value'])) {
                                     unset($data[$ke]['Attribute'][$k]);
                                 }
@@ -117,14 +117,14 @@ class Whitelist extends AppModel
         return $data;
     }
 
-    // A simplified whitelist removal, for when we just want to throw values against the list instead of attributes / events
-    public function removeWhitelistedValuesFromArray($data)
+    // A simplified allowedlist removal, for when we just want to throw values against the list instead of attributes / events
+    public function removeAllowedlistedValuesFromArray($data)
     {
-        $whitelists = $this->getBlockedValues();
-        // if we don't have any whitelist items in the db, don't loop through each attribute
-        if (!empty($whitelists)) {
+        $allowedlists = $this->getBlockedValues();
+        // if we don't have any allowedlist items in the db, don't loop through each attribute
+        if (!empty($allowedlists)) {
             foreach ($data as $k => $value) {
-                foreach ($whitelists as $wlitem) {
+                foreach ($allowedlists as $wlitem) {
                     if (preg_match($wlitem, $value)) {
                         unset($data[$k]);
                     }

app/Model/AppModel.php

@@ -37,6 +37,8 @@ class AppModel extends Model
 
     public $start = 0;
 
+    public $assetCache = [];
+
     public $inserted_ids = array();
 
     private $__redisConnection = null;
@@ -44,7 +46,9 @@ class AppModel extends Model
     private $__profiler = array();
 
     public $elasticSearchClient = false;
-    public $s3Client = false;
+
+    /** @var AttachmentTool|null */
+    private $attachmentTool;
 
     public function __construct($id = false, $table = null, $ds = null)
     {
@@ -192,16 +196,16 @@ class AppModel extends Model
                 $this->Sighting->deleteAll(array('NOT' => array('Sighting.type' => array(0, 1, 2))));
                 break;
             case '2.4.71':
-                $this->OrgBlacklist = Classregistry::init('OrgBlacklist');
+                $this->OrgBlocklist = Classregistry::init('OrgBlocklist');
                 $values = array(
                     array('org_uuid' => '58d38339-7b24-4386-b4b4-4c0f950d210f', 'org_name' => 'Setec Astrononomy', 'comment' => 'default example'),
                     array('org_uuid' => '58d38326-eda8-443a-9fa8-4e12950d210f', 'org_name' => 'Acme Finance', 'comment' => 'default example')
                 );
                 foreach ($values as $value) {
-                    $found = $this->OrgBlacklist->find('first', array('conditions' => array('org_uuid' => $value['org_uuid']), 'recursive' => -1));
+                    $found = $this->OrgBlocklist->find('first', array('conditions' => array('org_uuid' => $value['org_uuid']), 'recursive' => -1));
                     if (empty($found)) {
-                        $this->OrgBlacklist->create();
-                        $this->OrgBlacklist->save($value);
+                        $this->OrgBlocklist->create();
+                        $this->OrgBlocklist->save($value);
                     }
                 }
                 $dbUpdateSuccess = $this->updateDatabase($command);
@@ -1389,18 +1393,18 @@ class AppModel extends Model
                 break;
             case 51:
                 $sqlArray[] = "ALTER TABLE `feeds` ADD `orgc_id` int(11) NOT NULL DEFAULT 0";
-                $this->__addIndex('feeds', 'orgc_id');
+                $indexArray[] = array('feeds', 'orgc_id');
                 break;
             case 52:
                 if (!empty($this->query("SHOW COLUMNS FROM `admin_settings` LIKE 'key';"))) {
                     $sqlArray[] = "ALTER TABLE admin_settings CHANGE `key` `setting` varchar(255) COLLATE utf8_bin NOT NULL;";
-                    $this->__addIndex('admin_settings', 'setting');
+                    $indexArray[] = array('admin_settings', 'setting');
                 }
                 break;
             case 53:
                 if (!empty($this->query("SHOW COLUMNS FROM `user_settings` LIKE 'key';"))) {
                     $sqlArray[] = "ALTER TABLE user_settings CHANGE `key` `setting` varchar(255) COLLATE utf8_bin NOT NULL;";
-                    $this->__addIndex('user_settings', 'setting');
+                    $indexArray[] = array('user_settings', 'setting');
                 }
                 break;
             case 54:
@@ -1414,6 +1418,12 @@ class AppModel extends Model
                 $this->__dropIndex('correlations', 'sharing_group_id');
                 $this->__dropIndex('correlations', 'a_sharing_group_id');
                 break;
+            case 56:
+                //rename tables
+                $sqlArray[] = "RENAME TABLE `org_blacklists` TO `org_blocklists`;";
+                $sqlArray[] = "RENAME TABLE `event_blacklists` TO `event_blocklists`;";
+                $sqlArray[] = "RENAME TABLE `whitelist` TO `allowedlist`;";
+                break;
             case 'fixNonEmptySharingGroupID':
                 $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
                 $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
@@ -1567,7 +1577,7 @@ class AppModel extends Model
                             break;
                         }
                     } else {
-                        $logMessage['change'] = $logMessage['change'] . PHP_EOL . __('However, as this error is whitelisted, the update went through.');
+                        $logMessage['change'] = $logMessage['change'] . PHP_EOL . __('However, as this error is allowed, the update went through.');
                     }
                     $this->Log->save($logMessage);
                 }
@@ -2299,9 +2309,9 @@ class AppModel extends Model
         ));
         $counter = 0;
 
-        // load this so we can remove the blacklist item that will be created, this is the one case when we do not want it.
-        if (Configure::read('MISP.enableEventBlacklisting') !== false) {
-            $this->EventBlacklist = ClassRegistry::init('EventBlacklist');
+        // load this so we can remove the blocklist item that will be created, this is the one case when we do not want it.
+        if (Configure::read('MISP.enableEventBlocklisting') !== false) {
+            $this->EventBlocklist = ClassRegistry::init('EventBlocklist');
         }
 
         foreach ($duplicates as $duplicate) {
@@ -2315,10 +2325,10 @@ class AppModel extends Model
                     $this->Event->delete($event['Event']['id']);
                     $this->Log->createLogEntry('SYSTEM', 'delete', 'Event', $event['Event']['id'], __('Removed event (%s)', $event['Event']['id']), __('Event\'s UUID duplicated (%s)', $event['Event']['uuid']));
                     $counter++;
-                    // remove the blacklist entry that we just created with the event deletion, if the feature is enabled
+                    // remove the blocklist entry that we just created with the event deletion, if the feature is enabled
                     // We do not want to block the UUID, since we just deleted a copy
-                    if (Configure::read('MISP.enableEventBlacklisting') !== false) {
-                        $this->EventBlacklist->deleteAll(array('EventBlacklist.event_uuid' => $uuid));
+                    if (Configure::read('MISP.enableEventBlocklisting') !== false) {
+                        $this->EventBlocklist->deleteAll(array('EventBlocklist.event_uuid' => $uuid));
                     }
                 }
             }
@@ -2551,29 +2561,6 @@ class AppModel extends Model
         $this->elasticSearchClient = $client;
     }
 
-    public function getS3Client()
-    {
-        if (!$this->s3Client) {
-            $this->s3Client = $this->loadS3Client();
-        }
-
-        return $this->s3Client;
-    }
-
-    public function loadS3Client()
-    {
-        App::uses('AWSS3Client', 'Tools');
-        $client = new AWSS3Client();
-        $client->initTool();
-        return $client;
-    }
-
-    public function attachmentDirIsS3()
-    {
-        // Naive way to detect if we're working in S3
-        return substr(Configure::read('MISP.attachments_dir'), 0, 2) === "s3";
-    }
-
     public function checkVersionRequirements($versionString, $minVersion)
     {
         $version = explode('.', $versionString);
@@ -3002,6 +2989,8 @@ class AppModel extends Model
     }
 
     /**
+     * Log exception with backtrace and with nested exceptions.
+     *
      * @param string $message
      * @param Exception $exception
      * @param int $type
@@ -3082,4 +3071,16 @@ class AppModel extends Model
             $input
         );
     }
+
+    /**
+     * @return AttachmentTool
+     */
+    protected function loadAttachmentTool()
+    {
+        if ($this->attachmentTool === null) {
+            $this->attachmentTool = new AttachmentTool();
+        }
+
+        return $this->attachmentTool;
+    }
 }

app/Model/Attribute.php

@@ -5,9 +5,12 @@ App::uses('Folder', 'Utility');
 App::uses('File', 'Utility');
 App::uses('FinancialTool', 'Tools');
 App::uses('RandomTool', 'Tools');
-App::uses('MalwareTool', 'Tools');
+App::uses('AttachmentTool', 'Tools');
 App::uses('TmpFileTool', 'Tools');
 
+/**
+ * @property Event $Event
+ */
 class Attribute extends AppModel
 {
     public $combinedKeys = array('event_id', 'category', 'type');
@@ -93,16 +96,16 @@ class Attribute extends AppModel
             'Payload delivery' => array(
                     'desc' => __('Information about how the malware is delivered'),
                     'formdesc' => __('Information about the way the malware payload is initially delivered, for example information about the email or web-site, vulnerability used, originating IP etc. Malware sample itself should be attached here.'),
-                    'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|authentihash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash','filename|impfuzzy', 'filename|pehash', 'mac-address', 'mac-eui-64', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'hostname', 'domain', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'email-body', 'url', 'user-agent', 'AS', 'pattern-in-file', 'pattern-in-traffic', 'stix2-pattern', 'yara', 'sigma', 'mime-type', 'attachment', 'malware-sample', 'link', 'malware-type', 'comment', 'text', 'hex', 'vulnerability', 'weakness', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'other', 'hostname|port', 'email-dst-display-name', 'email-src-display-name', 'email-header', 'email-reply-to', 'email-x-mailer', 'email-mime-boundary', 'email-thread-index', 'email-message-id', 'mobile-application-id', 'chrome-extension-id', 'whois-registrant-email', 'anonymised')
+                    'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash','filename|impfuzzy', 'filename|pehash', 'mac-address', 'mac-eui-64', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'hostname', 'domain', 'email-src', 'email-dst', 'email-subject', 'email-attachment', 'email-body', 'url', 'user-agent', 'AS', 'pattern-in-file', 'pattern-in-traffic', 'stix2-pattern', 'yara', 'sigma', 'mime-type', 'attachment', 'malware-sample', 'link', 'malware-type', 'comment', 'text', 'hex', 'vulnerability', 'weakness', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'other', 'hostname|port', 'email-dst-display-name', 'email-src-display-name', 'email-header', 'email-reply-to', 'email-x-mailer', 'email-mime-boundary', 'email-thread-index', 'email-message-id', 'mobile-application-id', 'chrome-extension-id', 'whois-registrant-email', 'anonymised')
                     ),
             'Artifacts dropped' => array(
                     'desc' => __('Any artifact (files, registry keys etc.) dropped by the malware or other modifications to the system'),
-                    'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|authentihash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy','filename|pehash', 'regkey', 'regkey|value', 'pattern-in-file', 'pattern-in-memory','pdb', 'stix2-pattern', 'yara', 'sigma', 'attachment', 'malware-sample', 'named pipe', 'mutex', 'windows-scheduled-task', 'windows-service-name', 'windows-service-displayname', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'cookie', 'gene', 'kusto-query', 'mime-type', 'anonymised')
+                    'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'vhash', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy','filename|pehash', 'regkey', 'regkey|value', 'pattern-in-file', 'pattern-in-memory','pdb', 'stix2-pattern', 'yara', 'sigma', 'attachment', 'malware-sample', 'named pipe', 'mutex', 'windows-scheduled-task', 'windows-service-name', 'windows-service-displayname', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'other', 'cookie', 'gene', 'kusto-query', 'mime-type', 'anonymised')
                     ),
             'Payload installation' => array(
                     'desc' => __('Info on where the malware gets installed in the system'),
                     'formdesc' => __('Location where the payload was placed in the system and the way it was installed. For example, a filename|md5 type attribute can be added here like this: c:\\windows\\system32\\malicious.exe|41d8cd98f00b204e9800998ecf8427e.'),
-                    'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|authentihash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy', 'filename|pehash', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'stix2-pattern', 'yara', 'sigma', 'vulnerability', 'weakness', 'attachment', 'malware-sample', 'malware-type', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'mobile-application-id', 'chrome-extension-id', 'other', 'mime-type', 'anonymised')
+                    'types' => array('md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|impfuzzy', 'filename|pehash', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'stix2-pattern', 'yara', 'sigma', 'vulnerability', 'weakness', 'attachment', 'malware-sample', 'malware-type', 'comment', 'text', 'hex', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'mobile-application-id', 'chrome-extension-id', 'other', 'mime-type', 'anonymised')
                     ),
             'Persistence mechanism' => array(
                     'desc' => __('Mechanisms used by the malware to start at boot'),
@@ -125,7 +128,7 @@ class Attribute extends AppModel
             'External analysis' => array(
                     'desc' => __('Any other result from additional analysis of the malware like tools output'),
                     'formdesc' => __('Any other result from additional analysis of the malware like tools output Examples: pdf-parser output, automated sandbox analysis, reverse engineering report.'),
-                    'types' => array('md5', 'sha1', 'sha256','filename', 'filename|md5', 'filename|sha1', 'filename|sha256', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'mac-address', 'mac-eui-64', 'hostname', 'domain', 'domain|ip', 'url', 'user-agent', 'regkey', 'regkey|value', 'AS', 'snort', 'bro', 'zeek', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'vulnerability', 'weakness', 'attachment', 'malware-sample', 'link', 'comment', 'text', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'github-repository', 'other', 'cortex', 'anonymised', 'community-id')
+                    'types' => array('md5', 'sha1', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'ip-src', 'ip-dst', 'ip-dst|port', 'ip-src|port', 'mac-address', 'mac-eui-64', 'hostname', 'domain', 'domain|ip', 'url', 'user-agent', 'regkey', 'regkey|value', 'AS', 'snort', 'bro', 'zeek', 'pattern-in-file', 'pattern-in-traffic', 'pattern-in-memory', 'vulnerability', 'weakness', 'attachment', 'malware-sample', 'link', 'comment', 'text', 'x509-fingerprint-sha1', 'x509-fingerprint-md5', 'x509-fingerprint-sha256', 'ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'github-repository', 'other', 'cortex', 'anonymised', 'community-id')
                     ),
             'Financial fraud' => array(
                     'desc' => __('Financial Fraud indicators'),
@@ -234,6 +237,7 @@ class Attribute extends AppModel
             'malware-type' => array('desc' => '', 'default_category' => 'Payload delivery', 'to_ids' => 0),
             'uri' => array('desc' => __('Uniform Resource Identifier'), 'default_category' => 'Network activity', 'to_ids' => 1),
             'authentihash' => array('desc' => __('Authenticode executable signature hash'), 'formdesc' => __("You are encouraged to use filename|authentihash instead. Authenticode executable signature hash, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
+            'vhash' => array('desc' => __('A VirusTotal checksum'), 'formdesc' => __("You are encouraged to use filename|vhash instead. A checksum from VirusTotal, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'ssdeep' => array('desc' => __('A checksum in ssdeep format'), 'formdesc' => __("You are encouraged to use filename|ssdeep instead. A checksum in the SSDeep format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'imphash' => array('desc' => __('Import hash - a hash created based on the imports in the sample.'), 'formdesc' => __("You are encouraged to use filename|imphash instead. A hash created based on the imports in the sample, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'pehash' => array('desc' => __('PEhash - a hash calculated based of certain pieces of a PE executable file'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
@@ -243,9 +247,14 @@ class Attribute extends AppModel
             'sha512' => array('desc' => __('A checksum in sha-512 format'), 'formdesc' => __("You are encouraged to use filename|sha512 instead. A checksum in sha512 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'sha512/224' => array('desc' => __('A checksum in the sha-512/224 format'), 'formdesc' => __("You are encouraged to use filename|sha512/224 instead. A checksum in sha512/224 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'sha512/256' => array('desc' => __('A checksum in the sha-512/256 format'), 'formdesc' => __("You are encouraged to use filename|sha512/256 instead. A checksum in sha512/256 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
+            'sha3-224' => array('desc' => __('A checksum in sha3-224 format'), 'formdesc' => __("You are encouraged to use filename|sha3-224 instead. A checksum in sha3-224 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
+            'sha3-256' => array('desc' => __('A checksum in sha3-256 format'), 'formdesc' => __("You are encouraged to use filename|sha3-256 instead. A checksum in sha3-256 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
+            'sha3-384' => array('desc' => __('A checksum in sha3-384 format'), 'formdesc' => __("You are encouraged to use filename|sha3-384 instead. A checksum in sha3-384 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
+            'sha3-512' => array('desc' => __('A checksum in sha3-512 format'), 'formdesc' => __("You are encouraged to use filename|sha3-512 instead. A checksum in sha3-512 format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'tlsh' => array('desc' => __('A checksum in the Trend Micro Locality Sensitive Hash format'), 'formdesc' => __("You are encouraged to use filename|tlsh instead. A checksum in the Trend Micro Locality Sensitive Hash format, only use this if you don't know the correct filename"), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'cdhash' => array('desc' => __('An Apple Code Directory Hash, identifying a code-signed Mach-O executable file'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'filename|authentihash' => array('desc' => __('A checksum in md5 format'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
+            'filename|vhash' => array('desc' => __('A filename and a VirusTotal hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'filename|ssdeep' => array('desc' => __('A checksum in ssdeep format'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'filename|imphash' => array('desc' => __('Import hash - a hash created based on the imports in the sample.'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'filename|impfuzzy' => array('desc' => __('Import fuzzy hash - a fuzzy hash created based on the imports in the sample.'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
@@ -255,6 +264,10 @@ class Attribute extends AppModel
             'filename|sha512' => array('desc' => __('A filename and a sha-512 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'filename|sha512/224' => array('desc' => __('A filename and a sha-512/224 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'filename|sha512/256' => array('desc' => __('A filename and a sha-512/256 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
+            'filename|sha3-224' => array('desc' => __('A filename and an sha3-224 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
+            'filename|sha3-256' => array('desc' => __('A filename and an sha3-256 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
+            'filename|sha3-384' => array('desc' => __('A filename and an sha3-384 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
+            'filename|sha3-512' => array('desc' => __('A filename and an sha3-512 hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'filename|tlsh' => array('desc' => __('A filename and a Trend Micro Locality Sensitive Hash separated by a |'), 'default_category' => 'Payload delivery', 'to_ids' => 1),
             'windows-scheduled-task' => array('desc' => __('A scheduled task in windows'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0),
             'windows-service-name' => array('desc' => __('A windows service name. This is the name used internally by windows. Not to be confused with the windows-service-displayname.'), 'default_category' => 'Artifacts dropped', 'to_ids' => 0),
@@ -407,6 +420,7 @@ class Attribute extends AppModel
     public $validFormats = array(
         'attack-sightings' => array('json', 'AttackSightingsExport', 'json'),
         'cache' => array('txt', 'CacheExport', 'cache'),
+        'count' => array('txt', 'CountExport', 'txt'),
         'csv' => array('csv', 'CsvExport', 'csv'),
         'hashes' => array('txt', 'HashesExport', 'txt'),
         'json' => array('json', 'JsonExport', 'json'),
@@ -432,7 +446,12 @@ class Attribute extends AppModel
             'sha512' => 'Payload delivery',
             'sha512/224' => 'Payload delivery',
             'sha512/256' => 'Payload delivery',
+            'sha3-224' =>'Payload delivery',
+            'sha3-256' =>'Payload delivery',
+            'sha3-384' =>'Payload delivery',
+            'sha3-512' =>'Payload delivery',
             'authentihash' => 'Payload delivery',
+            'vhash' => 'Payload delivery',
             'imphash' => 'Payload delivery',
             'impfuzzy'=> 'Payload delivery',
             'pehash' => 'Payload delivery',
@@ -472,7 +491,7 @@ class Attribute extends AppModel
     // whilst filenames and hashes are file related attribute types
     // This helps generate quick filtering for the event view, but we may reuse this and enhance it in the future for other uses (such as the API?)
     public $typeGroupings = array(
-        'file' => array('attachment', 'pattern-in-file', 'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|authentihash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|pehash', 'malware-sample', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'x509-fingerprint-md5'),
+        'file' => array('attachment', 'pattern-in-file', 'md5', 'sha1', 'sha224', 'sha256', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'ssdeep', 'imphash', 'impfuzzy', 'authentihash', 'vhash', 'pehash', 'tlsh', 'cdhash', 'filename', 'filename|md5', 'filename|sha1', 'filename|sha224', 'filename|sha256', 'filename|sha384', 'filename|sha512', 'filename|sha512/224', 'filename|sha512/256', 'filename|sha3-224', 'filename|sha3-256', 'filename|sha3-384', 'filename|sha3-512', 'filename|authentihash', 'filename|vhash', 'filename|ssdeep', 'filename|tlsh', 'filename|imphash', 'filename|pehash', 'malware-sample', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256', 'x509-fingerprint-md5'),
         'network' => array('ip-src', 'ip-dst', 'ip-src|port', 'ip-dst|port', 'mac-address', 'mac-eui-64', 'hostname', 'hostname|port', 'domain', 'domain|ip', 'email-dst', 'url', 'uri', 'user-agent', 'http-method', 'AS', 'snort', 'bro', 'zeek',  'pattern-in-traffic', 'x509-fingerprint-md5', 'x509-fingerprint-sha1', 'x509-fingerprint-sha256','ja3-fingerprint-md5', 'hassh-md5', 'hasshserver-md5', 'community-id'),
         'financial' => array('btc', 'xmr', 'iban', 'bic', 'bank-account-nr', 'aba-rtn', 'bin', 'cc-number', 'prtn', 'phone-number')
     );
@@ -770,27 +789,7 @@ class Attribute extends AppModel
         // delete attachments from the disk
         $this->read(); // first read the attribute from the db
         if ($this->typeIsAttachment($this->data['Attribute']['type'])) {
-            // only delete the file if it exists
-            $attachments_dir = Configure::read('MISP.attachments_dir');
-            if (empty($attachments_dir)) {
-                $attachments_dir = $this->getDefaultAttachments_dir();
-            }
-
-            // Special case - If using S3, we have to delete from there
-            if ($this->attachmentDirIsS3()) {
-                // We're working in S3
-                $s3 = $this->getS3Client();
-                $s3->delete($this->data['Attribute']['event_id'] . DS . $this->data['Attribute']['id']);
-            } else {
-                // Standard delete
-                $filepath = $attachments_dir . DS . $this->data['Attribute']['event_id'] . DS . $this->data['Attribute']['id'];
-                $file = new File($filepath);
-                if ($file->exists()) {
-                    if (!$file->delete()) {
-                        throw new InternalErrorException(__('Delete of file attachment failed. Please report to administrator.'));
-                    }
-                }
-            }
+            $this->loadAttachmentTool()->delete($this->data['Attribute']['event_id'], $this->data['Attribute']['id']);
         }
         // update correlation..
         $this->__beforeDeleteCorrelation($this->data['Attribute']['id']);
@@ -1046,7 +1045,11 @@ class Attribute extends AppModel
         'sha384' => 96,
         'sha512' => 128,
         'sha512/224' => 56,
-        'sha512/256' => 64
+        'sha512/256' => 64,
+        'sha3-224' => 56,
+        'sha3-256' => 64,
+        'sha3-384' => 96,
+        'sha3-512' => 128
     );
 
     public function runValidation($value, $type)
@@ -1063,6 +1066,10 @@ class Attribute extends AppModel
             case 'sha512':
             case 'sha512/224':
             case 'sha512/256':
+            case 'sha3-224':
+            case 'sha3-256':
+            case 'sha3-384':
+            case 'sha3-512':
             case 'authentihash':
             case 'ja3-fingerprint-md5':
             case 'hassh-md5':
@@ -1145,6 +1152,10 @@ class Attribute extends AppModel
             case 'filename|sha512':
             case 'filename|sha512/224':
             case 'filename|sha512/256':
+            case 'filename|sha3-224':
+            case 'filename|sha3-256':
+            case 'filename|sha3-384':
+            case 'filename|sha3-512':
             case 'filename|authentihash':
                 $parts = explode('|', $type);
                 $length = $this->__hexHashLengths[$parts[1]];
@@ -1178,6 +1189,13 @@ class Attribute extends AppModel
                     $returnValue = __('Checksum has an invalid length or format (expected: filename|at least 35 hexadecimal characters). Please double check the value or select type "other".');
                 }
                 break;
+            case 'filename|vhash':
+                if (preg_match("#^.+\|[a-zA-Z0-9]+$#", $value)) {
+                    $returnValue = true;
+                } else {
+                    $returnValue = __('Checksum has an invalid length or format (expected: filename|string characters). Please double check the value or select type "other".');
+                }
+                break;
             case 'ip-src':
             case 'ip-dst':
                 if (strpos($value, '/') !== false) {
@@ -1431,6 +1449,7 @@ class Attribute extends AppModel
             case 'btc':
             case 'dash':
             case 'xmr':
+            case 'vhash':
                 if (preg_match('/^[a-zA-Z0-9]+$/', $value)) {
                     $returnValue = true;
                 }
@@ -1477,12 +1496,17 @@ class Attribute extends AppModel
             case 'sha512':
             case 'sha512/224':
             case 'sha512/256':
+            case 'sha3-224':
+            case 'sha3-256':
+            case 'sha3-384':
+            case 'sha3-512':
             case 'ja3-fingerprint-md5':
             case 'hassh-md5':
             case 'hasshserver-md5':
             case 'hostname':
             case 'pehash':
             case 'authentihash':
+            case 'vhash':
             case 'imphash':
             case 'tlsh':
             case 'anonymised':
@@ -1530,7 +1554,12 @@ class Attribute extends AppModel
             case 'filename|sha512':
             case 'filename|sha512/224':
             case 'filename|sha512/256':
+            case 'filename|sha3-224':
+            case 'filename|sha3-256':
+            case 'filename|sha3-384':
+            case 'filename|sha3-512':
             case 'filename|authentihash':
+            case 'filename|vhash':
             case 'filename|pehash':
             case 'filename|tlsh':
                 $pieces = explode('|', $value);
@@ -1750,81 +1779,49 @@ class Attribute extends AppModel
 
     public function typeIsMalware($type)
     {
-        if (in_array($type, $this->zippedDefinitions)) {
-            return true;
-        } else {
-            return false;
-        }
+        return in_array($type, $this->zippedDefinitions);
     }
 
     public function typeIsAttachment($type)
     {
-        if ((in_array($type, $this->zippedDefinitions)) || (in_array($type, $this->uploadDefinitions))) {
-            return true;
-        } else {
-            return false;
-        }
+        return in_array($type, $this->zippedDefinitions) || in_array($type, $this->uploadDefinitions);
     }
 
     public function getAttachment($attribute, $path_suffix='')
     {
-        $attachments_dir = Configure::read('MISP.attachments_dir');
-        if (empty($attachments_dir)) {
-            $attachments_dir = $this->getDefaultAttachments_dir();
-        }
+        return $this->loadAttachmentTool()->getContent($attribute['event_id'], $attribute['id'], $path_suffix);
+    }
 
-        if ($this->attachmentDirIsS3()) {
-            // S3 - we have to first get the object then we can encode it
-            $s3 = $this->getS3Client();
-            // This will return the content of the object
-            $content = $s3->download($attribute['event_id'] . DS . $attribute['id'] . $path_suffix);
-        } else {
-            // Standard filesystem
-            $filepath = $attachments_dir . DS . $attribute['event_id'] . DS . $attribute['id'] . $path_suffix;
-            $file = new File($filepath);
-            if (!$file->readable()) {
-                return '';
-            }
-            $content = $file->read();
-        }
-        return $content;
+    /**
+     * @param array $attribute
+     * @param string $path_suffix
+     * @return File
+     * @throws Exception
+     */
+    public function getAttachmentFile(array $attribute, $path_suffix='')
+    {
+        return $this->loadAttachmentTool()->getFile($attribute['event_id'], $attribute['id'], $path_suffix);
     }
 
     public function saveAttachment($attribute, $path_suffix='')
     {
-        $attachments_dir = Configure::read('MISP.attachments_dir');
-        if (empty($attachments_dir)) {
-            $attachments_dir = $this->getDefaultAttachments_dir();
-        }
-
-        if ($this->attachmentDirIsS3()) {
-            // This is the cloud!
-            // We don't need your fancy directory structures and
-            // PEE AICH PEE meddling
-            $s3 = $this->getS3Client();
-            $data = $attribute['data'];
-            $key = $attribute['event_id'] . DS . $attribute['id'] . $path_suffix;
-            $s3->upload($key, $data);
-            return true;
-        } else {
-            // Plebian filesystem operations
-            $rootDir = $attachments_dir . DS . $attribute['event_id'];
-            $dir = new Folder($rootDir, true);                      // create directory structure
-            $destpath = $rootDir . DS . $attribute['id'] . $path_suffix;
-            $file = new File($destpath, true);                      // create the file
-            $decodedData = $attribute['data'];       // decode
-            if ($file->write($decodedData)) {                       // save the data
-                return true;
-            } else {
-                // error
-                return false;
-            }
-        }
+        return $this->loadAttachmentTool()->save($attribute['event_id'], $attribute['id'], $attribute['data'], $path_suffix);
     }
 
-    public function base64EncodeAttachment($attribute)
+    /**
+     * Returns attribute attachment content as base64 encoded string. If file doesn't exists, empty string is returned.
+     *
+     * @param array $attribute
+     * @return string
+     */
+    public function base64EncodeAttachment(array $attribute)
     {
-        return base64_encode($this->getAttachment($attribute));
+        try {
+            return base64_encode($this->getAttachment($attribute));
+        } catch (NotFoundException $e) {
+            $this->log($e->getMessage(), LOG_NOTICE);
+            return '';
+        }
     }
 
     public function saveBase64EncodedAttachment($attribute)
@@ -1857,9 +1854,10 @@ class Attribute extends AppModel
         if ($thumbnail && extension_loaded('gd')) {
             if ($maxWidth == 200 && $maxHeight == 200) {
                 // Return thumbnail directly if already exists
-                $imageData = $this->getAttachment($attribute['Attribute'], $path_suffix='_thumbnail');
-                if ($imageData !== '') {
-                    return $imageData;
+                try {
+                    return $this->getAttachment($attribute['Attribute'], $path_suffix = '_thumbnail');
+                } catch (NotFoundException $e) {
+                    // pass
                 }
             }
 
@@ -2097,7 +2095,7 @@ class Attribute extends AppModel
             $extraConditions = $this->__cidrCorrelation($a);
         } else if ($a['type'] === 'ssdeep' && function_exists('ssdeep_fuzzy_compare')) {
             $this->FuzzyCorrelateSsdeep = ClassRegistry::init('FuzzyCorrelateSsdeep');
-            $fuzzyIds = $this->FuzzyCorrelateSsdeep->query_ssdeep_chunks($a['value'], $a['id']);
+            $fuzzyIds = $this->FuzzyCorrelateSsdeep->query_ssdeep_chunks($a['value1'], $a['id']);
             if (!empty($fuzzyIds)) {
                 $ssdeepIds = $this->find('list', array(
                     'recursive' => -1,
@@ -2110,7 +2108,7 @@ class Attribute extends AppModel
                 $threshold = Configure::read('MISP.ssdeep_correlation_threshold') ?: 40;
                 $attributeIds = array();
                 foreach ($ssdeepIds as $attributeId => $v) {
-                    $ssdeep_value = ssdeep_fuzzy_compare($a['value'], $v);
+                    $ssdeep_value = ssdeep_fuzzy_compare($a['value1'], $v);
                     if ($ssdeep_value >= $threshold) {
                         $attributeIds[] = $attributeId;
                     }
@@ -2773,8 +2771,8 @@ class Attribute extends AppModel
                 }
                 $conditions['AND'][] = $temp;
             }
-            $this->Whitelist = ClassRegistry::init('Whitelist');
-            $this->whitelist = $this->Whitelist->getBlockedValues();
+            $this->Allowedlist = ClassRegistry::init('Allowedlist');
+            $this->allowedlist = $this->Allowedlist->getBlockedValues();
             $instanceString = 'MISP';
             if (Configure::read('MISP.host_org_id') && Configure::read('MISP.host_org_id') > 0) {
                 $this->Event->Orgc->id = Configure::read('MISP.host_org_id');
@@ -2785,7 +2783,7 @@ class Attribute extends AppModel
             $mispTypes = $export->getMispTypes($type);
             foreach ($mispTypes as $mispType) {
                 $conditions['AND']['Attribute.type'] = $mispType[0];
-                $intel = array_merge($intel, $this->__bro($user, $conditions, $mispType[1], $export, $this->whitelist, $instanceString, $enforceWarninglist));
+                $intel = array_merge($intel, $this->__bro($user, $conditions, $mispType[1], $export, $this->allowedlist, $instanceString, $enforceWarninglist));
             }
         }
         natsort($intel);
@@ -2796,7 +2794,7 @@ class Attribute extends AppModel
         return $intel;
     }
 
-    private function __bro($user, $conditions, $valueField, $export, $whitelist, $instanceString, $enforceWarninglist)
+    private function __bro($user, $conditions, $valueField, $export, $allowedlist, $instanceString, $enforceWarninglist)
     {
         $attributes = $this->fetchAttributes(
             $user,
@@ -2814,7 +2812,7 @@ class Attribute extends AppModel
         $orgs = $this->Event->Orgc->find('list', array(
                 'fields' => array('Orgc.id', 'Orgc.name')
         ));
-        return $export->export($attributes, $orgs, $valueField, $whitelist, $instanceString);
+        return $export->export($attributes, $orgs, $valueField, $allowedlist, $instanceString);
     }
 
     public function generateCorrelation($jobId = false, $startPercentage = 0, $eventId = false, $attributeId = false)
@@ -2827,7 +2825,11 @@ class Attribute extends AppModel
 
         // get all attributes..
         if (!$eventId) {
-            $eventIds = $this->Event->find('list', array('recursive' => -1, 'fields' => array('Event.id')));
+            $eventIds = $this->Event->find('list', [
+                'recursive' => -1,
+                'fields' => ['Event.id'],
+                'conditions' => ['Event.disable_correlation' => 0],
+            ]);
             $full = true;
         } else {
             $eventIds = array($eventId);
@@ -2836,17 +2838,18 @@ class Attribute extends AppModel
         $attributeCount = 0;
         if (Configure::read('MISP.background_jobs') && $jobId) {
             $this->Job = ClassRegistry::init('Job');
-            $this->Job->id = $jobId;
             $eventCount = count($eventIds);
+        } else {
+            $jobId = false;
         }
         foreach (array_values($eventIds) as $j => $id) {
-            if ($jobId && Configure::read('MISP.background_jobs')) {
+            if ($jobId) {
                 if ($attributeId) {
-                    $this->Job->saveField('message', 'Correlating Attribute ' . $attributeId);
+                    $message = 'Correlating Attribute ' . $attributeId;
                 } else {
-                    $this->Job->saveField('message', 'Correlating Event ' . $id);
+                    $message = 'Correlating Event ' . $id;
                 }
-                $this->Job->saveField('progress', ($startPercentage + ($j / $eventCount * (100 - $startPercentage))));
+                $this->Job->saveProgress($jobId, $message, $startPercentage + ($j / $eventCount * (100 - $startPercentage)));
             }
             $event = $this->Event->find('first', array(
                     'recursive' => -1,
@@ -2865,14 +2868,29 @@ class Attribute extends AppModel
             if ($attributeId) {
                 $attributeConditions['Attribute.id'] = $attributeId;
             }
-            $attributes = $this->find('all', array('recursive' => -1, 'conditions' => $attributeConditions, 'order' => array()));
-            foreach ($attributes as $k => $attribute) {
+            $attributes = $this->find('all', [
+                'recursive' => -1,
+                'conditions' => $attributeConditions,
+                // fetch just necessary fields to save memory
+                'fields' => [
+                    'Attribute.id',
+                    'Attribute.event_id',
+                    'Attribute.type',
+                    'Attribute.value1',
+                    'Attribute.value2',
+                    'Attribute.distribution',
+                    'Attribute.sharing_group_id',
+                    'Attribute.disable_correlation',
+                ],
+                'order' => [],
+            ]);
+            foreach ($attributes as $attribute) {
                 $this->__afterSaveCorrelation($attribute['Attribute'], $full, $event);
                 $attributeCount++;
             }
         }
-        if ($jobId && Configure::read('MISP.background_jobs')) {
-            $this->Job->saveField('message', 'Job done.');
+        if ($jobId) {
+            $this->Job->saveStatus($jobId, true);
         }
         return $attributeCount;
     }
@@ -3341,12 +3359,9 @@ class Attribute extends AppModel
         if (isset($options['limit'])) {
             $params['limit'] = $options['limit'];
         }
-        if (!empty($options['includeGalaxy'])) {
-            $this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
-        }
         if (
-            Configure::read('MISP.proposals_block_attributes') &&
-            !empty($options['allow_proposal_blocking'])
+            !empty($options['allow_proposal_blocking']) &&
+            Configure::read('MISP.proposals_block_attributes')
         ) {
             $this->bindModel(array('hasMany' => array('ShadowAttribute' => array('foreignKey' => 'old_id'))));
             $proposalRestriction =  array(
@@ -3417,7 +3432,8 @@ class Attribute extends AppModel
         if (isset($options['group'])) {
             $params['group'] = empty($options['group']) ? $options['group'] : false;
         }
-        if (Configure::read('MISP.unpublishedprivate')) {
+        // Site admin can access even unpublished event attributes if `unpublishedprivate` option is enabled
+        if (!$user['Role']['perm_site_admin'] && Configure::read('MISP.unpublishedprivate')) {
             $params['conditions']['AND'][] = array('OR' => array('Event.published' => 1, 'Event.orgc_id' => $user['org_id'], 'Event.org_id' => $user['org_id']));
         }
         if (!empty($options['list'])) {
@@ -3625,10 +3641,10 @@ class Attribute extends AppModel
 
         $content = base64_decode($base64);
 
-        $malwareTool = new MalwareTool();
-        $hashes = $malwareTool->computeHashes($content, $hash_types);
+        $attachmentTool = $this->loadAttachmentTool();
+        $hashes = $attachmentTool->computeHashes($content, $hash_types);
         try {
-            $encrypted = $malwareTool->encrypt($original_filename, $content, $hashes['md5']);
+            $encrypted = $attachmentTool->encrypt($original_filename, $content, $hashes['md5']);
         } catch (Exception $e) {
             $this->logException("Could not create encrypted malware sample.", $e);
             return array('success' => false);
@@ -3643,9 +3659,8 @@ class Attribute extends AppModel
      */
     public function isAdvancedExtractionAvailable()
     {
-        $malwareTool = new MalwareTool();
         try {
-            $types = $malwareTool->checkAdvancedExtractionStatus($this->getPythonVersion());
+            $types = $this->loadAttachmentTool()->checkAdvancedExtractionStatus($this->getPythonVersion());
         } catch (Exception $e) {
             return false;
         }
@@ -4042,9 +4057,8 @@ class Attribute extends AppModel
 
     public function advancedAddMalwareSample($event_id, $attribute_settings, $filename, $tmpfile)
     {
-        $malwareTool = new MalwareTool();
         try {
-            $result = $malwareTool->advancedExtraction($this->getPythonVersion(), $tmpfile->path);
+            $result = $this->loadAttachmentTool()->advancedExtraction($this->getPythonVersion(), $tmpfile->path);
         } catch (Exception $e) {
             $this->logException("Could not finish advanced extraction", $e);
             return $this->simpleAddMalwareSample($event_id, $attribute_settings, $filename, $tmpfile);
@@ -4621,7 +4635,7 @@ class Attribute extends AppModel
 
     private function __iteratedFetch($user, &$params, &$loop, TmpFileTool $tmpfile, $exportTool, $exportToolParams, &$elementCounter = 0)
     {
-        $this->Whitelist = ClassRegistry::init('Whitelist');
+        $this->Allowedlist = ClassRegistry::init('Allowedlist');
         $continue = true;
         while ($continue) {
             $results = $this->fetchAttributes($user, $params, $continue);
@@ -4630,7 +4644,7 @@ class Attribute extends AppModel
                 $results = $this->Sightingdb->attachToAttributes($results, $user);
             }
             $params['page'] += 1;
-            $results = $this->Whitelist->removeWhitelistedFromArray($results, true);
+            $results = $this->Allowedlist->removeAllowedlistedFromArray($results, true);
             $i = 0;
             $temp = '';
             foreach ($results as $attribute) {

app/Model/AttributeTag.php

@@ -1,6 +1,9 @@
 <?php
 App::uses('AppModel', 'Model');
 
+/**
+ * @property Attribute $Attribute
+ */
 class AttributeTag extends AppModel
 {
     public $actsAs = array('Containable');
@@ -177,11 +180,30 @@ class AttributeTag extends AppModel
         }
     }
 
-    public function countForTag($tag_id, $user)
+    /**
+     * Count number of not deleted attributes that contains given tag for given user. Tag must contains 'AttributeTag'.
+     *
+     * @param array $tag
+     * @param array $user
+     * @return int
+     */
+    public function countForTag(array $tag, array $user)
     {
-        return $this->find('count', array(
-            'recursive' => -1,
-            'conditions' => array('AttributeTag.tag_id' => $tag_id)
+        $attributeIds = [];
+        foreach ($tag['AttributeTag'] as $attributeTag) {
+            $attributeIds[] = $attributeTag['attribute_id'];
+        }
+
+        if (empty($attributeIds)) {
+            return 0;
+        }
+
+        $conditions = $this->Attribute->buildConditions($user);
+        $conditions['Attribute.id'] = $attributeIds;
+        $conditions['Attribute.deleted'] = 0;
+        return $this->Attribute->find('count', array(
+            'recursive' => 0,
+            'conditions' => $conditions,
         ));
     }
 
@@ -236,22 +258,12 @@ class AttributeTag extends AppModel
 
 
     // find all tags that belong to a list of attributes (contained in the same event)
-    public function getAttributesTags($user, $requestedEventId, $attributeIds=false, $includeGalaxies=false) {
-        $conditions = array('Attribute.event_id' => $requestedEventId);
-        if (is_array($attributeIds) && $attributeIds !== false) {
-            $conditions['Attribute.id'] = $attributeIds;
-        }
-
-        $allTags = array();
-        $attributes = $this->Attribute->fetchAttributes($user, array(
-            'conditions' => $conditions,
-            'flatten' => 1,
-            'includeAllTags' => 1
-        ));
-
+    public function getAttributesTags(array $attributes, $includeGalaxies=false)
+    {
         if (empty($attributes)) {
             return array();
         }
+
         $this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
         $cluster_names = $this->GalaxyCluster->find('list', array(
                 'recursive' => -1,
@@ -260,7 +272,7 @@ class AttributeTag extends AppModel
         $allTags = array();
         foreach ($attributes as $attribute) {
             $attributeTags = $attribute['AttributeTag'];
-            foreach ($attributeTags as $k => $attributeTag) {
+            foreach ($attributeTags as $attributeTag) {
                 if ($includeGalaxies || !isset($cluster_names[$attributeTag['Tag']['name']])) {
                     $allTags[$attributeTag['Tag']['id']] = $attributeTag['Tag'];
                 }
@@ -270,16 +282,8 @@ class AttributeTag extends AppModel
     }
 
     // find all galaxies that belong to a list of attributes (contains in the same event)
-    public function getAttributesClusters($user, $requestedEventId, $attributeIds=false) {
-        $conditions = array('Attribute.event_id' => $requestedEventId);
-        if (is_array($attributeIds) && $attributeIds !== false) {
-            $conditions['Attribute.id'] = $attributeIds;
-        }
-
-        $attributes = $this->Attribute->fetchAttributes($user, array(
-            'conditions' => $conditions,
-            'flatten' => 1,
-        ));
+    public function getAttributesClusters(array $attributes)
+    {
         if (empty($attributes)) {
             return array();
         }
@@ -294,7 +298,7 @@ class AttributeTag extends AppModel
         foreach ($attributes as $attribute) {
             $attributeTags = $attribute['AttributeTag'];
 
-            foreach ($attributeTags as $k => $attributeTag) {
+            foreach ($attributeTags as $attributeTag) {
                 if (isset($cluster_names[$attributeTag['Tag']['name']])) {
                     $cluster = $this->GalaxyCluster->find('first', array(
                             'conditions' => array('GalaxyCluster.tag_name' => $attributeTag['Tag']['name']),

app/Model/Behavior/RegexpBehavior.php

@@ -12,7 +12,7 @@ class RegexpBehavior extends ModelBehavior
     public $excluded_types = array('sigma', 'float');
 
     /**
-     * replace the current value according to the regexp rules, or block blacklisted regular expressions
+     * replace the current value according to the regexp rules, or block blocklisted regular expressions
      *
      * @param Model $Model
      * @param string $type

app/Model/Bruteforce.php

@@ -20,8 +20,8 @@ class Bruteforce extends AppModel
         );
         $this->save($bruteforceEntry);
         $title = 'Failed login attempt using username ' . $username . ' from IP: ' . $_SERVER['REMOTE_ADDR'] . '.';
-        if ($this->isBlacklisted($ip, $username)) {
-            $title .= 'This has tripped the bruteforce protection after  ' . $amount . ' failed attempts. The user is now blacklisted for ' . $expire . ' seconds.';
+        if ($this->isBlocklisted($ip, $username)) {
+            $title .= 'This has tripped the bruteforce protection after  ' . $amount . ' failed attempts. The user is now blocklisted for ' . $expire . ' seconds.';
         }
         $log = array(
                 'org' => 'SYSTEM',
@@ -47,7 +47,7 @@ class Bruteforce extends AppModel
         $this->query($sql);
     }
 
-    public function isBlacklisted($ip, $username)
+    public function isBlocklisted($ip, $username)
     {
         // first remove old expired rows
         $this->clean();

app/Model/EventBlocklist.php

@@ -1,8 +1,8 @@
 <?php
 App::uses('AppModel', 'Model');
-class EventBlacklist extends AppModel
+class EventBlocklist extends AppModel
 {
-    public $useTable = 'event_blacklists';
+    public $useTable = 'event_blocklists';
 
     public $recursive = -1;
 
@@ -14,13 +14,15 @@ class EventBlacklist extends AppModel
             'Containable',
     );
 
-    public $blacklistFields = array('event_uuid', 'comment', 'event_info', 'event_orgc');
+    public $blocklistFields = array('event_uuid', 'comment', 'event_info', 'event_orgc');
+
+    public $blocklistTarget = 'event';
 
     public $validate = array(
             'event_uuid' => array(
                     'unique' => array(
                             'rule' => 'isUnique',
-                            'message' => 'Event already blacklisted.'
+                            'message' => 'Event already blocklisted.'
                     ),
                     'uuid' => array(
                             'rule' => array('uuid'),
@@ -34,14 +36,14 @@ class EventBlacklist extends AppModel
         parent::beforeValidate();
         $schema = $this->schema();
         if (!isset($schema['event_info'])) {
-            $this->updateDatabase('addEventBlacklistsContext');
+            $this->updateDatabase('addEventBlocklistsContext');
         }
         $date = date('Y-m-d H:i:s');
-        if (empty($this->data['EventBlacklist']['id'])) {
-            $this->data['EventBlacklist']['date_created'] = $date;
+        if (empty($this->data['EventBlocklist']['id'])) {
+            $this->data['EventBlocklist']['date_created'] = $date;
         }
-        if (empty($this->data['EventBlacklist']['comment'])) {
-            $this->data['EventBlacklist']['comment'] = '';
+        if (empty($this->data['EventBlocklist']['comment'])) {
+            $this->data['EventBlocklist']['comment'] = '';
         }
         return true;
     }

app/Model/EventTag.php

@@ -1,6 +1,9 @@
 <?php
 App::uses('AppModel', 'Model');
 
+/**
+ * @property Event $Event
+ */
 class EventTag extends AppModel
 {
     public $actsAs = array('Containable');
@@ -79,37 +82,6 @@ class EventTag extends AppModel
         $this->delete($id);
     }
 
-    // take an array of tag names to be included and an array with tagnames to be excluded and find all event IDs that fit the criteria
-    public function getEventIDsFromTags($includedTags, $excludedTags)
-    {
-        $conditions = array();
-        if (!empty($includedTags)) {
-            $conditions['OR'] = array('name' => $includedTags);
-        }
-        if (!empty($excludedTags)) {
-            $conditions['NOT'] = array('name' => $excludedTags);
-        }
-        $tags = $this->Tag->find('all', array(
-            'recursive' => -1,
-            'fields' => array('id', 'name'),
-            'conditions' => $conditions
-        ));
-        $tagIDs = array();
-        foreach ($tags as $tag) {
-            $tagIDs[] = $tag['Tag']['id'];
-        }
-        $eventTags = $this->find('all', array(
-            'recursive' => -1,
-            'conditions' => array('tag_id' => $tagIDs)
-        ));
-        $eventIDs = array();
-        foreach ($eventTags as $eventTag) {
-            $eventIDs[] = $eventTag['EventTag']['event_id'];
-        }
-        $eventIDs = array_unique($eventIDs);
-        return $eventIDs;
-    }
-
     public function handleEventTag($event_id, $tag, &$nothingToChange = false)
     {
         if (empty($tag['deleted'])) {
@@ -188,11 +160,29 @@ class EventTag extends AppModel
         return $tags;
     }
 
-    public function countForTag($tag_id, $user)
+    /**
+     * Count number of event that contains given tag for given user. Tag must contains 'EventTag'.
+     *
+     * @param array $tag
+     * @param array $user
+     * @return int
+     */
+    public function countForTag(array $tag, array $user)
     {
-        return $this->find('count', array(
+        $eventIds = [];
+        foreach ($tag['EventTag'] as $eventTag) {
+            $eventIds[] = $eventTag['event_id'];
+        }
+
+        if (empty($eventIds)) {
+            return 0;
+        }
+
+        $conditions = $this->Event->createEventConditions($user);
+        $conditions['Event.id'] = $eventIds;
+        return $this->Event->find('count', array(
             'recursive' => -1,
-            'conditions' => array('EventTag.tag_id' => $tag_id)
+            'conditions' => $conditions,
         ));
     }
 

app/Model/Feed.php

@@ -127,13 +127,14 @@ class Feed extends AppModel
 
     /**
      * Gets the event UUIDs from the feed by ID
-     * Returns an array with the UUIDs of events that are new or that need updating
+     * Returns an array with the UUIDs of events that are new or that need updating.
+     *
      * @param array $feed
-     * @param HttpSocket $HttpSocket
+     * @param HttpSocket|null $HttpSocket
      * @return array
      * @throws Exception
      */
-    public function getNewEventUuids($feed, $HttpSocket)
+    public function getNewEventUuids($feed, HttpSocket $HttpSocket = null)
     {
         $manifest = $this->downloadManifest($feed, $HttpSocket);
         $this->Event = ClassRegistry::init('Event');
@@ -161,11 +162,11 @@ class Feed extends AppModel
 
     /**
      * @param array $feed
-     * @param HttpSocket $HttpSocket
+     * @param HttpSocket|null $HttpSocket Null can be for local feed
      * @return Generator|array
      * @throws Exception
      */
-    public function getCache(array $feed, HttpSocket $HttpSocket)
+    public function getCache(array $feed, HttpSocket $HttpSocket = null)
     {
         $uri = $feed['Feed']['url'] . '/hashes.csv';
         $data = $this->feedGetUri($feed, $uri, $HttpSocket);
@@ -187,30 +188,29 @@ class Feed extends AppModel
 
     /**
      * @param array $feed
-     * @param HttpSocket $HttpSocket
+     * @param HttpSocket|null $HttpSocket Null can be for local feed
      * @return array
      * @throws Exception
      */
-    private function downloadManifest($feed, $HttpSocket)
+    private function downloadManifest($feed, HttpSocket $HttpSocket = null)
     {
         $manifestUrl = $feed['Feed']['url'] . '/manifest.json';
         $data = $this->feedGetUri($feed, $manifestUrl, $HttpSocket, true);
 
-        $manifest = json_decode($data, true);
-        if ($manifest === null) {
-            throw new Exception('Could not parse manifest JSON: ' . json_last_error_msg(), json_last_error());
+        try {
+            return $this->jsonDecode($data);
+        } catch (Exception $e) {
+            throw new Exception("Could not parse '$manifestUrl' manifest JSON", 0, $e);
         }
-
-        return $manifest;
     }
 
     /**
      * @param array $feed
-     * @param HttpSocket $HttpSocket
+     * @param HttpSocket|null $HttpSocket Null can be for local feed
      * @return array
      * @throws Exception
      */
-    public function getManifest($feed, $HttpSocket)
+    public function getManifest($feed, HttpSocket $HttpSocket = null)
     {
         $events = $this->downloadManifest($feed, $HttpSocket);
         $events = $this->__filterEventsIndex($events, $feed);
@@ -219,7 +219,7 @@ class Feed extends AppModel
 
     /**
      * @param array $feed
-     * @param HttpSocket $HttpSocket
+     * @param HttpSocket|null $HttpSocket Null can be for local feed
      * @param string $type
      * @param int|string $page
      * @param int $limit
@@ -227,7 +227,7 @@ class Feed extends AppModel
      * @return array|bool
      * @throws Exception
      */
-    public function getFreetextFeed($feed, $HttpSocket, $type = 'freetext', $page = 1, $limit = 60, &$params = array())
+    public function getFreetextFeed($feed, HttpSocket $HttpSocket = null, $type = 'freetext', $page = 1, $limit = 60, &$params = array())
     {
         $isLocal = $this->isFeedLocal($feed);
         $data = false;
@@ -467,7 +467,7 @@ class Feed extends AppModel
         return $objects;
     }
 
-    public function downloadFromFeed($actions, $feed, $HttpSocket, $user, $jobId = false)
+    public function downloadFromFeed($actions, $feed, HttpSocket $HttpSocket = null, $user, $jobId = false)
     {
         $total = count($actions['add']) + count($actions['edit']);
         $currentItem = 0;
@@ -661,7 +661,7 @@ class Feed extends AppModel
     public function downloadEventFromFeed(array $feed, $uuid)
     {
         $filerRules = $this->__prepareFilterRules($feed);
-        $HttpSocket = $this->isFeedLocal($feed) ? false : $this->__setupHttpSocket($feed);
+        $HttpSocket = $this->isFeedLocal($feed) ? null : $this->__setupHttpSocket($feed);
         $event = $this->downloadAndParseEventFromFeed($feed, $uuid, $HttpSocket);
         return $this->__prepareEvent($event, $feed, $filerRules);
     }
@@ -770,11 +770,11 @@ class Feed extends AppModel
     {
         App::uses('SyncTool', 'Tools');
         $syncTool = new SyncTool();
-        return ($syncTool->setupHttpSocketFeed($feed));
+        return $syncTool->setupHttpSocketFeed($feed);
     }
 
     /**
-     * @param HttpSocket $HttpSocket
+     * @param HttpSocket|null $HttpSocket
      * @param array $feed
      * @param string $uuid
      * @param $user
@@ -782,7 +782,7 @@ class Feed extends AppModel
      * @return array|bool|string
      * @throws Exception
      */
-    private function __addEventFromFeed($HttpSocket, $feed, $uuid, $user, $filterRules)
+    private function __addEventFromFeed(HttpSocket $HttpSocket = null, $feed, $uuid, $user, $filterRules)
     {
         $event = $this->downloadAndParseEventFromFeed($feed, $uuid, $HttpSocket);
         $event = $this->__prepareEvent($event, $feed, $filterRules);
@@ -795,7 +795,7 @@ class Feed extends AppModel
     }
 
     /**
-     * @param HttpSocket $HttpSocket
+     * @param HttpSocket|null $HttpSocket Null can be for local feed
      * @param array $feed
      * @param string $uuid
      * @param int $eventId
@@ -804,7 +804,7 @@ class Feed extends AppModel
      * @return mixed
      * @throws Exception
      */
-    private function __updateEventFromFeed($HttpSocket, $feed, $uuid, $eventId, $user, $filterRules)
+    private function __updateEventFromFeed(HttpSocket $HttpSocket = null, $feed, $uuid, $eventId, $user, $filterRules)
     {
         $event = $this->downloadAndParseEventFromFeed($feed, $uuid, $HttpSocket);
         $event = $this->__prepareEvent($event, $feed, $filterRules);
@@ -844,7 +844,7 @@ class Feed extends AppModel
             $this->data['Feed']['settings'] = json_decode($this->data['Feed']['settings'], true);
         }
 
-        $HttpSocket = $this->isFeedLocal($this->data) ? false : $this->__setupHttpSocket($this->data);
+        $HttpSocket = $this->isFeedLocal($this->data) ? null : $this->__setupHttpSocket($this->data);
         if ($this->data['Feed']['source_format'] == 'misp') {
             $this->jobProgress($jobId, 'Fetching event manifest.');
             try {
@@ -869,7 +869,7 @@ class Feed extends AppModel
                 $temp = $this->getFreetextFeed($this->data, $HttpSocket, $this->data['Feed']['source_format'], 'all');
             } catch (Exception $e) {
                 $this->logException("Could not get freetext feed $feedId", $e);
-                $this->jobProgress($jobId, 'Could not fetch freetext feed. See log for more details.');
+                $this->jobProgress($jobId, 'Could not fetch freetext feed. See error log for more details.');
                 return false;
             }
 
@@ -1118,7 +1118,7 @@ class Feed extends AppModel
 
     private function __cacheFeed($feed, $redis, $jobId = false)
     {
-        $HttpSocket = $this->isFeedLocal($feed) ? false : $this->__setupHttpSocket($feed);
+        $HttpSocket = $this->isFeedLocal($feed) ? null : $this->__setupHttpSocket($feed);
         if ($feed['Feed']['source_format'] == 'misp') {
             return $this->__cacheMISPFeed($feed, $redis, $HttpSocket, $jobId);
         } else {
@@ -1126,7 +1126,7 @@ class Feed extends AppModel
         }
     }
 
-    private function __cacheFreetextFeed(array $feed, $redis, HttpSocket $HttpSocket, $jobId = false)
+    private function __cacheFreetextFeed(array $feed, $redis, HttpSocket $HttpSocket = null, $jobId = false)
     {
         $feedId = $feed['Feed']['id'];
 
@@ -1154,10 +1154,9 @@ class Feed extends AppModel
         return true;
     }
 
-    private function __cacheMISPFeedTraditional($feed, $redis, $HttpSocket, $jobId = false)
+    private function __cacheMISPFeedTraditional($feed, $redis, HttpSocket $HttpSocket = null, $jobId = false)
     {
         $feedId = $feed['Feed']['id'];
-        $this->Attribute = ClassRegistry::init('Attribute');
         try {
             $manifest = $this->getManifest($feed, $HttpSocket);
         } catch (Exception $e) {
@@ -1177,6 +1176,7 @@ class Feed extends AppModel
             }
 
             if (!empty($event['Event']['Attribute'])) {
+                $this->Attribute = ClassRegistry::init('Attribute');
                 $pipe = $redis->multi(Redis::PIPELINE);
                 foreach ($event['Event']['Attribute'] as $attribute) {
                     if (!in_array($attribute['type'], $this->Attribute->nonCorrelatingTypes)) {
@@ -1206,7 +1206,7 @@ class Feed extends AppModel
         return true;
     }
 
-    private function __cacheMISPFeedCache($feed, $redis, $HttpSocket, $jobId = false)
+    private function __cacheMISPFeedCache($feed, $redis, HttpSocket $HttpSocket = null, $jobId = false)
     {
         $feedId = $feed['Feed']['id'];
 
@@ -1228,7 +1228,7 @@ class Feed extends AppModel
         return true;
     }
 
-    private function __cacheMISPFeed($feed, $redis, $HttpSocket, $jobId = false)
+    private function __cacheMISPFeed($feed, $redis, HttpSocket $HttpSocket = null, $jobId = false)
     {
         $result = true;
         if (!$this->__cacheMISPFeedCache($feed, $redis, $HttpSocket, $jobId)) {
@@ -1607,13 +1607,14 @@ class Feed extends AppModel
 
     /**
      * Download and parse event from feed.
+     *
      * @param array $feed
      * @param string $eventUuid
-     * @param HttpSocket $HttpSocket
+     * @param HttpSocket|null $HttpSocket Null can be for local feed
      * @return array
      * @throws Exception
      */
-    private function downloadAndParseEventFromFeed($feed, $eventUuid, $HttpSocket)
+    private function downloadAndParseEventFromFeed($feed, $eventUuid, HttpSocket $HttpSocket = null)
     {
         if (!Validation::uuid($eventUuid)) {
             throw new InvalidArgumentException("Given event UUID '$eventUuid' is invalid.");
@@ -1621,23 +1622,23 @@ class Feed extends AppModel
 
         $path = $feed['Feed']['url'] . '/' . $eventUuid . '.json';
         $data = $this->feedGetUri($feed, $path, $HttpSocket);
-        $event = json_decode($data, true);
-        if ($event === null) {
-            throw new Exception('Could not parse event JSON: ' . json_last_error_msg(), json_last_error());
-        }
 
-        return $event;
+        try {
+            return $this->jsonDecode($data);
+        } catch (Exception $e) {
+            throw new Exception("Could not parse event JSON with UUID '$eventUuid' from feed", 0, $e);
+        }
     }
 
     /**
      * @param array $feed
      * @param string $uri
-     * @param HttpSocket $HttpSocket
+     * @param HttpSocket|null $HttpSocket Null can be for local feed
      * @param bool $followRedirect
      * @return string
      * @throws Exception
      */
-    private function feedGetUri($feed, $uri, $HttpSocket, $followRedirect = false)
+    private function feedGetUri($feed, $uri, HttpSocket $HttpSocket = null, $followRedirect = false)
     {
         if ($this->isFeedLocal($feed)) {
             if (file_exists($uri)) {
@@ -1651,6 +1652,10 @@ class Feed extends AppModel
             }
         }
 
+        if ($HttpSocket === null) {
+            throw new Exception("Feed {$feed['Feed']['name']} is not local, but HttpSocket is not initialized.");
+        }
+
         $request = $this->__createFeedRequest($feed['Feed']['headers']);
 
         if ($followRedirect) {
@@ -1799,7 +1804,7 @@ class Feed extends AppModel
     private function unzipFirstFile(File $zipFile)
     {
         if (!class_exists('ZipArchive')) {
-            throw new Exception("ZIP archive decompressing is not supported.");
+            throw new Exception('ZIP archive decompressing is not supported. ZIP support is missing in PHP.');
         }
 
         $zip = new ZipArchive();

app/Model/GalaxyCluster.php

@@ -181,6 +181,10 @@ class GalaxyCluster extends AppModel
             }
         }
 
+        if (empty($clusterTagNames)) {
+            return $events;
+        }
+
         $clusters = $this->find('all', array(
             'conditions' => array('LOWER(GalaxyCluster.tag_name)' => $clusterTagNames),
             'contain' => array('Galaxy', 'GalaxyElement'),

app/Model/Log.php

@@ -26,7 +26,7 @@ class Log extends AppModel
                     'admin_email',
                     'auth',
                     'auth_fail',
-                    'blacklisted',
+                    'blocklisted',
                     'change_pw',
                     'delete',
                     'disable',
@@ -35,7 +35,7 @@ class Log extends AppModel
                     'edit',
                     'email',
                     'enable',
-                    'enrichment',
+		    'enrichment',
                     'error',
                     'export',
                     'fetchEvent',
@@ -104,8 +104,15 @@ class Log extends AppModel
         if (!empty(Configure::read('MISP.log_skip_db_logs_completely'))) {
             return false;
         }
-        if (Configure::read('MISP.log_client_ip') && isset($_SERVER['REMOTE_ADDR'])) {
-            $this->data['Log']['ip'] = $_SERVER['REMOTE_ADDR'];
+        if (Configure::read('MISP.log_client_ip')) {
+            $ip_header = 'REMOTE_ADDR';
+            if (Configure::read('MISP.log_client_ip_header')) {
+                $ip_header = Configure::read('MISP.log_client_ip_header');
+            }
+
+            if (isset($_SERVER[$ip_header])) {
+                $this->data['Log']['ip'] = $_SERVER[$ip_header];
+            }
         }
         $setEmpty = array('title' => '', 'model' => '', 'model_id' => 0, 'action' => '', 'user_id' => 0, 'change' => '', 'email' => '', 'org' => '', 'description' => '');
         foreach ($setEmpty as $field => $empty) {
@@ -186,6 +193,7 @@ class Log extends AppModel
      * @param string|array $change
      * @return array
      * @throws Exception
+     * @throws InvalidArgumentException
      */
     public function createLogEntry($user, $action, $model, $modelId = 0, $title = '', $change = '')
     {

app/Model/MispObject.php

@@ -2,6 +2,10 @@
 App::uses('AppModel', 'Model');
 App::uses('TmpFileTool', 'Tools');
 
+/**
+ * @property Event $Event
+ * @property SharingGroup $SharingGroup
+ */
 class MispObject extends AppModel
 {
     public $name = 'Object';
@@ -577,7 +581,7 @@ class MispObject extends AppModel
                 $params['page'] = $options['page'];
             }
         }
-        if (Configure::read('MISP.unpublishedprivate')) {
+        if (Configure::read('MISP.unpublishedprivate') && !$user['Role']['perm_site_admin']) {
             $params['conditions']['AND'][] = array('OR' => array('Event.published' => 1, 'Event.orgc_id' => $user['org_id']));
         }
         $results = $this->find('all', $params);
@@ -1052,7 +1056,7 @@ class MispObject extends AppModel
         return true;
     }
 
-    public function deleteObject($object, $hard=false, $unpublish=true)
+    public function deleteObject(array $object, $hard=false, $unpublish=true)
     {
         $id = $object['Object']['id'];
         if ($hard) {
@@ -1501,7 +1505,7 @@ class MispObject extends AppModel
         $continue = true;
         while ($continue) {
             $temp = '';
-            $this->Whitelist = ClassRegistry::init('Whitelist');
+            $this->Allowedlist = ClassRegistry::init('Allowedlist');
             $results = $this->fetchObjects($user, $params, $continue);
             if (empty($results)) {
                 $loop = false;
@@ -1515,7 +1519,7 @@ class MispObject extends AppModel
                 $results = $this->Sightingdb->attachToObjects($results, $user);
             }
             $params['page'] += 1;
-            $results = $this->Whitelist->removeWhitelistedFromArray($results, true);
+            $results = $this->Allowedlist->removeAllowedlistedFromArray($results, true);
             $results = array_values($results);
             $i = 0;
             foreach ($results as $object) {

app/Model/OrgBlocklist.php

@@ -1,8 +1,8 @@
 <?php
 App::uses('AppModel', 'Model');
-class OrgBlacklist extends AppModel
+class OrgBlocklist extends AppModel
 {
-    public $useTable = 'org_blacklists';
+    public $useTable = 'org_blocklists';
 
     public $recursive = -1;
 
@@ -13,13 +13,15 @@ class OrgBlacklist extends AppModel
                     'change' => 'full'),
             'Containable',
     );
-    public $blacklistFields = array('org_uuid', 'comment', 'org_name');
+    public $blocklistFields = array('org_uuid', 'comment', 'org_name');
+
+    public $blocklistTarget = 'org';
 
     public $validate = array(
             'org_uuid' => array(
                     'unique' => array(
                             'rule' => 'isUnique',
-                            'message' => 'Organisation already blacklisted.'
+                            'message' => 'Organisation already blocklisted.'
                     ),
                     'uuid' => array(
                         'rule' => array('custom', '/^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$/'),
@@ -31,8 +33,8 @@ class OrgBlacklist extends AppModel
     public function beforeValidate($options = array())
     {
         parent::beforeValidate();
-        if (empty($this->data['OrgBlacklist']['id'])) {
-            $this->data['OrgBlacklist']['date_created'] = date('Y-m-d H:i:s');
+        if (empty($this->data['OrgBlocklist']['id'])) {
+            $this->data['OrgBlocklist']['date_created'] = date('Y-m-d H:i:s');
         }
         return true;
     }

app/Model/Regexp.php

@@ -63,7 +63,7 @@ class Regexp extends AppModel
         }
         // if we found the original, let's try to find all of the regexp values that match the original in the regexp and replacement fields.
         // We should get a list of all the IDs (and their respective types) of regular expression entries that are duplicates created for various types.
-        // ip-src /127.0.0.1/ -> '' and ip-dst /127.0.0.1/ -> '' (entries that blacklists the ip-source and ip-destination addresses 127.0.0.1) will be returned when editing
+        // ip-src /127.0.0.1/ -> '' and ip-dst /127.0.0.1/ -> '' (entries that blocklists the ip-source and ip-destination addresses 127.0.0.1) will be returned when editing
         // ip-src /127.0.0.1/ -> '', but other /127.0.0.1/ -> 'localhost' will not
         if ($original != null) {
             foreach ($allRegexp as $k => $v) {

app/Model/Server.php

@@ -1,5 +1,6 @@
 <?php
 App::uses('AppModel', 'Model');
+App::uses('GpgTool', 'Tools');
 
 class Server extends AppModel
 {
@@ -180,7 +181,7 @@ class Server extends AppModel
                         'branch' => 1,
                         'baseurl' => array(
                                 'level' => 0,
-                                'description' => __('The base url of the application (in the format https://www.mymispinstance.com). Several features depend on this setting being correctly set to function.'),
+                                'description' => __('The base url of the application (in the format https://www.mymispinstance.com or https://myserver.com/misp). Several features depend on this setting being correctly set to function.'),
                                 'value' => '',
                                 'errorMessage' => __('The currenty set baseurl does not match the URL through which you have accessed the page. Disregard this if you are accessing the page via an alternate URL (for example via IP address).'),
                                 'test' => 'testBaseURL',
@@ -766,16 +767,16 @@ class Server extends AppModel
                                 'test' => 'testPasswordResetText',
                                 'type' => 'string'
                         ),
-                        'enableEventBlacklisting' => array(
+                        'enableEventBlocklisting' => array(
                                 'level' => 1,
-                                'description' => __('Since version 2.3.107 you can start blacklisting event UUIDs to prevent them from being pushed to your instance. This functionality will also happen silently whenever an event is deleted, preventing a deleted event from being pushed back from another instance.'),
+                                'description' => __('Since version 2.3.107 you can start blocklisting event UUIDs to prevent them from being pushed to your instance. This functionality will also happen silently whenever an event is deleted, preventing a deleted event from being pushed back from another instance.'),
                                 'value' => true,
                                 'type' => 'boolean',
                                 'test' => 'testBool'
                         ),
-                        'enableOrgBlacklisting' => array(
+                        'enableOrgBlocklisting' => array(
                                 'level' => 1,
-                                'description' => __('Blacklisting organisation UUIDs to prevent the creation of any event created by the blacklisted organisation.'),
+                                'description' => __('Blocklisting organisation UUIDs to prevent the creation of any event created by the blocklisted organisation.'),
                                 'value' => true,
                                 'type' => 'boolean',
                                 'test' => 'testBool'
@@ -789,6 +790,15 @@ class Server extends AppModel
                                 'type' => 'boolean',
                                 'beforeHook' => 'ipLogBeforeHook'
                         ),
+                        'log_client_ip_header' => array(
+                            'level' => 1,
+                            'description' => __('If log_client_ip is enabled, you can customize which header field contains the client\'s IP address. This is generally used when you have a reverse proxy infront of your MISP instance.'),
+                            'value' => 'REMOTE_ADDR',
+                            'errorMessage' => '',
+                            'test' => 'testForEmpty',
+                            'type' => 'string',
+                            'null' => true,
+                        ),
                         'log_auth' => array(
                                 'level' => 1,
                                 'description' => __('If enabled, MISP will log all successful authentications using API keys. The requested URLs are also logged.'),
@@ -1142,6 +1152,14 @@ class Server extends AppModel
                                 'errorMessage' => '',
                                 'test' => 'testForEmpty',
                                 'type' => 'string',
+                        ),
+                        'obscure_subject' => array(
+                                'level' => 2,
+                                'description' => __('When enabled, subject in signed and encrypted e-mails will not send in unencrypted form.'),
+                                'value' => false,
+                                'errorMessage' => '',
+                                'test' => 'testBool',
+                                'type' => 'boolean',
                         )
                 ),
                 'SMIME' => array(
@@ -2391,47 +2409,53 @@ class Server extends AppModel
         return true;
     }
 
-    private function __getEventIdListBasedOnPullTechnique($technique, $server, $force = false)
+    /**
+     * @param int|string $technique 'full', 'update', remote event ID or remote event UUID
+     * @param array $server
+     * @param bool $force
+     * @return array
+     */
+    private function __getEventIdListBasedOnPullTechnique($technique, array $server, $force = false)
     {
-        if ("full" === $technique) {
-            // get a list of the event_ids on the server
-            $eventIds = $this->getEventIdsFromServer($server, false, null, false, false, 'events', $force);
-            if ($eventIds === 403) {
-                return array('error' => array(1, null));
-            } elseif (is_string($eventIds)) {
-                return array('error' => array(2, $eventIds));
-            }
-
-            // reverse array of events, to first get the old ones, and then the new ones
-            if (!empty($eventIds)) {
-                $eventIds = array_reverse($eventIds);
-            }
-        } elseif ("update" === $technique) {
-            $eventIds = $this->getEventIdsFromServer($server, false, null, true, true, 'events', $force);
-            if ($eventIds === 403) {
-                return array('error' => array(1, null));
-            } elseif (is_string($eventIds)) {
-                return array('error' => array(2, $eventIds));
-            }
-            $eventModel = ClassRegistry::init('Event');
-            $local_event_ids = $eventModel->find('list', array(
+        try {
+            if ("full" === $technique) {
+                // get a list of the event_ids on the server
+                $eventIds = $this->getEventIdsFromServer($server, false, null, false, 'events', $force);
+                // reverse array of events, to first get the old ones, and then the new ones
+                return array_reverse($eventIds);
+            } elseif ("update" === $technique) {
+                $eventIds = $this->getEventIdsFromServer($server, false, null, true, 'events', $force);
+                $eventModel = ClassRegistry::init('Event');
+                $local_event_ids = $eventModel->find('list', array(
                     'fields' => array('uuid'),
                     'recursive' => -1,
-            ));
-            $eventIds = array_intersect($eventIds, $local_event_ids);
-        } elseif (is_numeric($technique)) {
-            $eventIds[] = intval($technique);
-        } else {
-            return array('error' => array(4, null));
+                ));
+                return array_intersect($eventIds, $local_event_ids);
+            } elseif (is_numeric($technique)) {
+                return array(intval($technique));
+            } elseif (Validation::uuid($technique)) {
+                return array($technique);
+            } else {
+                return array('error' => array(4, null));
+            }
+        } catch (HttpException $e) {
+            $this->logException("Could not fetch event IDs from server {$server['Server']['name']}", $e);
+            if ($e->getCode() === 403) {
+                return array('error' => array(1, null));
+            } else {
+                return array('error' => array(2, $e->getMessage()));
+            }
+        } catch (Exception $e) {
+            $this->logException("Could not fetch event IDs from server {$server['Server']['name']}", $e);
+            return array('error' => array(2, $e->getMessage()));
         }
-        return $eventIds;
     }
 
     private function __checkIfEventIsBlockedBeforePull($event)
     {
-        if (Configure::read('MISP.enableEventBlacklisting') !== false) {
-            $this->EventBlacklist = ClassRegistry::init('EventBlacklist');
-            $r = $this->EventBlacklist->find('first', array('conditions' => array('event_uuid' => $event['Event']['uuid'])));
+        if (Configure::read('MISP.enableEventBlocklisting') !== false) {
+            $this->EventBlocklist = ClassRegistry::init('EventBlocklist');
+            $r = $this->EventBlocklist->find('first', array('conditions' => array('event_uuid' => $event['Event']['uuid'])));
             if (!empty($r)) {
                 return true;
             }
@@ -2737,17 +2761,19 @@ class Server extends AppModel
         return $final;
     }
 
-    private function __orgRuleDowngrade($HttpSocket, $request, $server, $filter_rules)
+    /**
+     * @param HttpSocket $HttpSocket
+     * @param array $request
+     * @param array $server
+     * @param array $filter_rules
+     * @return array
+     * @throws JsonException
+     */
+    private function __orgRuleDowngrade(HttpSocket $HttpSocket, array $request, array $server, array $filter_rules)
     {
         $uri = $server['Server']['url'] . '/servers/getVersion';
-        try {
-            $version_response = $HttpSocket->get($uri, false, $request);
-            $body = $version_response->body;
-            $version_response = json_decode($body, true);
-            $version = $version_response['version'];
-        } catch (Exception $e) {
-            return $e->getMessage();
-        }
+        $version_response = $HttpSocket->get($uri, false, $request);
+        $version = $this->jsonDecode($version_response->body)['version'];
         $version = explode('.', $version);
         if ($version[0] <= 2 && $version[1] <= 4 && $version[0] <= 123) {
             $filter_rules['org'] = implode('|', $filter_rules['org']);
@@ -2755,115 +2781,112 @@ class Server extends AppModel
         return $filter_rules;
     }
 
-    // Get an array of event_ids that are present on the remote server
-    public function getEventIdsFromServer($server, $all = false, $HttpSocket=null, $force_uuid=false, $ignoreFilterRules = false, $scope = 'events', $force = false)
+    /**
+     * Get an array of event UUIDs that are present on the remote server.
+     *
+     * @param array $server
+     * @param bool $all
+     * @param HttpSocket|null $HttpSocket
+     * @param bool $ignoreFilterRules
+     * @param string $scope 'events' or 'sightings'
+     * @param bool $force
+     * @return array Array of event UUIDs.
+     * @throws JsonException
+     * @throws InvalidArgumentException
+     */
+    public function getEventIdsFromServer(array $server, $all = false, HttpSocket $HttpSocket = null, $ignoreFilterRules = false, $scope = 'events', $force = false)
     {
-        $url = $server['Server']['url'];
-        if ($ignoreFilterRules) {
-            $filter_rules = array();
-        } else {
-            $filter_rules = $this->filterRuleToParameter($server['Server']['pull_rules']);
-        }
-        $HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
-        $request = $this->setupSyncRequest($server);
-        if (!empty($filter_rules['org'])) {
-            $filter_rules = $this->__orgRuleDowngrade($HttpSocket, $request, $server, $filter_rules);
-        }
-        $uri = $url . '/events/index';
-        $filter_rules['minimal'] = 1;
-        $filter_rules['published'] = 1;
-        try {
-            $response = $HttpSocket->post($uri, json_encode($filter_rules), $request);
-            if ($response->isOk()) {
-                $eventArray = json_decode($response->body, true);
-                // correct $eventArray if just one event
-                if (is_array($eventArray) && isset($eventArray['id'])) {
-                    $tmp = $eventArray;
-                    unset($eventArray);
-                    $eventArray[0] = $tmp;
-                    unset($tmp);
-                }
-                $eventIds = array();
-                if ($all) {
-                    if (!empty($eventArray)) {
-                        if ($scope === 'sightings') {
-                            foreach ($eventArray as $event) {
-                                $localEvent = $this->Event->find('first', array(
-                                        'recursive' => -1,
-                                        'fields' => array('Event.uuid', 'Event.sighting_timestamp'),
-                                        'conditions' => array('Event.uuid' => $event['uuid'])
-                                    ));
-                                if (!empty($localEvent) && $localEvent['Event']['sighting_timestamp'] > $event['sighting_timestamp']) {
-                                    $eventIds[] = $event['uuid'];
-                                }
-                            }
-                        } else {
-                            foreach ($eventArray as $event) {
-                                $eventIds[] = $event['uuid'];
-                            }
-                        }
-                    }
-                } else {
-                    // multiple events, iterate over the array
-                    $this->Event = ClassRegistry::init('Event');
-                    $blacklisting = array();
-                    if (Configure::read('MISP.enableEventBlacklisting') !== false) {
-                        $this->EventBlacklist = ClassRegistry::init('EventBlacklist');
-                        $blacklisting['EventBlacklist'] = array(
-                            'index_field' => 'uuid',
-                            'blacklist_field' => 'event_uuid'
-                        );
-                    }
-                    if (Configure::read('MISP.enableOrgBlacklisting') !== false) {
-                        $this->OrgBlacklist = ClassRegistry::init('OrgBlacklist');
-                        $blacklisting['OrgBlacklist'] = array(
-                            'index_field' => 'orgc_uuid',
-                            'blacklist_field' => 'org_uuid'
-                        );
-                    }
-                    foreach ($eventArray as $k => $event) {
-                        if (1 != $event['published']) {
-                            unset($eventArray[$k]); // do not keep non-published events
-                            continue;
-                        }
-                        foreach ($blacklisting as $type => $blacklist) {
-                            if (!empty($eventArray[$k][$blacklist['index_field']])) {
-                                $blacklist_hit = $this->{$type}->find('first', array(
-                                    'conditions' => array($blacklist['blacklist_field'] => $eventArray[$k][$blacklist['index_field']]),
-                                    'recursive' => -1,
-                                    'fields' => array($type . '.id')
-                                ));
-                                if (!empty($blacklist_hit)) {
-                                    unset($eventArray[$k]);
-                                    continue 2;
-                                }
-                            }
-                        }
-                    }
-                    if (!$force) {
-                        $this->Event->removeOlder($eventArray, $scope);
-                    }
-                    if (!empty($eventArray)) {
-                        foreach ($eventArray as $event) {
-                            if ($force_uuid) {
-                                $eventIds[] = $event['uuid'];
-                            } else {
-                                $eventIds[] = $event['uuid'];
-                            }
-                        }
-                    }
-                }
-                return $eventIds;
-            }
-            if ($response->code == '403') {
-                return 403;
-            }
-        } catch (SocketException $e) {
-            return $e->getMessage();
+        if (!in_array($scope, array('events', 'sightings'))) {
+            throw new InvalidArgumentException("Scope mus be 'events' or 'sightings', '$scope' given.");
         }
 
-        // error, so return error message, since that is handled and everything is expecting an array
-        return "Error: got response code " . $response->code;
+        if ($ignoreFilterRules) {
+            $filterRules = array();
+        } else {
+            $filterRules = $this->filterRuleToParameter($server['Server']['pull_rules']);
+        }
+
+        $HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
+        $request = $this->setupSyncRequest($server);
+        if (!empty($filterRules['org'])) {
+            $filterRules = $this->__orgRuleDowngrade($HttpSocket, $request, $server, $filterRules);
+        }
+        $filterRules['minimal'] = 1;
+        $filterRules['published'] = 1;
+
+        $uri = $server['Server']['url'] . '/events/index';
+        $response = $HttpSocket->post($uri, json_encode($filterRules), $request);
+        if ($response === false) {
+            throw new Exception("Could not reach '$uri'.");
+        }
+        if (!$response->isOk()) {
+            throw new HttpException("Fetching the '$uri' failed with HTTP error {$response->code}: {$response->reasonPhrase}", intval($response->code));
+        }
+
+        $eventArray = $this->jsonDecode($response->body);
+        // correct $eventArray if just one event
+        if (isset($eventArray['id'])) {
+            $eventArray = array($eventArray);
+        }
+        if ($all) {
+            if ($scope === 'sightings') {
+                $this->Event = ClassRegistry::init('Event');
+                $localEvents = $this->Event->find('list', array(
+                    'recursive' => -1,
+                    'fields' => array('Event.uuid', 'Event.sighting_timestamp'),
+                    'conditions' => array('Event.uuid' => array_column($eventArray, 'uuid'))
+                ));
+
+                $eventUuids = array();
+                foreach ($eventArray as $event) {
+                    if (!isset($localEvents[$event['uuid']]) && $localEvents[$event['uuid']] > $event['sighting_timestamp']) {
+                        $eventUuids[] = $event['uuid'];
+                    }
+                }
+            } else {
+                $eventUuids = array_column($eventArray, 'uuid');
+            }
+        } else {
+            if (Configure::read('MISP.enableEventBlocklisting') !== false) {
+                $this->EventBlocklist = ClassRegistry::init('EventBlocklist');
+                $blocklistHits = $this->EventBlocklist->find('list', array(
+                    'recursive' => -1,
+                    'conditions' => array('EventBlocklist.event_uuid' => array_column($eventArray, 'uuid')),
+                    'fields' => array('EventBlocklist.event_uuid', 'EventBlocklist.event_uuid'),
+                ));
+                foreach ($eventArray as $k => $event) {
+                    if (isset($blocklistHits[$event['uuid']])) {
+                        unset($eventArray[$k]);
+                    }
+                }
+            }
+
+            if (Configure::read('MISP.enableOrgBlocklisting') !== false) {
+                $this->OrgBlocklist = ClassRegistry::init('OrgBlocklist');
+                $blocklistHits = $this->OrgBlocklist->find('list', array(
+                    'recursive' => -1,
+                    'conditions' => array('OrgBlocklist.org_uuid' => array_unique(array_column($eventArray, 'orgc_uuid'))),
+                    'fields' => array('OrgBlocklist.org_uuid', 'OrgBlocklist.org_uuid'),
+                ));
+                foreach ($eventArray as $k => $event) {
+                    if (isset($blocklistHits[$event['orgc_uuid']])) {
+                        unset($eventArray[$k]);
+                    }
+                }
+            }
+
+            foreach ($eventArray as $k => $event) {
+                if (1 != $event['published']) {
+                    unset($eventArray[$k]); // do not keep non-published events
+                }
+            }
+            if (!$force) {
+                $this->Event = ClassRegistry::init('Event');
+                $this->Event->removeOlder($eventArray, $scope);
+            }
+            $eventUuids = array_column($eventArray, 'uuid');
+        }
+        return $eventUuids;
     }
 
     public function push($id = null, $technique=false, $jobId = false, $HttpSocket, $user)
@@ -3073,19 +3096,22 @@ class Server extends AppModel
         }
         $this->Sighting = ClassRegistry::init('Sighting');
         $HttpSocket = $this->setupHttpSocket($server, $HttpSocket);
-        $eventIds = $this->getEventIdsFromServer($server, true, $HttpSocket, false, true, 'sightings');
+        try {
+            $eventIds = $this->getEventIdsFromServer($server, true, $HttpSocket, true, 'sightings');
+        } catch (Exception $e) {
+            $this->logException("Could not fetch event IDs from server {$server['Server']['name']}", $e);
+            return $successes;
+        }
         // now process the $eventIds to push each of the events sequentially
-        if (!empty($eventIds)) {
-            // check each event and push sightings when needed
-            foreach ($eventIds as $k => $eventId) {
-                $event = $eventModel->fetchEvent($user, $options = array('event_uuid' => $eventId, 'metadata' => true));
-                if (!empty($event)) {
-                    $event = $event[0];
-                    $event['Sighting'] = $this->Sighting->attachToEvent($event, $user);
-                    $result = $eventModel->uploadEventToServer($event, $server, $HttpSocket, 'sightings');
-                    if ($result === 'Success') {
-                        $successes[] = 'Sightings for event ' .  $event['Event']['id'];
-                    }
+        // check each event and push sightings when needed
+        foreach ($eventIds as $k => $eventId) {
+            $event = $eventModel->fetchEvent($user, $options = array('event_uuid' => $eventId, 'metadata' => true));
+            if (!empty($event)) {
+                $event = $event[0];
+                $event['Sighting'] = $this->Sighting->attachToEvent($event, $user);
+                $result = $eventModel->uploadEventToServer($event, $server, $HttpSocket, 'sightings');
+                if ($result === 'Success') {
+                    $successes[] = 'Sightings for event ' .  $event['Event']['id'];
                 }
             }
         }
@@ -3099,9 +3125,10 @@ class Server extends AppModel
         if ($sa_id == null) {
             if ($event_id == null) {
                 // event_id is null when we are doing a push
-                $ids = $this->getEventIdsFromServer($server, true, $HttpSocket, false, true);
-                // error return strings or ints or throw exceptions
-                if (!is_array($ids)) {
+                try {
+                    $ids = $this->getEventIdsFromServer($server, true, $HttpSocket, true);
+                } catch (Exception $e) {
+                    $this->logException("Could not fetch event IDs from server {$server['Server']['name']}", $e);
                     return false;
                 }
                 $conditions = array('uuid' => $ids);
@@ -3550,7 +3577,10 @@ class Server extends AppModel
         if ($this->testForEmpty($value) !== true) {
             return $this->testForEmpty($value);
         }
-        if ($value != strtolower($this->getProto()) . '://' . $this->getHost()) {
+        $regex = "%^(?<proto>https?)://(?<host>(?:(?:\w|-)+\.)+[a-z]{2,5})(?::(?<port>[0-9]+))?(?<base>/[a-z0-9_\-\.]+)?$%i";
+	if ( !preg_match($regex, $value, $matches)
+                || strtolower($matches['proto']) != strtolower($this->getProto())
+                || strtolower($matches['host']) != strtolower($this->getHost()) ) {
             return 'Invalid baseurl, it has to be in the "https://FQDN" format.';
         }
         return true;
@@ -4722,9 +4752,9 @@ class Server extends AppModel
         $dbActualIndexes = array();
         $dataSource = $this->getDataSource()->config['datasource'];
         if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
-            $sqlGetTable = sprintf('SELECT TABLE_NAME FROM information_schema.tables WHERE table_schema = %s;', "'" . $this->getDataSource()->config['database'] . "'");
+            $sqlGetTable = sprintf('SELECT TABLE_NAME FROM information_schema.tables WHERE table_schema = %s ORDER BY TABLE_NAME;', "'" . $this->getDataSource()->config['database'] . "'");
             $sqlResult = $this->query($sqlGetTable);
-            $tables = HASH::extract($sqlResult, '{n}.tables.TABLE_NAME');
+            $tables = Hash::extract($sqlResult, '{n}.tables.TABLE_NAME');
             foreach ($tables as $table) {
                 $sqlSchema = sprintf(
                     "SELECT %s
@@ -4747,7 +4777,7 @@ class Server extends AppModel
     public function compareDBSchema($dbActualSchema, $dbExpectedSchema)
     {
         // Column that should be ignored while performing the comparison
-        $whiteListFields = array(
+        $allowedlistFields = array(
             'users' => array('external_auth_required', 'external_auth_key'),
         );
         $nonCriticalColumnElements = array('is_nullable', 'collation_name');
@@ -4779,8 +4809,8 @@ class Server extends AppModel
 
                 $additionalKeysInActualSchema = array_diff($existingColumnKeys, $expectedColumnKeys);
                 foreach($additionalKeysInActualSchema as $additionalKeys) {
-                    if (isset($whiteListFields[$tableName]) && in_array($additionalKeys, $whiteListFields[$tableName])) {
-                        continue; // column is whitelisted
+                    if (isset($allowedlistFields[$tableName]) && in_array($additionalKeys, $allowedlistFields[$tableName])) {
+                        continue; // column is allowedlisted
                     }
                     $dbDiff[$tableName][] = array(
                         'description' => sprintf(__('Column `%s` exists but should not'), $additionalKeys),
@@ -4790,8 +4820,8 @@ class Server extends AppModel
                     );
                 }
                 foreach ($keyedExpectedColumn as $columnName => $column) {
-                    if (isset($whiteListFields[$tableName]) && in_array($columnName, $whiteListFields[$tableName])) {
-                        continue; // column is whitelisted
+                    if (isset($allowedlistFields[$tableName]) && in_array($columnName, $allowedlistFields[$tableName])) {
+                        continue; // column is allowedlisted
                     }
                     if (isset($keyedActualColumn[$columnName])) {
                         $colDiff = array_diff_assoc($column, $keyedActualColumn[$columnName]);
@@ -4848,53 +4878,120 @@ class Server extends AppModel
         return $dbDiff;
     }
 
-    public function compareDBIndexes($actualIndex, $expectedIndex, $dbExpectedSchema)
+    /**
+     * Returns `true` if given column for given table contains just unique values.
+     *
+     * @param string $tableName
+     * @param string $columnName
+     * @return bool
+     */
+    private function checkIfColumnContainsJustUniqueValues($tableName, $columnName)
     {
-        $defaultIndexKeylength = 255;
-        $whitelistTables = array();
+        $db = $this->getDataSource();
+        $duplicates = $this->query(
+            sprintf('SELECT %s, COUNT(*) c FROM %s GROUP BY %s HAVING c > 1;',
+                $db->name($columnName), $db->name($tableName), $db->name($columnName))
+        );
+        return empty($duplicates);
+    }
+
+    private function generateSqlDropIndexQuery($tableName, $columnName)
+    {
+        return sprintf('DROP INDEX `%s` ON %s;',
+            $columnName,
+            $tableName
+        );
+    }
+
+    private function generateSqlIndexQuery(array $dbExpectedSchema, $tableName, $columnName, $shouldBeUnique = false, $defaultIndexKeylength = 255)
+    {
+        $columnData = Hash::extract($dbExpectedSchema['schema'][$tableName], "{n}[column_name=$columnName]");
+        if (empty($columnData)) {
+            throw new Exception("Index in db_schema.json is defined for `$tableName.$columnName`, but this column is not defined.");
+        }
+
+        $columnData = $columnData[0];
+        if ($columnData['data_type'] === 'varchar') {
+            $keyLength = sprintf('(%s)', $columnData['character_maximum_length'] < $defaultIndexKeylength ? $columnData['character_maximum_length'] : $defaultIndexKeylength);
+        } elseif ($columnData['data_type'] === 'text') {
+            $keyLength = sprintf('(%s)', $defaultIndexKeylength);
+        } else {
+            $keyLength = '';
+        }
+        return sprintf('CREATE%s INDEX `%s` ON `%s` (`%s`%s);',
+            $shouldBeUnique ? ' UNIQUE' : '',
+            $columnName,
+            $tableName,
+            $columnName,
+            $keyLength
+        );
+    }
+
+    public function compareDBIndexes(array $actualIndex, array $expectedIndex, array $dbExpectedSchema)
+    {
+        $allowedlistTables = array();
         $indexDiff = array();
-        foreach($expectedIndex as $tableName => $indexes) {
+        foreach ($expectedIndex as $tableName => $indexes) {
             if (!array_key_exists($tableName, $actualIndex)) {
                 continue; // If table does not exists, it is covered by the schema diagnostic
-            } elseif(in_array($tableName, $whitelistTables)) {
-                continue; // Ignore whitelisted tables
+            } elseif(in_array($tableName, $allowedlistTables)) {
+                continue; // Ignore allowedlisted tables
             } else {
-                $tableIndexDiff = array_diff($indexes, $actualIndex[$tableName]); // check for missing indexes
-                if (count($tableIndexDiff) > 0) {
-                    foreach($tableIndexDiff as $columnDiff) {
-                        $columnData  = Hash::extract($dbExpectedSchema['schema'][$tableName], sprintf('{n}[column_name=%s]', $columnDiff))[0];
-                        $message = sprintf(__('Column `%s` should be indexed'), $columnDiff);
-                        if ($columnData['data_type'] == 'varchar') {
-                            $keyLength = sprintf('(%s)', $columnData['character_maximum_length'] < $defaultIndexKeylength ? $columnData['character_maximum_length'] : $defaultIndexKeylength);
-                        } elseif ($columnData['data_type'] == 'text') {
-                            $keyLength = sprintf('(%s)', $defaultIndexKeylength);
-                        } else {
-                            $keyLength = '';
-                        }
-                        $sql = sprintf('CREATE INDEX `%s` ON `%s` (`%s`%s);',
-                            $columnDiff,
-                            $tableName,
-                            $columnDiff,
-                            $keyLength
-                        );
+                $tableIndexDiff = array_diff(array_keys($indexes), array_keys($actualIndex[$tableName])); // check for missing indexes
+                foreach ($tableIndexDiff as $columnDiff) {
+                    $shouldBeUnique = $indexes[$columnDiff];
+                    if ($shouldBeUnique && !$this->checkIfColumnContainsJustUniqueValues($tableName, $columnDiff)) {
                         $indexDiff[$tableName][$columnDiff] = array(
-                            'message' => $message,
-                            'sql' => $sql
+                            'message' => __('Column `%s` should be unique indexed, but contains duplicate values', $columnDiff),
+                            'sql' => '',
                         );
+                        continue;
                     }
+
+                    $message = __('Column `%s` should be indexed', $columnDiff);
+                    $indexDiff[$tableName][$columnDiff] = array(
+                        'message' => $message,
+                        'sql' => $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $columnDiff, $shouldBeUnique),
+                    );
                 }
-                $tableIndexDiff = array_diff($actualIndex[$tableName], $indexes); // check for additional indexes
-                if (count($tableIndexDiff) > 0) {
-                    foreach($tableIndexDiff as $columnDiff) {
-                        $message = sprintf(__('Column `%s` is indexed but should not'), $columnDiff);
-                        $sql = sprintf('DROP INDEX `%s` ON %s;',
-                            $columnDiff,
-                            $tableName
-                        );
-                        $indexDiff[$tableName][$columnDiff] = array(
-                            'message' => $message,
-                            'sql' => $sql
-                        );
+                $tableIndexDiff = array_diff(array_keys($actualIndex[$tableName]), array_keys($indexes)); // check for additional indexes
+                foreach ($tableIndexDiff as $columnDiff) {
+                    $message = __('Column `%s` is indexed but should not', $columnDiff);
+                    $indexDiff[$tableName][$columnDiff] = array(
+                        'message' => $message,
+                        'sql' => $this->generateSqlDropIndexQuery($tableName, $columnDiff),
+                    );
+                }
+                foreach ($indexes as $column => $unique) {
+                    if (isset($actualIndex[$tableName][$column]) && $actualIndex[$tableName][$column] != $unique) {
+                        if ($actualIndex[$tableName][$column]) {
+                            $sql = $this->generateSqlDropIndexQuery($tableName, $column);
+                            $sql .= '<br>' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, false);
+
+                            $message = __('Column `%s` has unique index, but should be non unique', $column);
+                            $indexDiff[$tableName][$column] = array(
+                                'message' => $message,
+                                'sql' => $sql,
+                            );
+                        } else {
+                            if (!$this->checkIfColumnContainsJustUniqueValues($tableName, $column)) {
+                                $message = __('Column `%s` should be unique index, but contains duplicate values', $column);
+                                $indexDiff[$tableName][$column] = array(
+                                    'message' => $message,
+                                    'sql' => '',
+                                );
+                                continue;
+                            }
+
+                            $sql = $this->generateSqlDropIndexQuery($tableName, $column);
+                            $sql .= '<br>' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, true);
+
+                            $message = __('Column `%s` should be unique index', $column);
+                            $indexDiff[$tableName][$column] = array(
+                                'message' => $message,
+                                'sql' => $sql,
+                            );
+                        }
                     }
                 }
             }
@@ -4902,16 +4999,27 @@ class Server extends AppModel
         return $indexDiff;
     }
 
+    /**
+     * Returns indexes for given schema and table in array, where key is column name and value is `true` if
+     * index is index is unique, `false` otherwise.
+     *
+     * @param string $database
+     * @param string $table
+     * @return array
+     */
     public function getDatabaseIndexes($database, $table)
     {
         $sqlTableIndex = sprintf(
-            "SELECT DISTINCT TABLE_NAME, COLUMN_NAME FROM information_schema.statistics WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s';",
+            "SELECT DISTINCT TABLE_NAME, COLUMN_NAME, NON_UNIQUE FROM information_schema.statistics WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s';",
             $database,
             $table
         );
         $sqlTableIndexResult = $this->query($sqlTableIndex);
-        $tableIndex = Hash::extract($sqlTableIndexResult, '{n}.statistics.COLUMN_NAME');
-        return $tableIndex;
+        $output = [];
+        foreach ($sqlTableIndexResult as $index) {
+            $output[$index['statistics']['COLUMN_NAME']] = $index['statistics']['NON_UNIQUE'] == 0;
+        }
+        return $output;
     }
 
     public function writeableDirsDiagnostics(&$diagnostic_errors)
@@ -5042,18 +5150,9 @@ class Server extends AppModel
         $gpgStatus = 0;
         if (Configure::read('GnuPG.email') && Configure::read('GnuPG.homedir')) {
             $continue = true;
+            $gpgTool = new GpgTool();
             try {
-                if (!class_exists('Crypt_GPG')) {
-                    if (!stream_resolve_include_path('Crypt/GPG.php')) {
-                        throw new Exception("Crypt_GPG is not installed");
-                    }
-                    require_once 'Crypt/GPG.php';
-                }
-                $gpg = new Crypt_GPG(array(
-                    'homedir' => Configure::read('GnuPG.homedir'),
-                    'gpgconf' => Configure::read('GnuPG.gpgconf'),
-                    'binary' => Configure::read('GnuPG.binary') ?: '/usr/bin/gpg'
-                ));
+                $gpg = $gpgTool->initializeGpg();
             } catch (Exception $e) {
                 $this->logException("Error during initializing GPG.", $e, LOG_NOTICE);
                 $gpgStatus = 2;

app/Model/ShadowAttribute.php

@@ -3,7 +3,11 @@
 App::uses('AppModel', 'Model');
 App::uses('Folder', 'Utility');
 App::uses('File', 'Utility');
+App::uses('AttachmentTool', 'Tools');
 
+/**
+ * @property Event $Event
+ */
 class ShadowAttribute extends AppModel
 {
     public $combinedKeys = array('event_id', 'category', 'type');
@@ -266,24 +270,7 @@ class ShadowAttribute extends AppModel
         if (isset($this->data['ShadowAttribute']['deleted']) && $this->data['ShadowAttribute']['deleted']) {
             $sa = $this->find('first', array('conditions' => array('ShadowAttribute.id' => $this->data['ShadowAttribute']['id']), 'recursive' => -1, 'fields' => array('ShadowAttribute.id', 'ShadowAttribute.event_id', 'ShadowAttribute.type')));
             if ($this->typeIsAttachment($sa['ShadowAttribute']['type'])) {
-                // only delete the file if it exists
-                if ($this->attachmentDirIsS3()) {
-                    $s3 = $this->getS3Client();
-                    $s3->delete('shadow' . DS . $sa['ShadowAttribute']['event_id'] . DS . $sa['ShadowAttribute']['id']);
-                } else {
-                    $attachments_dir = Configure::read('MISP.attachments_dir');
-                    if (empty($attachments_dir)) {
-                        $my_server = ClassRegistry::init('Server');
-                        $attachments_dir = $my_server->getDefaultAttachments_dir();
-                    }
-                    $filepath = $attachments_dir . DS . 'shadow' . DS . $sa['ShadowAttribute']['event_id'] . DS . $sa['ShadowAttribute']['id'];
-                    $file = new File($filepath);
-                    if ($file->exists()) {
-                        if (!$file->delete()) {
-                            throw new InternalErrorException('Delete of file attachment failed. Please report to administrator.');
-                        }
-                    }
-                }
+                $this->loadAttachmentTool()->deleteShadow($sa['ShadowAttribute']['event_id'], $sa['ShadowAttribute']['id']);
             }
         } else {
             if (isset($this->data['ShadowAttribute']['type']) && $this->typeIsAttachment($this->data['ShadowAttribute']['type']) && !empty($this->data['ShadowAttribute']['data'])) {
@@ -308,24 +295,7 @@ class ShadowAttribute extends AppModel
         // delete attachments from the disk
         $this->read(); // first read the attribute from the db
         if ($this->typeIsAttachment($this->data['ShadowAttribute']['type'])) {
-            // only delete the file if it exists
-            if ($this->attachmentDirIsS3()) {
-                $s3 = $this->getS3Client();
-                $s3->delete('shadow' . DS . $this->data['ShadowAttribute']['event_id'] . DS . $this->data['ShadowAttribute']['id']);
-            } else {
-                $attachments_dir = Configure::read('MISP.attachments_dir');
-                if (empty($attachments_dir)) {
-                    $my_server = ClassRegistry::init('Server');
-                    $attachments_dir = $my_server->getDefaultAttachments_dir();
-                }
-                $filepath = $attachments_dir . DS . 'shadow' . DS . $this->data['ShadowAttribute']['event_id'] . DS . $this->data['ShadowAttribute']['id'];
-                $file = new File($filepath);
-                if ($file->exists()) {
-                    if (!$file->delete()) {
-                        throw new InternalErrorException('Delete of file attachment failed. Please report to administrator.');
-                    }
-                }
-            }
+            $this->loadAttachmentTool()->deleteShadow($this->data['ShadowAttribute']['event_id'], $this->data['ShadowAttribute']['id']);
         }
     }
 
@@ -416,69 +386,44 @@ class ShadowAttribute extends AppModel
 
     public function typeIsMalware($type)
     {
-        if (in_array($type, $this->zippedDefinitions)) {
-            return true;
-        } else {
-            return false;
-        }
+        return in_array($type, $this->zippedDefinitions);
     }
 
     public function typeIsAttachment($type)
     {
-        if ((in_array($type, $this->zippedDefinitions)) || (in_array($type, $this->uploadDefinitions))) {
-            return true;
-        } else {
-            return false;
+        return in_array($type, $this->zippedDefinitions) || in_array($type, $this->uploadDefinitions);
+    }
+
+    public function base64EncodeAttachment(array $attribute)
+    {
+        try {
+            return base64_encode($this->getAttachment($attribute));
+        } catch (NotFoundException $e) {
+            $this->log($e->getMessage(), LOG_NOTICE);
+            return '';
         }
     }
 
-    public function base64EncodeAttachment($attribute)
+    public function getAttachment($attribute, $path_suffix='')
     {
-        $attachments_dir = Configure::read('MISP.attachments_dir');
-        if (empty($attachments_dir)) {
-            $my_server = ClassRegistry::init('Server');
-            $attachments_dir = $my_server->getDefaultAttachments_dir();
-        }
-
-        if ($this->attachmentDirIsS3()) {
-            $s3 = $this->getS3Client();
-            $content = $s3->download('shadow' . DS . $attribute['event_id'] . DS. $attribute['id']);
-        } else {
-            $filepath = $attachments_dir . DS . 'shadow' . DS . $attribute['event_id'] . DS. $attribute['id'];
-            $file = new File($filepath);
-            if (!$file->exists()) {
-                return '';
-            }
-            $content = $file->read();
-        }
-        return base64_encode($content);
+        return $this->loadAttachmentTool()->getShadowContent($attribute['event_id'], $attribute['id'], $path_suffix);
     }
 
     public function saveBase64EncodedAttachment($attribute)
     {
-        $attachments_dir = Configure::read('MISP.attachments_dir');
-        if (empty($attachments_dir)) {
-            $my_server = ClassRegistry::init('Server');
-            $attachments_dir = $my_server->getDefaultAttachments_dir();
-        }
-        if ($this->attachmentDirIsS3()) {
-            $s3 = $this->getS3Client();
-            $decodedData = base64_decode($attribute['data']);
-            $s3->upload('shadow' . DS . $attribute['event_id'], $decodedData);
-            return true;
-        } else {
-            $rootDir = $attachments_dir . DS . 'shadow' . DS . $attribute['event_id'];
-            $dir = new Folder($rootDir, true);                      // create directory structure
-            $destpath = $rootDir . DS . $attribute['id'];
-            $file = new File($destpath, true);                      // create the file
-            $decodedData = base64_decode($attribute['data']);       // decode
-            if ($file->write($decodedData)) {                       // save the data
-                return true;
-            } else {
-                // error
-                return false;
-            }
-        }
+        $data = base64_decode($attribute['data']);
+        return $this->loadAttachmentTool()->saveShadow($attribute['event_id'], $attribute['id'], $data);
+    }
+
+    /**
+     * @param array $shadowAttribute
+     * @param string $path_suffix
+     * @return File
+     * @throws Exception
+     */
+    public function getAttachmentFile(array $shadowAttribute, $path_suffix='')
+    {
+        return $this->loadAttachmentTool()->getShadowFile($shadowAttribute['event_id'], $shadowAttribute['id'], $path_suffix);
     }
 
     public function checkComposites()
@@ -584,7 +529,7 @@ class ShadowAttribute extends AppModel
                         'contactalert' => 1,
                         'disabled' => 0
                 ),
-                'fields' => array('email', 'gpgkey', 'certif_public', 'contactalert', 'id')
+                'fields' => array('email', 'gpgkey', 'certif_public', 'contactalert', 'id', 'disabled'),
         ));
 
         $body = "Hello, \n\n";
@@ -757,19 +702,20 @@ class ShadowAttribute extends AppModel
                 $objectDistribution['(SELECT sharing_group_id FROM objects WHERE objects.id = Attribute.object_id)'] = $sgids;
                 $attributeDistribution['Attribute.sharing_group_id'] = $sgids;
             }
+            $unpublishedPrivate = Configure::read('MISP.unpublishedprivate');
             $conditions = array(
                 'AND' => array(
                     'OR' => array(
                         'Event.org_id' => $user['org_id'],
-                        'AND' => array(
-                            'OR' => array(
-                                'Event.distribution' => array(1,2,3,5),
-                                'AND '=> array(
-                                    'Event.distribution' => 4,
-                                    'Event.sharing_group_id' => $sgids,
-                                )
-                            )
-                        )
+                        ['AND' => [
+                            'Event.distribution' => array(1,2,3,5),
+                            $unpublishedPrivate ? ['Event.published' => 1] : [],
+                        ]],
+                        ['AND' => [
+                            'Event.distribution' => 4,
+                            'Event.sharing_group_id' => $sgids,
+                            $unpublishedPrivate ? ['Event.published' => 1] : [],
+                        ]],
                     ),
                     array(
                         'OR' => array(

app/Model/SharingGroup.php

@@ -815,15 +815,6 @@ class SharingGroup extends AppModel
             } else {
                 $id = $id['SharingGroup']['id'];
             }
-        } else {
-            $temp = $this->find('first', array(
-                'conditions' => array('SharingGroup.id' => $id),
-                'recursive' => -1,
-                'fields' => array('SharingGroup.id')
-            ));
-            if (empty($temp)) {
-                return false;
-            }
         }
         if ($readOnly) {
             if (!$this->checkIfAuthorised($user, $id)) {

app/Model/Sighting.php

@@ -207,7 +207,7 @@ class Sighting extends AppModel
     /**
      * @param array $event
      * @param array $user
-     * @param array|int|null $attribute Attribute array or attribute ID
+     * @param array|int|null $attribute Attribute model or attribute ID
      * @param bool $extraConditions
      * @return array|int
      */
@@ -220,7 +220,7 @@ class Sighting extends AppModel
 
         $contain = [];
         $conditions = array('Sighting.event_id' => $event['Event']['id']);
-        if (is_array($attribute)) {
+        if (isset($attribute['Attribute']['id'])) {
             $conditions['Sighting.attribute_id'] = $attribute['Attribute']['id'];
         } elseif (is_numeric($attribute)) {
             $conditions['Sighting.attribute_id'] = $attribute;
@@ -528,14 +528,6 @@ class Sighting extends AppModel
         $conditions = array(
             'Sighting.date_sighting >' => $this->getMaximumRange(),
             ucfirst($context) . 'Tag.tag_id' => $tagList
-
-        );
-        $contain = array(
-            ucfirst($context) => array(
-                ucfirst($context) . 'Tag' => array(
-                    'Tag'
-                )
-            )
         );
         if ($type !== false) {
             $conditions['Sighting.type'] = $type;
@@ -545,15 +537,16 @@ class Sighting extends AppModel
             'recursive' => -1,
             'contain' => array(ucfirst($context) . 'Tag'),
             'conditions' => $conditions,
-            'fields' => array('Sighting.id', 'Sighting.' . $context . '_id', 'Sighting.date_sighting', ucfirst($context) . 'Tag.tag_id')
+            'fields' => array('Sighting.' . $context . '_id', 'Sighting.date_sighting')
         ));
         $sightingsRearranged = array();
         foreach ($sightings as $sighting) {
             $date = date("Y-m-d", $sighting['Sighting']['date_sighting']);
-            if (isset($sightingsRearranged[$sighting['Sighting'][$context . '_id']][$date])) {
-                $sightingsRearranged[$sighting['Sighting'][$context . '_id']][$date]++;
+            $contextId = $sighting['Sighting'][$context . '_id'];
+            if (isset($sightingsRearranged[$contextId][$date])) {
+                $sightingsRearranged[$contextId][$date]++;
             } else {
-                $sightingsRearranged[$sighting['Sighting'][$context . '_id']][$date] = 1;
+                $sightingsRearranged[$contextId][$date] = 1;
             }
         }
         return $sightingsRearranged;
@@ -825,32 +818,34 @@ class Sighting extends AppModel
     {
         $HttpSocket = $this->setupHttpSocket($server);
         $this->Server = ClassRegistry::init('Server');
-        $eventIds = $this->Server->getEventIdsFromServer($server, false, $HttpSocket, false, false, 'sightings');
+        try {
+            $eventIds = $this->Server->getEventIdsFromServer($server, false, $HttpSocket, false, 'sightings');
+        } catch (Exception $e) {
+            $this->logException("Could not fetch event IDs from server {$server['Server']['name']}", $e);
+            return 0;
+        }
         $saved = 0;
         // now process the $eventIds to pull each of the events sequentially
-        if (!empty($eventIds)) {
-            // download each event and save sightings
-            foreach ($eventIds as $k => $eventId) {
-                try {
-                    $event = $this->Event->downloadEventFromServer($eventId, $server);
-                } catch (Exception $e) {
-                    $this->logException('Failed downloading the event ' . $eventId, $e);
-                    continue;
-                }
-
-                $sightings = array();
-                if(!empty($event) && !empty($event['Event']['Attribute'])) {
-                    foreach($event['Event']['Attribute'] as $attribute) {
-                        if(!empty($attribute['Sighting'])) {
-                            $sightings = array_merge($sightings, $attribute['Sighting']);
-                        }
+        // download each event and save sightings
+        foreach ($eventIds as $k => $eventId) {
+            try {
+                $event = $this->Event->downloadEventFromServer($eventId, $server);
+            } catch (Exception $e) {
+                $this->logException("Failed downloading the event $eventId from {$server['Server']['name']}.", $e);
+                continue;
+            }
+            $sightings = array();
+            if (!empty($event) && !empty($event['Event']['Attribute'])) {
+                foreach ($event['Event']['Attribute'] as $attribute) {
+                    if (!empty($attribute['Sighting'])) {
+                        $sightings = array_merge($sightings, $attribute['Sighting']);
                     }
                 }
-                if(!empty($event) && !empty($sightings)) {
-                    $result = $this->bulkSaveSightings($event['Event']['uuid'], $sightings, $user, $server['Server']['id']);
-                    if (is_numeric($result)) {
-                        $saved += $result;
-                    }
+            }
+            if (!empty($event) && !empty($sightings)) {
+                $result = $this->bulkSaveSightings($event['Event']['uuid'], $sightings, $user, $server['Server']['id']);
+                if (is_numeric($result)) {
+                    $saved += $result;
                 }
             }
         }

app/Model/Tag.php

@@ -1,6 +1,10 @@
 <?php
 App::uses('AppModel', 'Model');
 
+/**
+ * @property EventTag $EventTag
+ * @property AttributeTag $AttributeTag
+ */
 class Tag extends AppModel
 {
     public $useTable = 'tags';
@@ -163,16 +167,15 @@ class Tag extends AppModel
         }
     }
 
-    public function fetchUsableTags($user)
+    public function fetchUsableTags(array $user)
     {
         $conditions = array();
         if (!$user['Role']['perm_site_admin']) {
-            $conditions['Tag.org_id'] = array(0, $this->Auth->user('org_id'));
-            $conditions['Tag.user_id'] = array(0, $this->Auth->user('id'));
+            $conditions['Tag.org_id'] = array(0, $user['User']['org_id']);
+            $conditions['Tag.user_id'] = array(0, $user['User']['id']);
             $conditions['Tag.hide_tag'] = 0;
         }
-        $tags = $this->find('all', array('conditions' => $conditions, 'recursive' => -1));
-        return $tags;
+        return $this->find('all', array('conditions' => $conditions, 'recursive' => -1));
     }
 
     // find all of the tag ids that belong to the accepted tag names and the rejected tag names

app/Model/TagCollection.php

@@ -31,7 +31,7 @@ class TagCollection extends AppModel
         )
     );
 
-    public $whitelistedItems = false;
+    public $allowedlistedItems = false;
 
     public $validate = array(
         'name' => array(

app/Model/Taxonomy.php

@@ -197,8 +197,8 @@ class Taxonomy extends AppModel
         $conditions = array();
         if ($user) {
             if (!$user['Role']['perm_site_admin']) {
-                $conditions = array('Tag.org_id' => array(0, $user['org_id']));
-                $conditions = array('Tag.user_id' => array(0, $user['id']));
+                $conditions[] = array('Tag.org_id' => array(0, $user['org_id']));
+                $conditions[] = array('Tag.user_id' => array(0, $user['id']));
             }
         }
         if (Configure::read('MISP.incoming_tags_disabled_by_default')) {
@@ -570,6 +570,7 @@ class Taxonomy extends AppModel
             $tagShortened = $this->stripLastTagComponent($tagName);
             if ($newTagShortened == $tagShortened) {
                 $prefixIsFree = false;
+                break;
             }
         }
         if (!$prefixIsFree) {

app/Model/User.php

@@ -3,6 +3,7 @@ App::uses('AppModel', 'Model');
 App::uses('AuthComponent', 'Controller/Component');
 App::uses('RandomTool', 'Tools');
 App::uses('GpgTool', 'Tools');
+App::uses('SendEmail', 'Tools');
 
 /**
  * @property Log $Log
@@ -221,6 +222,9 @@ class User extends AppModel
         'Containable'
     );
 
+    /** @var Crypt_GPG|null|false */
+    private $gpg;
+
     public function beforeValidate($options = array())
     {
         if (!isset($this->data['User']['id'])) {
@@ -310,20 +314,18 @@ class User extends AppModel
         }
 
         // we have a clean, hopefully public, key here
+        $gpg = $this->initializeGpg();
+        if (!$gpg) {
+            return true;
+        }
         try {
-            $gpg = $this->initializeGpg();
-            try {
-                $keyImportOutput = $gpg->importKey($check['gpgkey']);
-                if (!empty($keyImportOutput['fingerprint'])) {
-                    return true;
-                }
-            } catch (Exception $e) {
-                $this->log($e->getMessage());
-                return false;
+            $keyImportOutput = $gpg->importKey($check['gpgkey']);
+            if (!empty($keyImportOutput['fingerprint'])) {
+                return true;
             }
         } catch (Exception $e) {
-            $this->log($e->getMessage());
-            return true;
+            $this->logException("Exception during importing GPG key", $e);
+            return false;
         }
     }
 
@@ -450,12 +452,11 @@ class User extends AppModel
                 )));
     }
 
-    public function verifySingleGPG($user, $gpg = false)
+    public function verifySingleGPG($user, $gpg = null)
     {
-        if (!$gpg) {
-            try {
-                $gpg = $this->initializeGpg();
-            } catch (Exception $e) {
+        if ($gpg === null) {
+            $gpg = $this->initializeGpg();
+            if (!$gpg) {
                 $result[2] = 'GnuPG is not configured on this system.';
                 $result[0] = true;
                 return $result;
@@ -503,7 +504,6 @@ class User extends AppModel
     public function verifyGPG($id = false)
     {
         $this->Behaviors->detach('Trim');
-        $results = array();
         $conditions = array('not' => array('gpgkey' => ''));
         if ($id !== false) {
             $conditions['User.id'] = $id;
@@ -513,9 +513,13 @@ class User extends AppModel
             'recursive' => -1,
         ));
         if (empty($users)) {
-            return $results;
+            return [];
         }
         $gpg = $this->initializeGpg();
+        if (!$gpg) {
+            return [];
+        }
+        $results = [];
         foreach ($users as $k => $user) {
             $results[$user['User']['id']] = $this->verifySingleGPG($user, $gpg);
         }
@@ -524,49 +528,17 @@ class User extends AppModel
 
     private function testSmimeCertificate($certif_public)
     {
-        $result = array();
+        $sendEmail = new SendEmail();
         try {
-            App::uses('Folder', 'Utility');
-            App::uses('FileAccessTool', 'Tools');
-            $fileAccessTool = new FileAccessTool();
-            $dir = APP . 'tmp' . DS . 'SMIME';
-            if (!file_exists($dir)) {
-                if (!mkdir($dir, 0750, true)) {
-                    throw new MethodNotAllowedException('The SMIME temp directory is not writeable (app/tmp/SMIME).');
-                }
-            }
-            $tempFile = $fileAccessTool->createTempFile($dir, 'SMIME');
-            $msg_test = $fileAccessTool->writeToFile($tempFile, 'test');
-            $msg_test_encrypted = $fileAccessTool->createTempFile($dir, 'SMIME');
-            // encrypt it
-            if (openssl_pkcs7_encrypt($msg_test, $msg_test_encrypted, $certif_public, null, 0, OPENSSL_CIPHER_AES_256_CBC)) {
-                $parse = openssl_x509_parse($certif_public);
-                // Valid certificate ?
-                $now = new DateTime("now");
-                $validTo_time_t_epoch = $parse['validTo_time_t'];
-                $validTo_time_t = new DateTime("@$validTo_time_t_epoch");
-                if ($validTo_time_t > $now) {
-                    // purposes smimeencrypt ?
-                    if (($parse['purposes'][5][0] == 1) && ($parse['purposes'][5][2] == 'smimeencrypt')) {
-                        $result = true;
-                    } else {
-                        // openssl_pkcs7_encrypt good -- Model/User purposes is NOT GOOD'
-                        $result = 'This certificate cannot be used to encrypt email';
-                    }
-                } else {
-                    // openssl_pkcs7_encrypt good -- Model/User expired;
-                    $result = 'This certificate is expired';
-                }
-            } else {
-                // openssl_pkcs7_encrypt NOT good -- Model/User
-                $result = 'This certificate cannot be used to encrypt email';
-            }
+            $sendEmail->testSmimeCertificate($certif_public);
+            return true;
         } catch (Exception $e) {
-            $this->log($e->getMessage());
+            if ($e->getPrevious()) {
+                return $e->getMessage() . ": " . $e->getPrevious()->getMessage();
+            }
+
+            return $e->getMessage();
         }
-        unlink($msg_test);
-        unlink($msg_test_encrypted);
-        return $result;
     }
 
     public function verifyCertificate()
@@ -586,56 +558,64 @@ class User extends AppModel
         return $results;
     }
 
-    public function getPGP($id)
+    /**
+     * If you want to check if user has GPG or X.509 or send encrypted emails to that user, you need user keys. But by
+     * default, keys are part of default user model. This method add that keys to user model.
+     *
+     * @param array $user
+     * @return array
+     * @throws Exception
+     */
+    public function fillKeysToUser(array $user)
     {
+        if (empty($user['id'])) {
+            throw new InvalidArgumentException("Invalid user model provided, not ID found.");
+        }
         $result = $this->find('first', array(
             'recursive' => -1,
-            'fields' => array('id', 'gpgkey'),
-            'conditions' => array('id' => $id),
+            'fields' => array('certif_public', 'gpgkey'),
+            'conditions' => array('id' => $user['id']),
         ));
-        return $result['User']['gpgkey'];
+        if (!$result) {
+            throw new Exception("User with ID {$user['id']} not found.");
+        }
+        $user['gpgkey'] = $result['User']['gpgkey'];
+        $user['certif_public'] = $result['User']['certif_public'];
+        return $user;
     }
 
-    public function getCertificate($id)
-    {
-        $result = $this->find('first', array(
-            'recursive' => -1,
-            'fields' => array('id', 'certif_public'),
-            'conditions' => array('id' => $id),
-        ));
-        return $result['User']['certif_public'];
-    }
-
-    // get the current user and rearrange it to be in the same format as in the auth component
-    public function getAuthUser($id)
+    /**
+     * @param int $id
+     * @return array|null
+     */
+    public function getUserById($id)
     {
         if (empty($id)) {
             throw new NotFoundException('Invalid user ID.');
         }
-        $conditions = array('User.id' => $id);
-        $user = $this->find(
+        return $this->find(
             'first',
             array(
-                'conditions' => $conditions,
+                'conditions' => array('User.id' => $id),
                 'recursive' => -1,
                 'contain' => array(
                     'Organisation',
                     'Role',
                     'Server',
-                    'UserSetting'
+                    'UserSetting',
                 )
             )
         );
+    }
+
+    // get the current user and rearrange it to be in the same format as in the auth component
+    public function getAuthUser($id)
+    {
+        $user = $this->getUserById($id);
         if (empty($user)) {
             return $user;
         }
-        // Rearrange it a bit to match the Auth object created during the login
-        $user['User']['Role'] = $user['Role'];
-        $user['User']['Organisation'] = $user['Organisation'];
-        $user['User']['Server'] = $user['Server'];
-        $user['User']['UserSetting'] = $user['UserSetting'];
-        unset($user['Organisation'], $user['Role'], $user['Server']);
-        return $user['User'];
+        return $this->rearrangeToAuthForm($user);
     }
 
     // get the current user and rearrange it to be in the same format as in the auth component
@@ -646,11 +626,7 @@ class User extends AppModel
         if (empty($user)) {
             return $user;
         }
-        // Rearrange it a bit to match the Auth object created during the login
-        $user['User']['Role'] = $user['Role'];
-        $user['User']['Organisation'] = $user['Organisation'];
-        $user['User']['Server'] = $user['Server'];
-        return $user['User'];
+        return $this->rearrangeToAuthForm($user);
     }
 
     public function getAuthUserByExternalAuth($auth_key)
@@ -671,11 +647,28 @@ class User extends AppModel
         if (empty($user)) {
             return $user;
         }
-        // Rearrange it a bit to match the Auth object created during the login
+        return $this->rearrangeToAuthForm($user);
+    }
+
+    /**
+     * User model is a mess. Sometimes it is necessary to convert User model to form that is created during the login
+     * process. This method do that work for you.
+     *
+     * @param array $user
+     * @return array
+     */
+    public function rearrangeToAuthForm(array $user)
+    {
+        if (!isset($user['User'])) {
+            throw new InvalidArgumentException('Invalid user model provided.');
+        }
+
         $user['User']['Role'] = $user['Role'];
         $user['User']['Organisation'] = $user['Organisation'];
         $user['User']['Server'] = $user['Server'];
-        unset($user['Organisation'], $user['Role'], $user['Server']);
+        if (isset($user['UserSetting'])) {
+            $user['User']['UserSetting'] = $user['UserSetting'];
+        }
         return $user['User'];
     }
 
@@ -683,7 +676,6 @@ class User extends AppModel
     // parameters are an array of org IDs that are owners (for an event this would be orgc and org)
     public function getUsersWithAccess($owners = array(), $distribution, $sharing_group_id = 0, $userConditions = array())
     {
-        $sgModel = ClassRegistry::init('SharingGroup');
         $conditions = array();
         $validOrgs = array();
         $all = true;
@@ -696,6 +688,7 @@ class User extends AppModel
 
         // add all orgs to the conditions that can see the SG
         if ($distribution == 4) {
+            $sgModel = ClassRegistry::init('SharingGroup');
             $sgOrgs = $sgModel->getOrgsWithAccess($sharing_group_id);
             if ($sgOrgs === true) {
                 $all = true;
@@ -723,8 +716,8 @@ class User extends AppModel
         $users = $this->find('all', array(
             'conditions' => $conditions,
             'recursive' => -1,
-            'fields' => array('id', 'email', 'gpgkey', 'certif_public', 'org_id'),
-            'contain' => array('Role' => array('fields' => array('perm_site_admin'))),
+            'fields' => array('id', 'email', 'gpgkey', 'certif_public', 'org_id', 'disabled'),
+            'contain' => ['Role' => ['fields' => ['perm_site_admin']], 'Organisation' => ['fields' => ['id']]],
         ));
         foreach ($users as $k => $user) {
             $user = $user['User'];
@@ -734,57 +727,17 @@ class User extends AppModel
         return $users;
     }
 
-    public function sendEmailExternal($user, $params)
+    /**
+     * @param $user - deprecated
+     * @param array $params
+     * @throws Crypt_GPG_Exception
+     * @throws SendEmailException
+     */
+    public function sendEmailExternal($user, array $params)
     {
-        $this->Log = ClassRegistry::init('Log');
-        $params['body'] = str_replace('\n', PHP_EOL, $params['body']);
-        $Email = new CakeEmail();
-        $recipient = array('User' => array('email' => $params['to']));
-        $failed = false;
-        $mock = false;
-        if (!empty($params['gpgkey'])) {
-            $recipient['User']['gpgkey'] = $params['gpgkey'];
-            $encryptionResult = $this->__encryptUsingGPG($Email, $params['body'], $params['subject'], $recipient);
-            if (isset($encryptionResult['failed'])) {
-                $mock = true;
-            }
-            if (isset($encryptionResult['failureReason'])) {
-                $failureReason = $encryptionResult['failureReason'];
-            }
-        }
-        if (!$failed) {
-            $replyToLog = '';
-            $user = array('User' => $user);
-            $attachments = array();
-            $Email->replyTo($params['reply-to']);
-            if (!empty($params['requestor_gpgkey'])) {
-                $attachments['gpgkey.asc'] = array(
-                    'data' => $params['requestor_gpgkey']
-                );
-            }
-            $Email->from(Configure::read('MISP.email'));
-            $Email->returnPath(Configure::read('MISP.email'));
-            $Email->to($params['to']);
-            $Email->subject($params['subject']);
-            $Email->emailFormat('text');
-            if (!empty($params['attachments'])) {
-                foreach ($params['attachments'] as $key => $value) {
-                    $attachments[$k] = array('data' => $value);
-                }
-            }
-            $Email->attachments($attachments);
-            if (!empty(Configure::read('MISP.disable_emailing')) || !empty($params['mock'])) {
-                $Email->transport('Debug');
-                $mock = true;
-            }
-            $result = $Email->send($params['body']);
-            $Email->reset();
-            if ($result && !$mock) {
-                return true;
-            }
-            return $result;
-        }
-        return false;
+        $gpg = $this->initializeGpg();
+        $sendEmail = new SendEmail($gpg);
+        $sendEmail->sendExternal($params);
     }
 
     // all e-mail sending is now handled by this method
@@ -792,257 +745,50 @@ class User extends AppModel
     // the remaining two parameters are the e-mail subject and a secondary user object which will be used as the replyto address if set. If it is set and an encryption key for the replyTo user exists, then his/her public key will also be attached
     public function sendEmail($user, $body, $bodyNoEnc = false, $subject, $replyToUser = false)
     {
+        if ($user['User']['disabled']) {
+            return true;
+        }
+
         $this->Log = ClassRegistry::init('Log');
-        if (Configure::read('MISP.disable_emailing')) {
-            $this->Log->create();
-            $this->Log->save(array(
-                    'org' => 'SYSTEM',
-                    'model' => 'User',
-                    'model_id' => $user['User']['id'],
-                    'email' => $user['User']['email'],
-                    'action' => 'email',
-                    'title' => 'Email to ' . $user['User']['email'] . ', titled "' . $subject . '" failed. Reason: Emailing is currently disabled on this instance.',
-                    'change' => null,
-            ));
-            return true;
-        }
-        if (!empty($user['User']['disabled'])) {
-            return true;
-        }
-        $failed = false;
-        $failureReason = "";
-        // check if the e-mail can be encrypted
-        $canEncryptGPG = isset($user['User']['gpgkey']) && !empty($user['User']['gpgkey']);
-        $canEncryptSMIME = isset($user['User']['certif_public']) && !empty($user['User']['certif_public']) && Configure::read('SMIME.enabled');
+        $replyToLog = $replyToUser ? ' from ' . $replyToUser['User']['email'] : '';
 
-        // If bodyonlyencrypted is enabled and the user has no encryption key, use the alternate body (if it exists)
-        if (Configure::read('GnuPG.bodyonlyencrypted') && !$canEncryptSMIME && !$canEncryptGPG && $bodyNoEnc) {
-            $body = $bodyNoEnc;
-        }
-        $body = str_replace('\n', PHP_EOL, $body);
-
-        $Email = new CakeEmail();
-        // If we cannot encrypt the mail and the server settings restricts sending unencrypted messages, return false
-        if (!$failed && Configure::read('GnuPG.onlyencrypted') && !$canEncryptGPG && !$canEncryptSMIME) {
-            $failed = true;
-            $failureReason = " encrypted messages are enforced and the message could not be encrypted for this user as no valid encryption key was found.";
-        }
-        // Let's encrypt the message if we can
-        if (!$failed && $canEncryptGPG) {
-            $encryptionResult = $this->__encryptUsingGPG($Email, $body, $subject, $user);
-            if (isset($encryptionResult['failed'])) {
-                $failed = true;
-            }
-            if (isset($encryptionResult['failureReason'])) {
-                $failureReason = $encryptionResult['failureReason'];
-            }
-        }
-        // SMIME if not GPG key
-        if (!$failed && !$canEncryptGPG && $canEncryptSMIME) {
-            $encryptionResult = $this->__encryptUsingSmime($Email, $body, $subject, $user);
-            if (isset($encryptionResult['failed'])) {
-                $failed = true;
-            }
-            if (isset($encryptionResult['failureReason'])) {
-                $failureReason = $encryptionResult['failureReason'];
-            }
-        }
-        $replyToLog = '';
-        if (!$failed) {
-            $result = $this->__finaliseAndSendEmail($replyToUser, $Email, $replyToLog, $user, $subject, $body);
-        }
-        $this->Log = ClassRegistry::init('Log');
-        $this->Log->create();
-        if (!$failed && $result) {
-            $this->Log->save(array(
-                    'org' => 'SYSTEM',
-                    'model' => 'User',
-                    'model_id' => $user['User']['id'],
-                    'email' => $user['User']['email'],
-                    'action' => 'email',
-                    'title' => 'Email ' . $replyToLog  . ' to ' . $user['User']['email'] . ' sent, titled "' . $subject . '".',
-                    'change' => null,
-            ));
-            return true;
-        } else {
-            if (empty($failureReason)) {
-                $failureReason = " there was an error sending the e-mail.";
-            }
-            $this->Log->save(array(
-                    'org' => 'SYSTEM',
-                    'model' => 'User',
-                    'model_id' => $user['User']['id'],
-                    'email' => $user['User']['email'],
-                    'action' => 'email',
-                    'title' => 'Email ' . $replyToLog  . ' to ' . $user['User']['email'] . ', titled "' . $subject . '" failed. Reason: ' . $failureReason,
-                    'change' => null,
-            ));
-        }
-        return false;
-    }
-
-    private function __finaliseAndSendEmail($replyToUser, &$Email, &$replyToLog, $user, $subject, $body, $additionalAttachments = false)
-    {
-        // If the e-mail is sent on behalf of a user, then we want the target user to be able to respond to the sender
-        // For this reason we should also attach the public key of the sender along with the message (if applicable)
-        $attachments = array();
-        if ($replyToUser != false) {
-            $Email->replyTo($replyToUser['User']['email']);
-            if (!empty($replyToUser['User']['gpgkey'])) {
-                $attachments['gpgkey.asc'] = array(
-                    'data' => $replyToUser['User']['gpgkey']
-                );
-            } elseif (!empty($replyToUser['User']['certif_public'])) {
-                $attachments[$replyToUser['User']['email'] . '.pem'] = array(
-                    'data' => $replyToUser['User']['certif_public']
-                );
-            }
-            $replyToLog = 'from ' . $replyToUser['User']['email'];
-        }
-        $Email->from(Configure::read('MISP.email'));
-        $Email->returnPath(Configure::read('MISP.email'));
-        $Email->to($user['User']['email']);
-        $Email->subject($subject);
-        $Email->emailFormat('text');
-        if (!empty($additionalAttachments)) {
-            foreach ($additionalAttachments as $key => $value) {
-                $attachments[$k] = array('data' => $value);
-            }
-        }
-        $Email->attachments($attachments);
+        $gpg = $this->initializeGpg();
+        $sendEmail = new SendEmail($gpg);
         try {
-            $result = $Email->send($body);
-        } catch (Exception $e) {
-            $this->Log = ClassRegistry::init('Log');
+            $encrypted = $sendEmail->sendToUser($user, $subject, $body, $bodyNoEnc ?: null, $replyToUser ?: array());
+
+        } catch (SendEmailException $e) {
+            $this->logException("Exception during sending e-mail", $e);
+            $this->Log->create();
             $this->Log->save(array(
                 'org' => 'SYSTEM',
                 'model' => 'User',
                 'model_id' => $user['User']['id'],
                 'email' => $user['User']['email'],
-                'action' => 'send_mail',
-                'title' => sprintf(__('Could not send mail. Reasons: %s'), $e->getMessage()),
+                'action' => 'email',
+                'title' => 'Email' . $replyToLog . ' to ' . $user['User']['email'] . ', titled "' . $subject . '" failed. Reason: ' . $e->getMessage(),
                 'change' => null,
             ));
-            $result = false;
+            return false;
         }
-        $Email->reset();
-        return $result;
-    }
 
-    private function __encryptUsingGPG(&$Email, &$body, $subject, $user)
-    {
-        $failed = false;
-        // Sign the body
-        require_once 'Crypt/GPG.php';
-        try {
-            $gpg = new Crypt_GPG(array('homedir' => Configure::read('GnuPG.homedir'), 'gpgconf' => Configure::read('GnuPG.gpgconf'), 'binary' => (Configure::read('GnuPG.binary') ? Configure::read('GnuPG.binary') : '/usr/bin/gpg'), 'debug'));   // , 'debug' => true
-            if (Configure::read('GnuPG.sign')) {
-                $gpg->addSignKey(Configure::read('GnuPG.email'), Configure::read('GnuPG.password'));
-                $body = $gpg->sign($body, Crypt_GPG::SIGN_MODE_CLEAR);
-            }
-        } catch (Exception $e) {
-            $failureReason = " the message could not be signed. The following error message was returned by gpg: " . $e->getMessage();
-            $this->log($e->getMessage());
-            $failed = true;
-        }
-        if (!$failed) {
-            $keyImportOutput = $gpg->importKey($user['User']['gpgkey']);
-            try {
-                $key = $gpg->getKeys($keyImportOutput['fingerprint']);
-                $subKeys = $key[0]->getSubKeys();
-                $canEncryptGPG = false;
-                $currentTimestamp = time();
-                foreach ($subKeys as $subKey) {
-                    $expiration = $subKey->getExpirationDate();
-                    if (($expiration == 0 || $currentTimestamp < $expiration) && $subKey->canEncrypt()) {
-                        $canEncryptGPG = true;
-                    }
-                }
-                if ($canEncryptGPG) {
-                    $gpg->addEncryptKey($keyImportOutput['fingerprint']); // use the key that was given in the import
-                    $body = $gpg->encrypt($body, true);
-                } else {
-                    $failed = true;
-                    $failureReason = " the message could not be encrypted because the provided key is either expired or cannot be used for encryption.";
-                }
-            } catch (Exception $e) {
-                // despite the user having a GnuPG key and the signing already succeeding earlier, we get an exception. This must mean that there is an issue with the user's key.
-                $failureReason = " the message could not be encrypted because there was an issue with the user's GnuPG key. The following error message was returned by gpg: " . $e->getMessage();
-                $this->log($e->getMessage());
-                $failed = true;
-            }
-        }
-        if (!empty($failed)) {
-            return array('failed' => $failed, 'failureReason' => $failureReason);
-        }
+        $logTitle = $encrypted ? 'Encrypted email' : 'Email';
+        // Intentional two spaces to pass test :)
+        $logTitle .= $replyToLog  . '  to ' . $user['User']['email'] . ' sent, titled "' . $subject . '".';
+
+        $this->Log->create();
+        $this->Log->save(array(
+            'org' => 'SYSTEM',
+            'model' => 'User',
+            'model_id' => $user['User']['id'],
+            'email' => $user['User']['email'],
+            'action' => 'email',
+            'title' => $logTitle,
+            'change' => null,
+        ));
         return true;
     }
 
-    private function __encryptUsingSmime(&$Email, &$body, $subject, $user)
-    {
-        try {
-            $prependedBody = 'Content-Transfer-Encoding: 7bit' . PHP_EOL . 'Content-Type: text/plain;' . PHP_EOL . '    charset=us-ascii' . PHP_EOL . PHP_EOL . $body;
-            App::uses('Folder', 'Utility');
-            App::uses('FileAccessTool', 'Tools');
-            $fileAccessTool = new FileAccessTool();
-            $dir = APP . 'tmp' . DS . 'SMIME';
-            if (!file_exists($dir)) {
-                if (!mkdir($dir, 0750, true)) {
-                    throw new MethodNotAllowedException('The SMIME temp directory is not writeable (app/tmp/SMIME).');
-                }
-            }
-            // save message to file
-            $tempFile = $fileAccessTool->createTempFile($dir, 'SMIME');
-            $msg = $fileAccessTool->writeToFile($tempFile, $prependedBody);
-            $headers_smime = array("To" => $user['User']['email'], "From" => Configure::read('MISP.email'), "Subject" => $subject);
-            $canSign = true;
-            if (
-                !empty(Configure::read('SMIME.cert_public_sign')) &&
-                is_readable(Configure::read('SMIME.cert_public_sign')) &&
-                !empty(Configure::read('SMIME.key_sign')) &&
-                is_readable(Configure::read('SMIME.key_sign'))
-            ) {
-                $signed = $fileAccessTool->createTempFile($dir, 'SMIME');
-                if (openssl_pkcs7_sign($msg, $signed, 'file://'.Configure::read('SMIME.cert_public_sign'), array('file://'.Configure::read('SMIME.key_sign'), Configure::read('SMIME.password')), array(), PKCS7_TEXT)) {
-                    $bodySigned = $fileAccessTool->readFromFile($signed);
-                    unlink($msg);
-                    unlink($signed);
-                } else {
-                    unlink($msg);
-                    unlink($signed);
-                    throw new Exception('Failed while attempting to sign the SMIME message.');
-                }
-                // save message to file
-                $tempFile = $fileAccessTool->createTempFile($dir, 'SMIME');
-                $msg_signed = $fileAccessTool->writeToFile($tempFile, $bodySigned);
-            } else {
-                $msg_signed = $msg;
-            }
-            $msg_signed_encrypted = $fileAccessTool->createTempFile($dir, 'SMIME');
-            // encrypt it
-            if (openssl_pkcs7_encrypt($msg_signed, $msg_signed_encrypted, $user['User']['certif_public'], $headers_smime, 0, OPENSSL_CIPHER_AES_256_CBC)) {
-                $bodyEncSig = $fileAccessTool->readFromFile($msg_signed_encrypted);
-                unlink($msg_signed);
-                unlink($msg_signed_encrypted);
-                $parts = explode("\n\n", $bodyEncSig);
-                $bodyEncSig = $parts[1];
-                // SMIME transport (hardcoded headers
-                $Email = $Email->transport('Smime');
-                $body = $bodyEncSig;
-            } else {
-                unlink($msg_signed);
-                unlink($msg_signed_encrypted);
-                throw new Exception('Could not encrypt the SMIME message.');
-            }
-        } catch (Exception $e) {
-            // despite the user having a certificate. This must mean that there is an issue with the user's certificate.
-            $result['failureReason'] = " the message could not be encrypted because there was an issue with the user's public certificate. The following error message was returned by openssl: " . $e->getMessage();
-            $this->log($e->getMessage());
-            $result['failed'] = true;
-        }
-        return $result;
-    }
-
     public function adminMessageResolve($message)
     {
         $resolveVars = array('$contact' => 'MISP.contact', '$org' => 'MISP.org', '$misp' => 'MISP.baseurl');
@@ -1419,33 +1165,6 @@ class User extends AppModel
         $syslog->write('notice', "$description -- $action" . (empty($fieldResult) ? '' : ' -- ' . $result['Log']['change']));
     }
 
-    /**
-     * @return Crypt_GPG
-     * @throws Exception
-     */
-    private function initializeGpg()
-    {
-        if (!class_exists('Crypt_GPG')) {
-            if (!stream_resolve_include_path('Crypt/GPG.php')) {
-                throw new Exception("Crypt_GPG is not installed.");
-            }
-            require_once 'Crypt/GPG.php';
-        }
-
-        $homedir = Configure::read('GnuPG.homedir');
-        if ($homedir === null) {
-            throw new Exception("Configuration option 'GnuPG.homedir' is not set, Crypt_GPG cannot be initialized.");
-        }
-
-        $options = array(
-            'homedir' => $homedir,
-            'gpgconf' => Configure::read('GnuPG.gpgconf'),
-            'binary' => Configure::read('GnuPG.binary') ?: '/usr/bin/gpg',
-        );
-
-        return new Crypt_GPG($options);
-    }
-
     public function getOrgActivity($orgId, $params=array())
     {
         $conditions = array();
@@ -1578,6 +1297,31 @@ class User extends AppModel
             $this->Inbox->delete($registration['id']);
             return true;
         }
+    }
 
+    /**
+     * Initialize GPG. Returns `null` if initialization failed.
+     *
+     * @return null|Crypt_GPG
+     */
+    private function initializeGpg()
+    {
+        if ($this->gpg !== null) {
+            if ($this->gpg === false) { // initialization failed
+                return null;
+            }
+
+            return $this->gpg;
+        }
+
+        try {
+            $gpgTool = new GpgTool();
+            $this->gpg = $gpgTool->initializeGpg();
+            return $this->gpg;
+        } catch (Exception $e) {
+            $this->logException("GPG couldn't be initialized, GPG encryption and signing will be not available.", $e, LOG_NOTICE);
+            $this->gpg = false;
+            return null;
+        }
     }
 }

app/View/Allowedlists/admin_add.ctp

@@ -1,7 +1,7 @@
-<div class="whitelist form">
-<?php echo $this->Form->create('Whitelist');?>
+<div class="allowedlist form">
+<?php echo $this->Form->create('Allowedlist');?>
     <fieldset>
-        <legend><?php echo __('Add Signature Whitelist');?></legend>
+        <legend><?php echo __('Add Signature Allowedlist');?></legend>
     <?php
         echo $this->Form->input('name', array(
             'class' => 'input-xxlarge'
@@ -15,5 +15,5 @@ echo $this->Form->end();
 ?>
 </div>
 <?php
-    echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'whitelist', 'menuItem' => 'add'));
+    echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'allowedlist', 'menuItem' => 'add'));
 ?>

app/View/Allowedlists/admin_edit.ctp

@@ -1,7 +1,7 @@
-<div class="whitelist form">
-<?php echo $this->Form->create('Whitelist');?>
+<div class="allowedlist form">
+<?php echo $this->Form->create('Allowedlist');?>
     <fieldset>
-        <legend><?php echo __('Edit Signature Whitelist');?></legend>
+        <legend><?php echo __('Edit Signature Allowedlist');?></legend>
     <?php
         echo $this->Form->input('id');
         echo $this->Form->input('name', array(
@@ -15,5 +15,5 @@
 ?>
 </div>
 <?php
-    echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'whitelist', 'menuItem' => 'edit', 'id' => $this->Form->value('Whitelist.id')));
+    echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'allowedlist', 'menuItem' => 'edit', 'id' => $this->Form->value('Allowedlist.id')));
 ?>

app/View/Allowedlists/admin_index.ctp

@@ -1,5 +1,5 @@
-<div class="whitelist index">
-    <h2><?php echo __('Signature Whitelist');?></h2>
+<div class="allowedlist index">
+    <h2><?php echo __('Signature Allowedlist');?></h2>
     <p><?php echo __('Regex entries (in the standard php regex /{regex}/{modifier} format) entered below will restrict matching attributes from being included in the IDS flag sensitive exports (such as NIDS exports).');?></p>
     <div class="pagination">
         <ul>
@@ -25,11 +25,11 @@
     </tr><?php
 foreach ($list as $item):?>
     <tr>
-        <td class="short"><?php echo h($item['Whitelist']['id']);?>&nbsp;</td>
-        <td><?php echo h($item['Whitelist']['name']);?>&nbsp;</td>
+        <td class="short"><?php echo h($item['Allowedlist']['id']);?>&nbsp;</td>
+        <td><?php echo h($item['Allowedlist']['name']);?>&nbsp;</td>
         <td class="short action-links">
-            <?php echo $this->Html->link('', array('admin' => true, 'action' => 'edit', $item['Whitelist']['id']), array('class' => 'fa fa-edit', 'title' => __('Edit'), 'aria-label' => __('Edit')));?>
-            <?php echo $this->Form->postLink('', array('admin' => true, 'action' => 'delete', $item['Whitelist']['id']), array('class' => 'fa fa-trash', 'title' => __('Delete'), 'aria-label' => __('Delete')), __('Are you sure you want to delete "%s"?', $item['Whitelist']['name']));?>
+            <?php echo $this->Html->link('', array('admin' => true, 'action' => 'edit', $item['Allowedlist']['id']), array('class' => 'fa fa-edit', 'title' => __('Edit'), 'aria-label' => __('Edit')));?>
+            <?php echo $this->Form->postLink('', array('admin' => true, 'action' => 'delete', $item['Allowedlist']['id']), array('class' => 'fa fa-trash', 'title' => __('Delete'), 'aria-label' => __('Delete')), __('Are you sure you want to delete "%s"?', $item['Allowedlist']['name']));?>
         </td>
     </tr><?php
 endforeach;?>
@@ -52,4 +52,4 @@ endforeach;?>
     </div>
 </div>
 <?php
-    echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'whitelist', 'menuItem' => 'index'));
+    echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'allowedlist', 'menuItem' => 'index'));

app/View/Allowedlists/index.ctp

@@ -1,5 +1,5 @@
-<div class="whitelist index">
-    <h2><?php echo __('Signature Whitelist');?></h2>
+<div class="allowedlist index">
+    <h2><?php echo __('Signature Allowedlist');?></h2>
     <p><?php echo __('Regex entries (in the standard php regex /{regex}/{modifier} format) entered below will restrict matching attributes from being included in the IDS flag sensitive exports (such as NIDS exports).');?></p>
     <div class="pagination">
         <ul>
@@ -24,8 +24,8 @@
     </tr><?php
 foreach ($list as $item):?>
     <tr>
-        <td class="short"><?php echo h($item['Whitelist']['id']);?>&nbsp;</td>
-        <td><?php echo h($item['Whitelist']['name']);?>&nbsp;</td>
+        <td class="short"><?php echo h($item['Allowedlist']['id']);?>&nbsp;</td>
+        <td><?php echo h($item['Allowedlist']['name']);?>&nbsp;</td>
     </tr><?php
 endforeach;?>
     </table>
@@ -47,4 +47,4 @@ endforeach;?>
     </div>
 </div>
 <?php
-    echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'whitelist', 'menuItem' => 'index'));
+    echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'allowedlist', 'menuItem' => 'index'));

app/View/Attributes/add.ctp

@@ -80,7 +80,7 @@
                 'action' => $action,
                 'ajaxSubmit' => sprintf(
                     'submitPopoverForm(%s, %s, 0, 1)',
-                    "'" . ($action == 'add' ? h($event_id) : h($attribute['Attribute']['id'])) . "'",
+                    "'" . ($action === 'add' ? h($event['Event']['id']) : h($attribute['Attribute']['id'])) . "'",
                     "'" . h($action) . "'"
                 )
             ),
@@ -90,14 +90,12 @@
         )
     ));
     if (!$ajax) {
-        $event = ['Event' => ['id' => $event_id, 'published' => $published ]];
         echo $this->element('/genericElements/SideMenu/side_menu', array('menuList' => 'event', 'menuItem' => 'addAttribute', 'event' => $event));
     }
 ?>
 
 <script type="text/javascript">
     var notice_list_triggers = <?php echo $notice_list_triggers; ?>;
-    var composite_types = <?php echo json_encode($compositeTypes); ?>;
     var category_type_mapping = <?php echo json_encode(array_map(function($value) {
         return array_combine($value['types'], $value['types']);
     }, $categoryDefinitions)); ?>;

app/View/Attributes/ajax/attributeEditCategoryForm.ctp

@@ -1,5 +1,5 @@
 <?php
-    echo $this->Form->create('Attribute', array('class' => 'inline-form inline-field-form', 'id' => 'Attribute_' . $object['id'] . '_category_form', 'url' => '/attributes/editField/' . $object['id']));
+    echo $this->Form->create('Attribute', array('class' => 'inline-form inline-field-form', 'id' => 'Attribute_' . $object['id'] . '_category_form', 'url' => $baseurl . '/attributes/editField/' . $object['id']));
 ?>
 <div class='inline-input inline-input-container'>
 <div class="inline-input-accept inline-input-button inline-input-passive"><span class = "icon-ok" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>

app/View/Attributes/ajax/attributeEditCommentForm.ctp

@@ -1,5 +1,5 @@
 <?php
-    echo $this->Form->create('Attribute', array('class' => 'inline-form inline-field-form', 'id' => 'Attribute_' . $object['id'] . '_comment_form', 'url' => '/attributes/editField/' . $object['id']));
+    echo $this->Form->create('Attribute', array('class' => 'inline-form inline-field-form', 'id' => 'Attribute_' . $object['id'] . '_comment_form', 'url' => $baseurl . '/attributes/editField/' . $object['id']));
 ?>
     <div class='inline-input inline-input-container'>
     <div class="inline-input-accept inline-input-button inline-input-passive"><span class = "icon-ok" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>

app/View/Attributes/ajax/attributeEditDistributionForm.ctp

@@ -1,5 +1,5 @@
 <?php
-    echo $this->Form->create('Attribute', array('class' => 'inline-form inline-field-form', 'id' => 'Attribute_' . $object['id'] . '_distribution_form', 'url' => '/attributes/editField/' . $object['id']));
+    echo $this->Form->create('Attribute', array('class' => 'inline-form inline-field-form', 'id' => 'Attribute_' . $object['id'] . '_distribution_form', 'url' => $baseurl . '/attributes/editField/' . $object['id']));
 ?>
 <div class='inline-input inline-input-container'>
     <div class="inline-input-accept inline-input-button inline-input-passive"><span class = "icon-ok" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>

app/View/Attributes/ajax/attributeEditMassForm.ctp

@@ -1,6 +1,6 @@
 <div class="attributes">
 <?php
-    echo $this->Form->create('Attribute', array('url' => '/attributes/editSelected/' . $id));
+    echo $this->Form->create('Attribute', array('url' => $baseurl . '/attributes/editSelected/' . $id));
 ?>
     <fieldset>
         <legend><?php echo __('Mass Edit Attributes'); ?></legend>

app/View/Attributes/ajax/attributeEditTo_idsForm.ctp

@@ -1,6 +1,6 @@
 <div class="confirmation">
     <?php
-    echo $this->Form->create('Attribute', array('style' => 'margin:0px;', 'id' => 'PromptForm', 'url' => '/attributes/editField/' . $object['id']));
+    echo $this->Form->create('Attribute', array('style' => 'margin:0px;', 'id' => 'PromptForm', 'url' => $baseurl . '/attributes/editField/' . $object['id']));
     echo $this->Form->input('to_ids', array(
         'options' => array(0 => 'No', 1 => 'Yes'),
         'label' => false,

app/View/Attributes/ajax/attributeEditTypeForm.ctp

@@ -1,5 +1,5 @@
 <?php
-    echo $this->Form->create('Attribute', array('class' => 'inline-form inline-field-form', 'id' => 'Attribute_' . $object['id'] . '_type_form', 'url' => '/attributes/editField/' . $object['id']));
+    echo $this->Form->create('Attribute', array('class' => 'inline-form inline-field-form', 'id' => 'Attribute_' . $object['id'] . '_type_form', 'url' => $baseurl . '/attributes/editField/' . $object['id']));
 ?>
 <div class='inline-input inline-input-container'>
     <div class="inline-input-accept inline-input-button inline-input-passive"><span class = "icon-ok" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>

app/View/Attributes/ajax/attributeEditValueForm.ctp

@@ -1,6 +1,6 @@
 
 <?php
-    echo $this->Form->create('Attribute', array('class' => 'inline-form inline-field-form', 'url' => '/attributes/editField/' . $object['id'], 'id' => 'Attribute_' . $object['id'] . '_value_form', 'default' => false));
+    echo $this->Form->create('Attribute', array('class' => 'inline-form inline-field-form', 'url' => $baseurl . '/attributes/editField/' . $object['id'], 'id' => 'Attribute_' . $object['id'] . '_value_form', 'default' => false));
 ?>
     <div class='inline-input inline-input-container'>
     <div class="inline-input-accept inline-input-button inline-input-passive"><span class = "icon-ok" role="button" tabindex="0" aria-label="<?php echo __('Accept change'); ?>"></span></div>

app/View/Attributes/ajax/tagRemoveConfirmation.ctp

@@ -1,11 +1,16 @@
 <div class="confirmation">
     <?php
-    echo $this->Form->create($model, array('style' => 'margin:0px;', 'id' => 'PromptForm', 'url' => '/' . strtolower($model) . 's/removeTag/' . $id . '/' . $tag_id));
+    echo $this->Form->create($model, array('style' => 'margin:0px;', 'id' => 'PromptForm', 'url' => $baseurl . '/' . strtolower($model) . 's/removeTag/' . $id . '/' . $tag_id));
     $action = "removeObjectTag('" . $model . "', '" . h($id) . "', '" . h($tag_id) . "');";
     ?>
-    <legend><?php echo __('Remove Tag'); ?></legend>
     <div style="padding-left:5px;padding-right:5px;padding-bottom:5px;">
-    <p><?php echo __('Remove tag '); ?> (<?php echo h($tag_id); ?>) <?php echo __('from '); ?> <?php echo ucfirst(h($model)); ?> (<?php echo h($id); ?>)?</p>
+    <p><?= __('Remove %s tag %s from %s %s?',
+            isset($is_local) ? ($is_local ? __('local') : __('global')) : '',
+            $this->element('tag', ['tag' => $tag]),
+            str_replace('_', ' ', strtolower($model)),
+            h($model_name))
+        ?>
+    </p>
         <table>
             <tr>
                 <td style="vertical-align:top">

app/View/Attributes/ajax/toggle_correlation.ctp

@@ -1,6 +1,6 @@
 <div class="confirmation">
     <?php
-    echo $this->Form->create('Attribute', array('style' => 'margin:0px;', 'id' => 'PromptForm', 'url' => '/attributes/toggleCorrelation/' . $attribute['Attribute']['id']));
+    echo $this->Form->create('Attribute', array('style' => 'margin:0px;', 'id' => 'PromptForm', 'url' => $baseurl . '/attributes/toggleCorrelation/' . $attribute['Attribute']['id']));
     $extraTitle = "";
     ?>
     <legend><?php echo __('Toggle Correlation %s ', $attribute['Attribute']['disable_correlation'] ? __('on') : __('off'));?></legend>

app/View/Attributes/attribute_replace.ctp

@@ -1,6 +1,6 @@
 <div class="attribute_replace">
 <?php
-echo $this->Form->create('Attribute', array('id', 'url' => '/attributes/attributeReplace/' . $event_id));
+echo $this->Form->create('Attribute', array('id', 'url' => $baseurl . '/attributes/attributeReplace/' . $event_id));
 ?>
     <fieldset>
         <legend><?php echo __('Attribute Replace Tool'); ?></legend>

app/View/Attributes/search.ctp

@@ -26,7 +26,7 @@
         <?php
             echo $this->Form->input('to_ids', array(
                 'type' => 'checkbox',
-                'label' => __('Only find IOCs flagged as to_ids')
+                'label' => __('Only find IOCs flagged as to IDS')
             ));
             echo $this->Form->input('alternate', array(
                     'type' => 'checkbox',
@@ -44,7 +44,7 @@
             ));
         ?>
         <div class="clear">
-        <h3><?php echo __('First seen and Last seen.'); ?></h3>
+        <h3><?php echo __('First seen and Last seen'); ?></h3>
         <p><?php echo __('Attributes not having first seen or last seen set might not appear in the search'); ?></p>
     </div>
     </fieldset>
@@ -54,44 +54,9 @@
 </div>
 <?php echo $this->element('form_seen_input'); ?>
 <script type="text/javascript">
-//
-// Generate Category / Type filtering array
-//
-var category_type_mapping = new Array();
-
-<?php
-// all types for Category ALL
-echo "category_type_mapping['ALL'] = {";
-$first = true;
-foreach ($typeDefinitions as $type => $def) {
-        if ($first) $first = false;
-        else echo ', ';
-        echo "'" . addslashes($type) . "' : '" . addslashes($type) . "'";
-}
-echo "}; \n";
-
-// all types for empty Category
-echo "category_type_mapping[''] = {";
-$first = true;
-foreach ($typeDefinitions as $type => $def) {
-        if ($first) $first = false;
-        else echo ', ';
-        echo "'" . addslashes($type) . "' : '" . addslashes($type) . "'";
-}
-echo "}; \n";
-
-// Types per Category
-foreach ($categoryDefinitions as $category => $def) {
-    echo "category_type_mapping['" . addslashes($category) . "'] = {";
-    $first = true;
-    foreach ($def['types'] as $type) {
-        if ($first) $first = false;
-        else echo ', ';
-        echo "'" . addslashes($type) . "' : '" . addslashes($type) . "'";
-    }
-    echo "}; \n";
-}
-?>
+var category_type_mapping = <?= json_encode(array_map(function($value) {
+    return array_combine($value['types'], $value['types']);
+}, $categoryDefinitions)); ?>;
 
 //
 // Generate Type / Category filtering array
@@ -100,50 +65,28 @@ var type_category_mapping = new Array();
 
 <?php
 // all categories for Type ALL
-echo "type_category_mapping['ALL'] = {";
-$first = true;
+echo "type_category_mapping['ALL'] = {'ALL': 'ALL'";
 foreach ($categoryDefinitions as $type => $def) {
-        if ($first) $first = false;
-        else echo ', ';
-        echo "'" . addslashes($type) . "' : '" . addslashes($type) . "'";
+        echo ", '" . addslashes($type) . "': '" . addslashes($type) . "'";
 }
 echo "}; \n";
 
 // Categories per Type
 foreach ($typeDefinitions as $type => $def) {
-    echo "type_category_mapping['" . addslashes($type) . "'] = {";
-    $first = true;
+    echo "type_category_mapping['" . addslashes($type) . "'] = {'ALL': 'ALL'";
     foreach ($categoryDefinitions as $category => $def) {
         if ( in_array ( $type , $def['types'])) {
-            if ($first) $first = false;
-            else echo ', ';
-            echo "'" . addslashes($category) . "' : '" . addslashes($category) . "'";
+            echo ", '" . addslashes($category) . "': '" . addslashes($category) . "'";
         }
     }
     echo "}; \n";
 }
 ?>
 
-function formCategoryChanged(id) {
-    var alreadySelected = $('#AttributeType').val();
-    // empty the types
-    document.getElementById("AttributeType").options.length = 1;
-    // add new items to options
-    var options = $('#AttributeType').prop('options');
-    $.each(category_type_mapping[$('#AttributeCategory').val()], function(val, text) {
-        options[options.length] = new Option(text, val);
-        if (val == alreadySelected) {
-            options[options.length-1].selected = true;
-        }
-    });
-    // enable the form element
-    $('#AttributeType').prop('disabled', false);
-}
-
-function formTypeChanged(id) {
+function formTypeChanged() {
     var alreadySelected = $('#AttributeCategory').val();
     // empty the categories
-    document.getElementById("AttributeCategory").options.length = 2;
+    $('option', $('#AttributeCategory')).remove();
     // add new items to options
     var options = $('#AttributeCategory').prop('options');
     $.each(type_category_mapping[$('#AttributeType').val()], function(val, text) {
@@ -167,15 +110,14 @@ foreach ($categoryDefinitions as $category => $def) {
     $info = isset($def['formdesc']) ? $def['formdesc'] : $def['desc'];
     echo "formInfoValues['$category'] = \"$info\";\n";
 }
-$this->Js->get('#AttributeCategory')->event('change', 'formCategoryChanged("#AttributeCategory")');
-$this->Js->get('#AttributeType')->event('change', 'formTypeChanged("#AttributeType")');
+$this->Js->get('#AttributeCategory')->event('change', 'formCategoryChanged("Attribute")');
+$this->Js->get('#AttributeType')->event('change', 'formTypeChanged()');
 ?>
 
-formInfoValues['ALL'] = '';
 formInfoValues[''] = '';
 
 
-$(document).ready(function() {
+$(function() {
 
     $("#AttributeType, #AttributeCategory").on('mouseleave', function(e) {
         $('#'+e.currentTarget.id).popover('destroy');

app/View/Communities/index.ctp

@@ -72,14 +72,14 @@
             'description' => __('You can find a list of communities below that chose to advertise their existence to the general MISP user-base. Requesting access to any of those communities is of course no guarantee of being permitted access, it is only meant to simplify the means of finding the various communities that one may be eligible for. Get in touch with the MISP project maintainers if you would like your community to be included in the list.'),
             'actions' => array(
                 array(
-                    'url' => '/communities/view',
+                    'url' => $baseurl . '/communities/view',
                     'url_params_data_paths' => array(
                         'uuid'
                     ),
                     'icon' => 'eye'
                 ),
                 array(
-                    'url' => '/communities/requestAccess',
+                    'url' => $baseurl . '/communities/requestAccess',
                     'url_params_data_paths' => array(
                         'uuid'
                     ),

app/View/Elements/Events/View/eventFilteringQueryBuilder.ctp

@@ -503,7 +503,7 @@ function cleanRules(rules) {
 function performQuery(rules) {
     var res = cleanRules(rules);
 
-    var url = "/events/viewEventAttributes/<?php echo h($event['Event']['id']); ?>";
+    var url = "<?php echo $baseurl; ?>/events/viewEventAttributes/<?php echo h($event['Event']['id']); ?>";
     $.ajax({
         type:"post",
         url: url,

app/View/Elements/Events/View/row_attribute.ctp

@@ -338,7 +338,7 @@
             <?php
                 if ($object['distribution'] == 4):
             ?>
-                <a href="/sharing_groups/view/<?php echo h($object['sharing_group_id']); ?>"><?php echo h($object['SharingGroup']['name']);?></a>
+                <a href="<?php echo $baseurl;?>/sharing_groups/view/<?php echo h($object['sharing_group_id']); ?>"><?php echo h($object['SharingGroup']['name']);?></a>
             <?php
                 else:
                     echo h($shortDist[$object['distribution']]);

app/View/Elements/Events/View/row_object.ctp

@@ -107,7 +107,7 @@
         if ($object['objectType'] == 0) {
           if ($object['distribution'] == 4):
       ?>
-          <a href="/sharing_groups/view/<?php echo h($object['sharing_group_id']); ?>"><?php echo h($object['SharingGroup']['name']);?></a>
+        <a href="<?php echo $baseurl; ?>/sharing_groups/view/<?php echo h($object['sharing_group_id']); ?>"><?php echo h($object['SharingGroup']['name']);?></a>
       <?php
           else:
             echo h($shortDist[$object['distribution']]);

app/View/Elements/Events/View/row_object_reference.ctp

@@ -25,7 +25,7 @@
 <?php
   if ($mayModify):
 ?>
-    <span class="fa fa-plus-square useCursorPointer" title="<?php echo __('Add reference');?>" role="button" tabindex="0" aria-label="<?php echo __('Add reference');?>" onClick="genericPopup('<?php echo '/objectReferences/add/' . h($object['id']);?>', '#popover_form');"></span>
+    <span class="fa fa-plus-square useCursorPointer" title="<?php echo __('Add reference');?>" role="button" tabindex="0" aria-label="<?php echo __('Add reference');?>" onClick="genericPopup('<?php echo $baseurl . '/objectReferences/add/' . h($object['id']);?>', '#popover_form');"></span>
 <?php
   endif;
 ?>

app/View/Elements/Events/View/row_proposal.ctp

@@ -188,7 +188,7 @@
   <td class="short action-links">
     <?php
         if (($event['Orgc']['id'] == $me['org_id'] && $mayModify) || $isSiteAdmin) {
-          echo $this->Form->create('Shadow_Attribute', array('id' => 'ShadowAttribute_' . $object['id'] . '_accept', 'url' => '/shadow_attributes/accept/' . $object['id'], 'style' => 'display:none;'));
+          echo $this->Form->create('Shadow_Attribute', array('id' => 'ShadowAttribute_' . $object['id'] . '_accept', 'url' => $baseurl . '/shadow_attributes/accept/' . $object['id'], 'style' => 'display:none;'));
           echo $this->Form->end();
         ?>
           <span class="icon-ok icon-white useCursorPointer" title="<?php echo __('Accept Proposal');?>" role="button" tabindex="0" aria-label="<?php echo __('Accept proposal');?>" onClick="acceptObject('shadow_attributes', '<?php echo $object['id']; ?>', '<?php echo $event['Event']['id']; ?>');"></span>

app/View/Elements/Events/View/row_proposal_delete.ctp

@@ -74,7 +74,7 @@
   <td class="short action-links">
     <?php
         if (($event['Orgc']['id'] == $me['org_id'] && $mayModify) || $isSiteAdmin) {
-          echo $this->Form->create('Shadow_Attribute', array('id' => 'ShadowAttribute_' . $object['id'] . '_accept', 'url' => '/shadow_attributes/accept/' . $object['id'], 'style' => 'display:none;'));
+          echo $this->Form->create('Shadow_Attribute', array('id' => 'ShadowAttribute_' . $object['id'] . '_accept', 'url' => $baseurl . '/shadow_attributes/accept/' . $object['id'], 'style' => 'display:none;'));
           echo $this->Form->end();
         ?>
           <span class="icon-ok icon-white useCursorPointer" title="<?php echo __('Accept Proposal');?>" role="button" tabindex="0" aria-label="<?php echo __('Accept proposal');?>" onClick="acceptObject('shadow_attributes', '<?php echo $object['id']; ?>', '<?php echo $event['Event']['id']; ?>');"></span>

app/View/Elements/Events/View/sighting_field.ctp

@@ -1,7 +1,7 @@
 <td class="shortish">
   <span id="sightingForm_<?php echo h($object['id']);?>">
   <?php
-      echo $this->Form->create('Sighting', array('id' => 'Sighting_' . $object['id'], 'url' => '/sightings/add/' . $object['id'], 'style' => 'display:none;'));
+      echo $this->Form->create('Sighting', array('id' => 'Sighting_' . $object['id'], 'url' => $baseurl . '/sightings/add/' . $object['id'], 'style' => 'display:none;'));
       echo $this->Form->input('type', array('label' => false, 'id' => 'Sighting_' . $object['id'] . '_type'));
       echo $this->Form->end();
   ?>

app/View/Elements/Feeds/eventattribute.ctp

@@ -102,7 +102,7 @@
         </ul>
     </div>
 <script type="text/javascript">
-    var currentUri = "<?php echo isset($currentUri) ? h($currentUri) : '/feeds/previewEvent/' . h($feed['Feed']['id']) . '/' . h($event['Event']['uuid']); ?>";
+    var currentUri = "<?php echo isset($currentUri) ? h($currentUri) : $baseurl . '/feeds/previewEvent/' . h($feed['Feed']['id']) . '/' . h($event['Event']['uuid']); ?>";
     var lastSelected = false;
     var deleted = <?php echo (isset($deleted) && $deleted) ? 'true' : 'false';?>;
     $(document).ready(function() {

app/View/Elements/Servers/eventattribute.ctp

@@ -102,7 +102,7 @@
         </ul>
     </div>
 <script type="text/javascript">
-    var currentUri = "<?php echo isset($currentUri) ? h($currentUri) : '/servers/previewEvent/' . h($server['Server']['id']) . '/' . h($event['Event']['id']); ?>";
+    var currentUri = "<?php echo isset($currentUri) ? h($currentUri) : $baseurl . '/servers/previewEvent/' . h($server['Server']['id']) . '/' . h($event['Event']['id']); ?>";
     var lastSelected = false;
     var deleted = <?php echo (isset($deleted) && $deleted) ? 'true' : 'false';?>;
     $(document).ready(function() {

app/View/Elements/Users/userIndexTable.ctp

@@ -32,7 +32,7 @@
                     <?php echo h($user['User']['id']); ?>&nbsp;
                 </td>
                 <td class="short" ondblclick="document.location ='<?php echo $this->Html->url(array('admin' => true, 'action' => 'view', $user['User']['id']), true);?>';">
-                    <a href="/organisations/view/<?php echo $user['Organisation']['id'];?>"><?php echo h($user['Organisation']['name']); ?>&nbsp;</a>
+                    <a href="<?php echo $baseurl; ?>/organisations/view/<?php echo $user['Organisation']['id'];?>"><?php echo h($user['Organisation']['name']); ?>&nbsp;</a>
                 </td>
                 <td class="short" ondblclick="document.location ='<?php echo $this->Html->url(array('admin' => true, 'action' => 'view', $user['User']['id']), true);?>';">
                     <?php echo $this->Html->link($user['Role']['name'], array('controller' => 'roles', 'action' => 'view', $user['Role']['id'])); ?>

app/View/Elements/dashboard/Widgets/Achievements.ctp

@@ -0,0 +1,60 @@
+<div>
+<?php
+/*
+* This layout is split in two parts, the top part is the achiements already
+* unlocked and the bottom part contains the one to get next.
+*
+* The data array to be passed has therefore 2 root keys: locked and unlocked.
+* Each one is a list of item, each item must contain:
+*   - icon (url to img file)
+    - title (text description)
+    - help_page (optional, link to an article)
+*
+* So the structure of the $data parameter must be something like:
+* { 'locked': [{ 'icon': '/path/to/img.png', 'title': 'my great achievement', 'help_page': 'http://wikimedia'}, {...}], 'unlocked': [{...}]}
+*/
+    echo '<h3>'.__("Achievements Unlocked!").'</h3>';
+    if(empty($data['unlocked'])) {
+        echo '<p>'.__("You don't have any achievement yet. Check them below to get started!").'</p>';
+    } else {
+        echo '<table class="table table-striped table-hover table-condensed">';
+        foreach ($data['unlocked'] as $item) {
+            echo '<tr>';
+            echo '<td><img class="badge-img" width=48 src='.h($item['icon']).' alt="badge"/></td>';
+            echo '<td><div class="badge-description">'.h($item['title']).'</div></td>';
+            echo '</tr>';
+        }
+        echo '</table>';
+    }
+    echo '<h3>'.__("Next on your list:").'</h3>';
+    if(empty($data['locked'])) {
+        echo '<p>'.__("Well done! You got them all.").'</p>';
+    } else {
+        echo '<table class="table table-striped table-hover table-condensed">';
+        foreach ($data['locked'] as $item) {
+            echo '<tr>';
+            echo '<td><img class="badge-img" width=48 src='.h($item['icon']).' alt="badge"/></td>';
+            echo '<td><div class="badge-description">'.h($item['title']).'</div></td>';
+            if(!empty($item['help_page'])) {
+                echo '<td><a href='.h($item['help_page']).' target="_blank">'.__("Read more here").'</a></td>';
+            }
+            echo '</tr>';
+        }
+        echo '</table>';
+    }
+?>
+</div>
+
+<style widget-scoped>
+.badge-description {
+    font-size: 11pt;
+    padding: 5px;
+    height: 43px;
+    width: 100%;
+}
+.badge-img {
+    padding-top: 5px;
+    width: 48px;
+    max-width: 48px;
+}
+</style>

app/View/Elements/eventattribute.ctp

@@ -1,5 +1,5 @@
 <?php
-    $urlHere = $this->here;
+    $urlHere = Router::url(null);
     $urlHere = explode('/', $urlHere);
     foreach ($urlHere as $k => $v) {
         $urlHere[$k] = urlencode($v);
@@ -74,7 +74,7 @@
 <br />
 <div id="edit_object_div">
     <?php
-        $deleteSelectedUrl = '/attributes/deleteSelected/' . $event['Event']['id'];
+        $deleteSelectedUrl = $baseurl . '/attributes/deleteSelected/' . $event['Event']['id'];
         if (empty($event['Event']['publish_timestamp'])) {
             $deleteSelectedUrl .= '/1';
         }
@@ -88,7 +88,7 @@
         echo $this->Form->end();
     ?>
         <?php
-        echo $this->Form->create('ShadowAttribute', array('id' => 'accept_selected', 'url' => '/shadow_attributes/acceptSelected/' . $event['Event']['id']));
+        echo $this->Form->create('ShadowAttribute', array('id' => 'accept_selected', 'url' => $baseurl . '/shadow_attributes/acceptSelected/' . $event['Event']['id']));
         echo $this->Form->input('ids_accept', array(
             'type' => 'text',
             'value' => '',
@@ -98,7 +98,7 @@
         echo $this->Form->end();
     ?>
         <?php
-        echo $this->Form->create('ShadowAttribute', array('id' => 'discard_selected', 'url' => '/shadow_attributes/discardSelected/' . $event['Event']['id']));
+        echo $this->Form->create('ShadowAttribute', array('id' => 'discard_selected', 'url' => $baseurl . '/shadow_attributes/discardSelected/' . $event['Event']['id']));
         echo $this->Form->input('ids_discard', array(
             'type' => 'text',
             'value' => '',
@@ -259,7 +259,7 @@ attributes or the appropriate distribution level. If you think there is a mistak
         </ul>
     </div>
 <script type="text/javascript">
-    var currentUri = "<?php echo isset($currentUri) ? h($currentUri) : '/events/viewEventAttributes/' . h($event['Event']['id']); ?>";
+    var currentUri = "<?php echo isset($currentUri) ? h($currentUri) : $baseurl . '/events/viewEventAttributes/' . h($event['Event']['id']); ?>";
     var currentPopover = "";
     var ajaxResults = {"hover": [], "persistent": []};
     var timer;

app/View/Elements/eventattributecreation.ctp

@@ -78,7 +78,7 @@
                                     )
                             ),
                             'class'=>'btn btn-primary',
-                            'url' => '/attributes/add/' . $event['Event']['id']
+                            'url' => $baseurl . '/attributes/add/' . $event['Event']['id']
                     ));
                     ?>
                 </td>

app/View/Elements/eventdiscussion.ctp

@@ -132,9 +132,8 @@
     <?php endif; ?>
     <div class="comment">
     <?php
-        if (isset($currentEvent)) $url = '/posts/add/event/' . $currentEvent;
-        else $url = '/posts/add/thread/' . $thread['Thread']['id'];
-        $url = h($url);
+        if (isset($currentEvent)) $url = $baseurl . '/posts/add/event/' . $currentEvent;
+        else $url = $baseurl . '/posts/add/thread/' . $thread['Thread']['id'];
         echo $this->Form->create('Post', array('url' => $url));
     ?>
         <fieldset>

app/View/Elements/galaxyQuickViewMini.ctp

@@ -122,7 +122,7 @@
         }
         if (
             (!isset($local_tag_off) || !$local_tag_off) &&
-            ($isSiteAdmin || ($isAclTagger && Configure::read('MISP.host_org_id') == $me['org_id']))
+            ($isSiteAdmin || ($mayModify && $isAclTagger) || ($isAclTagger && Configure::read('MISP.host_org_id') == $me['org_id']))
         ) {
             echo sprintf(
                 '<button class="%s" data-target-type="%s" data-target-id="%s" data-local="true" role="button" tabindex="0" aria-label="' . __('Add new local cluster') . '" title="' . __('Add a local tag') . '" style="%s">%s</button>',

app/View/Elements/genericElements/Form/genericForm.ctp

@@ -16,7 +16,7 @@
         h(Inflector::singularize(Inflector::classify($this->request->params['controller']))) :
         h($data['model']);
     $fieldsString = '';
-    $simpleFieldWhitelist = array(
+    $simpleFieldAllowedlist = array(
         'default', 'type', 'options', 'placeholder', 'label', 'empty', 'rows', 'div', 'required'
     );
     $fieldsArrayForPersistence = array();
@@ -50,7 +50,7 @@
                 } else {
                     $params['class'] = '';
                 }
-                foreach ($simpleFieldWhitelist as $f) {
+                foreach ($simpleFieldAllowedlist as $f) {
                     if (!empty($fieldData[$f])) {
                         $params[$f] = $fieldData[$f];
                     }

app/View/Elements/genericElements/IndexTable/Fields/actions.ctp

@@ -68,7 +68,7 @@
                 array(
                     'class' => $this->FontAwesome->getClass($action['icon']) . ' black ' . (empty($action['class']) ? '' : h($action['class'])),
                     'title' => empty($action['title']) ? '' : h($action['title']),
-                    'aria-label' => empty($action['aria-label']) ? '' : h($action['aria-label']),
+                    'aria-label' => empty($action['title']) ? '' : h($action['title']),
                 ),
                 empty($action['postLinkConfirm'])? '' : $action['postLinkConfirm']
             );