From 2aa4ee30978b06bc479c4e0db5ebaf662346d1c8 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Fri, 28 Oct 2022 13:15:12 +0200
Subject: [PATCH 01/11] chg: [internal] Optimise sighting rest search

---
 app/Model/Sighting.php | 164 ++++++++++++++++++++++++++++-------------
 1 file changed, 114 insertions(+), 50 deletions(-)

diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php
index fbd09b2ba..1e3e8c44c 100644
--- a/app/Model/Sighting.php
+++ b/app/Model/Sighting.php
@@ -80,13 +80,7 @@ class Sighting extends AppModel
         $pubToZmq = $this->pubToZmq('sighting');
         $kafkaTopic = $this->kafkaTopic('sighting');
         if ($pubToZmq || $kafkaTopic) {
-            $user = array(
-                'org_id' => -1,
-                'Role' => array(
-                    'perm_site_admin' => 1
-                )
-            );
-            $sighting = $this->getSighting($this->id, $user);
+            $sighting = $this->getSighting($this->id);
             if ($pubToZmq) {
                 $pubSubTool = $this->getPubSubTool();
                 $pubSubTool->sighting_save($sighting, 'add');
@@ -104,13 +98,7 @@ class Sighting extends AppModel
         $pubToZmq = $this->pubToZmq('sighting');
         $kafkaTopic = $this->kafkaTopic('sighting');
         if ($pubToZmq || $kafkaTopic) {
-            $user = array(
-                'org_id' => -1,
-                'Role' => array(
-                    'perm_site_admin' => 1
-                )
-            );
-            $sighting = $this->getSighting($this->id, $user);
+            $sighting = $this->getSighting($this->id);
             if ($pubToZmq) {
                 $pubSubTool = $this->getPubSubTool();
                 $pubSubTool->sighting_save($sighting, 'delete');
@@ -187,12 +175,114 @@ class Sighting extends AppModel
     }
 
     /**
-     * @param int $id
+     * Fetch sightings with proper ACL checks
+     *
      * @param array $user
+     * @param array $ids
      * @param bool $withEvent
      * @return array
      */
-    public function getSighting($id, array $user, $withEvent = true)
+    private function getSightings(array $user, array $ids, $withEvent = true)
+    {
+        // Fetch all attribute IDs that are connected to sightings
+        $attributesIds = $this->find('column', [
+           'fields' => ['Sighting.attribute_id'],
+           'conditions' => ['Sighting.id' => $ids],
+           'unique' => true,
+        ]);
+        if (empty($attributesIds)) {
+            return [];
+        }
+
+        // Fetch all attributes that are connected to sightings and user can see them
+        $attributeConditions = $this->Attribute->buildConditions($user);
+        $attributeConditions['Attribute.id'] = $attributesIds;
+        $attributes = $this->Attribute->find('all', [
+            'recursive' => -1,
+            'conditions' => $attributeConditions,
+            'contain' => [
+                'Event' => [
+                    'fields' => $withEvent ? ['Event.id', 'Event.uuid', 'Event.orgc_id', 'Event.org_id', 'Event.info'] : ['Event.org_id']
+                ],
+                'Object',
+            ],
+            'fields' => ['Attribute.id', 'Attribute.value','Attribute.uuid', 'Attribute.type', 'Attribute.category', 'Attribute.to_ids', 'Event.org_id'],
+            'order' => [],
+        ]);
+
+        if (empty($attributes)) {
+            return [];
+        }
+
+        // Create conditions for fetching just sightings user can see according to sightings policy
+        $sightingsPolicy = $this->sightingsPolicy();
+        $attributesById = [];
+        $conditions = [];
+        foreach ($attributes as $attribute) {
+            $attributesById[$attribute['Attribute']['id']] = $attribute;
+
+            $attributeConditions = ['Sighting.attribute_id' => $attribute['Attribute']['id']];
+            $ownEvent = $user['Role']['perm_site_admin'] || $attribute['Event']['org_id'] == $user['org_id'];
+            if (!$ownEvent) {
+                if ($sightingsPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
+                    $attributeConditions['Sighting.org_id'] = $user['org_id'];
+                } else if ($sightingsPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) {
+                    if (!$this->isReporter($attribute['Event']['id'], $user['org_id'])) {
+                        continue; // skip attribute
+                    }
+                } else if ($sightingsPolicy === self::SIGHTING_POLICY_HOST_ORG) {
+                    $attributeConditions['Sighting.org_id'] = [$user['org_id'], Configure::read('MISP.host_org_id')];
+                }
+            }
+            $conditions['OR'][] = $attributeConditions;
+        }
+
+        if (empty($conditions)) {
+            return [];
+        }
+
+        $sightings = $this->find('all', [
+            'recursive' => -1,
+            'conditions' => $conditions,
+            'order' => 'Sighting.id',
+        ]);
+
+        if (empty($sightings)) {
+            return [];
+        }
+
+        $anonymise = Configure::read('Plugin.Sightings_anonymise');
+        $results = [];
+        foreach ($sightings as $sighting) {
+            $sightingAttribute = $attributesById[$sighting['Sighting']['attribute_id']];
+
+            if ($anonymise && $sighting['Sighting']['org_id'] != $user['org_id']) {
+                unset($sighting['Sighting']['org_id']);
+            }
+
+            // rearrange it to match the event format of fetchevent
+            $result = [
+                'Sighting' => $sighting['Sighting'],
+            ];
+            $result['Sighting']['Attribute'] = $sightingAttribute['Attribute'];
+            if ($withEvent) {
+                $sightingAttribute['Event']['Orgc']['name'] = $this->getOrganisationById($sightingAttribute['Event']['orgc_id'])['name'];
+                $result['Sighting']['Event'] = $sightingAttribute['Event'];
+            }
+            $results[] = $result;
+        }
+
+        return $results;
+    }
+
+    /**
+     * Return sighting without ACL checks
+     *
+     * @param int $id
+     * @param bool $withEvent
+     * @return array
+     */
+    public function getSighting($id, $withEvent = true)
     {
         $sighting = $this->find('first', array(
             'recursive' => -1,
@@ -210,37 +300,11 @@ class Sighting extends AppModel
             return array();
         }
 
-        $ownEvent = $user['Role']['perm_site_admin'] || $sighting['Event']['org_id'] == $user['org_id'];
-        if (!$ownEvent) {
-            $sightingPolicy = $this->sightingsPolicy();
-            // if sighting policy == 0 then return false if the sighting doesn't belong to the user
-            if ($sightingPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
-                if ($sighting['Sighting']['org_id'] != $user['org_id']) {
-                    return array();
-                }
-            }
-            // if sighting policy == 1, the user can only see the sighting if they've sighted something in the event once
-            else if ($sightingPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) {
-                if (!$this->isReporter($sighting['Sighting']['event_id'], $user['org_id'])) {
-                    return array();
-                }
-            }
-            else if ($sightingPolicy === self::SIGHTING_POLICY_HOST_ORG) {
-                if ($sighting['Sighting']['org_id'] != $user['org_id'] || $sighting['Sighting']['org_id'] != Configure::read('MISP.host_org_id')) {
-                    return array();
-                }
-            }
-        }
-
         // Put event organisation name from cache
         if ($withEvent) {
             $sighting['Event']['Orgc']['name'] = $this->getOrganisationById($sighting['Event']['orgc_id'])['name'];
         }
 
-        $anonymise = Configure::read('Plugin.Sightings_anonymise');
-        if ($anonymise && $sighting['Sighting']['org_id'] != $user['org_id']) {
-            unset($sighting['Sighting']['org_id']);
-        }
         // rearrange it to match the event format of fetchevent
         $result = array(
             'Sighting' => $sighting['Sighting']
@@ -935,7 +999,6 @@ class Sighting extends AppModel
         }
 
         if (isset($filters['org_id'])) {
-            $this->Organisation = ClassRegistry::init('Organisation');
             if (!is_array($filters['org_id'])) {
                 $filters['org_id'] = array($filters['org_id']);
             }
@@ -978,11 +1041,12 @@ class Sighting extends AppModel
             }
         }
 
-        // fetch sightings matching the query
+        // fetch sightings matching the query without ACL checks
         $sightingIds = $this->find('column', [
             'conditions' => $conditions,
             'fields' => ['Sighting.id'],
             'contain' => $contain,
+            'order' => 'Sighting.id',
         ]);
 
         $includeAttribute = isset($filters['includeAttribute']) && $filters['includeAttribute'];
@@ -1008,10 +1072,10 @@ class Sighting extends AppModel
         $tmpfile->write($exportTool->header($exportToolParams));
         $separator = $exportTool->separator($exportToolParams);
 
-        foreach ($sightingIds as $sightingId) {
-            // apply ACL and sighting policies
-            $sighting = $this->getSighting($sightingId, $user, $includeEvent);
-            if (!empty($sighting)) {
+        foreach (array_chunk($sightingIds, 500) as $chunk) {
+            // fetch sightings with ACL checks and sighting policies
+            $sightings = $this->getSightings($user, $chunk, $includeEvent);
+            foreach ($sightings as $sighting) {
                 $sighting['Sighting']['value'] = $sighting['Sighting']['Attribute']['value'];
                 if (!$includeAttribute) {
                     unset($sighting['Sighting']['Attribute']);
@@ -1108,9 +1172,9 @@ class Sighting extends AppModel
             $existingUuids = array_column($toSave, 'uuid');
             $this->Event->publishSightingsRouter($event['Event']['id'], $user, $passAlong, $existingUuids);
             return count($toSave);
-        } else {
-            return 0;
         }
+
+        return 0;
     }
 
     /**

From 7a29e18d233a105608fe8cf8ca67f236b12b617f Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Fri, 28 Oct 2022 14:41:50 +0200
Subject: [PATCH 02/11] chg: [sync] New way how to pull sightings

---
 app/Lib/Tools/ServerSyncTool.php |  18 +++++
 app/Model/Server.php             |   8 +++
 app/Model/Sighting.php           | 120 ++++++++++++++++++++++++++++---
 3 files changed, 135 insertions(+), 11 deletions(-)

diff --git a/app/Lib/Tools/ServerSyncTool.php b/app/Lib/Tools/ServerSyncTool.php
index eead9175a..e5f92030a 100644
--- a/app/Lib/Tools/ServerSyncTool.php
+++ b/app/Lib/Tools/ServerSyncTool.php
@@ -227,6 +227,24 @@ class ServerSyncTool
         return $this->get($url);
     }
 
+    /**
+     * @param array $eventUuids
+     * @return array
+     * @throws HttpSocketHttpException
+     * @throws HttpSocketJsonException
+     * @throws JsonException
+     */
+    public function fetchSightingsForEvents(array $eventUuids)
+    {
+        return $this->post('/sightings/restSearch/event', [
+            'returnFormat' => 'json',
+            'last' => 0, // fetch all
+            'includeAttribute' => true,
+            'includeEvent' => true,
+            'uuid' => $eventUuids,
+        ])->json()['response'];
+    }
+
     /**
      * @param array $event
      * @param array $sightingUuids
diff --git a/app/Model/Server.php b/app/Model/Server.php
index f70250996..06f91780c 100644
--- a/app/Model/Server.php
+++ b/app/Model/Server.php
@@ -247,6 +247,14 @@ class Server extends AppModel
         if (!empty($server['authkey']) && strlen($server['authkey']) === 40) {
             $server['authkey'] = EncryptedValue::encryptIfEnabled($server['authkey']);
         }
+
+        try {
+            // Clean caches when remote server setting changed
+            RedisTool::unlink(RedisTool::init(), ["misp:fetched_sightings:{$this->id}", "misp:event_index:{$this->id}"]);
+        } catch (Exception $e) {
+            // ignore
+        }
+
         return true;
     }
 
diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php
index 1e3e8c44c..be16e0650 100644
--- a/app/Model/Sighting.php
+++ b/app/Model/Sighting.php
@@ -1293,11 +1293,81 @@ class Sighting extends AppModel
         $eventUuids = [];
         foreach ($remoteEvents as $remoteEvent) {
             if (isset($localEvents[$remoteEvent['uuid']]) && $localEvents[$remoteEvent['uuid']] < $remoteEvent['sighting_timestamp']) {
-                $eventUuids[] = $remoteEvent['uuid'];
+                $eventUuids[$remoteEvent['uuid']] = $remoteEvent['sighting_timestamp'];
             }
         }
         unset($remoteEvents, $localEvents);
 
+        if (empty($eventUuids)) {
+            return 0;
+        }
+
+        $this->removeFetched($serverSync->serverId(), $eventUuids);
+        if (empty($eventUuids)) {
+            return 0;
+        }
+
+        return $this->pullSightingNewWay($user, $eventUuids, $serverSync);
+    }
+
+    /**
+     * New way how to fetch sighting for events without fetching the whole event.
+     *
+     * @param array $user
+     * @param array $eventUuids With UUID in key and remote sighting_timestamp as value
+     * @param ServerSyncTool $serverSync
+     * @return int
+     * @throws Exception
+     */
+    private function pullSightingNewWay(array $user, array $eventUuids, ServerSyncTool $serverSync)
+    {
+        $uuids = array_keys($eventUuids);
+        $saved = 0;
+        foreach (array_chunk($uuids, 100) as $chunk) {
+            try {
+                $sightings = $serverSync->fetchSightingsForEvents($chunk);
+            } catch (Exception $e) {
+                $this->logException("Failed downloading the sightings from {$serverSync->server()['Server']['name']}.", $e);
+                continue;
+            }
+
+            $sightingsToSave = [];
+            foreach ($sightings as $sighting) {
+                $sighting = $sighting['Sighting'];
+                $attributeUuid = $sighting['Attribute']['uuid'];
+                $eventUuid = $sighting['Event']['uuid'];
+                unset($sighting['Event'], $sighting['Attribute']);
+                $sighting['attribute_uuid'] = $attributeUuid;
+                $sightingsToSave[$eventUuid][] = $sighting;
+            }
+
+            $savedUuids = [];
+            foreach ($sightingsToSave as $eventUuid => $sightings) {
+                $savedForEvent = $this->bulkSaveSightings($eventUuid, $sightings, $user, $serverSync->serverId());
+                if ($savedForEvent) {
+                    $saved += $savedForEvent;
+                    $savedUuids[] = $eventUuid;
+                }
+            }
+
+            // Save to Redis that we fetched event sightings, that was not saved. This avoid fetching sightings for
+            // same event that has sightings not visible to user again and again.
+            foreach (array_diff($uuids, $savedUuids) as $notSavedUuid) {
+                $this->saveFetched($serverSync->serverId(), $notSavedUuid, $eventUuids[$notSavedUuid]);
+            }
+        }
+        return $saved;
+    }
+
+    /**
+     * @param array $user
+     * @param array $eventUuids With UUID in key and remote sighting_timestamp as value
+     * @param ServerSyncTool $serverSync
+     * @return int
+     * @throws Exception
+     */
+    private function pullSightingOldWay(array $user, array $eventUuids, ServerSyncTool $serverSync)
+    {
         $saved = 0;
         // We don't need some of the event data, like correlations and event reports
         $params = [
@@ -1313,7 +1383,7 @@ class Sighting extends AppModel
         ];
         // now process the $eventUuids to pull each of the events sequentially
         // download each event and save sightings
-        foreach ($eventUuids as $eventUuid) {
+        foreach ($eventUuids as $eventUuid => $sightingTimestamp) {
             try {
                 $event = $serverSync->fetchEvent($eventUuid, $params)->json();
             } catch (Exception $e) {
@@ -1343,12 +1413,45 @@ class Sighting extends AppModel
                 $result = $this->bulkSaveSightings($event['Event']['uuid'], $sightings, $user, $serverSync->serverId());
                 if (is_numeric($result)) {
                     $saved += $result;
+                } else {
+                    $this->saveFetched($serverSync->serverId(), $eventUuid, $sightingTimestamp);
                 }
             }
         }
         return $saved;
     }
 
+    /**
+     * Remove from fetching events that was already fetched with the same sighting_timestamp
+     * @param int $serverId
+     * @param array $eventUuids
+     * @return void
+     * @throws RedisException
+     */
+    private function removeFetched($serverId, array &$eventUuids)
+    {
+        $lastFetched = RedisTool::init()->hMGet("misp:fetched_sightings:$serverId", array_keys($eventUuids));
+        foreach ($lastFetched as $uuid => $savedTimestamp) {
+            if ($savedTimestamp == $eventUuids[$uuid]) {
+                unset($eventUuids[$uuid]); // event with the same sighting_timestamp was already fetched
+            }
+        }
+    }
+
+    /**
+     * @param int $serverId
+     * @param string $eventUuid
+     * @param int $sightingTimestamp
+     * @return void
+     * @throws RedisException
+     */
+    private function saveFetched($serverId, $eventUuid, $sightingTimestamp)
+    {
+        $redis = RedisTool::init();
+        $redis->hSet("misp:fetched_sightings:$serverId", $eventUuid, $sightingTimestamp);
+        $redis->expire("misp:fetched_sightings:$serverId", 24 * 3600); // keep for one day
+    }
+
     /**
      * @return int Timestamp
      */
@@ -1400,15 +1503,10 @@ class Sighting extends AppModel
      */
     private function isReporter($eventId, $orgId)
     {
-        return (bool)$this->find('first', array(
-            'recursive' => -1,
-            'callbacks' => false,
-            'fields' => ['Sighting.id'],
-            'conditions' => array(
-                'Sighting.event_id' => $eventId,
-                'Sighting.org_id' => $orgId,
-            )
-        ));
+        return $this->hasAny([
+            'Sighting.event_id' => $eventId,
+            'Sighting.org_id' => $orgId,
+        ]);
     }
 
     /**

From 5a1a8aace9c2f82a75cf5ceff9d8bbd82b182e31 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 29 Oct 2022 10:03:51 +0200
Subject: [PATCH 03/11] chg: [api] Allow to include uuids to sighting

---
 .../Component/RestResponseComponent.php       |  8 ++-
 app/Model/Sighting.php                        | 55 ++++++++++++-------
 2 files changed, 42 insertions(+), 21 deletions(-)

diff --git a/app/Controller/Component/RestResponseComponent.php b/app/Controller/Component/RestResponseComponent.php
index b5ef9d949..28d4a60e0 100644
--- a/app/Controller/Component/RestResponseComponent.php
+++ b/app/Controller/Component/RestResponseComponent.php
@@ -261,7 +261,7 @@ class RestResponseComponent extends Component
             'restSearch' => array(
                 'description' => "Search MISP sightings using a list of filter parameters and return the data in the JSON format. The search is available on an event, attribute or instance level, just select the scope via the URL (/sighting/restSearch/event vs /sighting/restSearch/attribute vs /sighting/restSearch/). id or uuid MUST be provided if context is set.",
                 'mandatory' => array('returnFormat'),
-                'optional' => array('id', 'uuid', 'type', 'from', 'to', 'last', 'org_id', 'source', 'includeAttribute', 'includeEvent'),
+                'optional' => array('id', 'uuid', 'type', 'from', 'to', 'last', 'org_id', 'source', 'includeAttribute', 'includeEvent', 'includeUuid'),
                 'params' => array('context')
             ),
         ),
@@ -1256,6 +1256,12 @@ class RestResponseComponent extends Component
                 'values' => array(1 => 'True', 0 => 'False' ),
                 'help' => 'Include all enabled decaying score'
             ),
+            'includeUuid' => array(
+                'input' => 'radio',
+                'type' => 'integer',
+                'values' => array(1 => 'True', 0 => 'False'),
+                'help' => __('Include matching event and attribute UUID to in the response'),
+            ),
             'includeEvent' => array(
                 'input' => 'radio',
                 'type' => 'integer',
diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php
index be16e0650..c5ee6b31c 100644
--- a/app/Model/Sighting.php
+++ b/app/Model/Sighting.php
@@ -178,11 +178,13 @@ class Sighting extends AppModel
      * Fetch sightings with proper ACL checks
      *
      * @param array $user
-     * @param array $ids
-     * @param bool $withEvent
+     * @param array $ids Sightings IDs
+     * @param bool $includeEvent
+     * @param bool $includeAttribute
+     * @param bool $includeUuid Add attribute and event UUID to sighting
      * @return array
      */
-    private function getSightings(array $user, array $ids, $withEvent = true)
+    private function getSightings(array $user, array $ids, $includeEvent = true, $includeAttribute = false, $includeUuid = false)
     {
         // Fetch all attribute IDs that are connected to sightings
         $attributesIds = $this->find('column', [
@@ -194,6 +196,16 @@ class Sighting extends AppModel
             return [];
         }
 
+        $eventFields = $includeEvent ? ['Event.id', 'Event.uuid', 'Event.orgc_id', 'Event.org_id', 'Event.info'] : ['Event.org_id'];
+        if ($includeUuid && !$includeEvent) {
+            $eventFields[] = 'Event.uuid';
+        }
+
+        $attributeFields = $includeAttribute ? ['Attribute.id', 'Attribute.value','Attribute.uuid', 'Attribute.type', 'Attribute.category', 'Attribute.to_ids'] : ['Attribute.id', 'Attribute.value'];
+        if ($includeUuid && !$includeAttribute) {
+            $attributeFields[] = 'Attribute.uuid';
+        }
+
         // Fetch all attributes that are connected to sightings and user can see them
         $attributeConditions = $this->Attribute->buildConditions($user);
         $attributeConditions['Attribute.id'] = $attributesIds;
@@ -202,11 +214,11 @@ class Sighting extends AppModel
             'conditions' => $attributeConditions,
             'contain' => [
                 'Event' => [
-                    'fields' => $withEvent ? ['Event.id', 'Event.uuid', 'Event.orgc_id', 'Event.org_id', 'Event.info'] : ['Event.org_id']
+                    'fields' => $eventFields,
                 ],
                 'Object',
             ],
-            'fields' => ['Attribute.id', 'Attribute.value','Attribute.uuid', 'Attribute.type', 'Attribute.category', 'Attribute.to_ids', 'Event.org_id'],
+            'fields' => $attributeFields,
             'order' => [],
         ]);
 
@@ -261,15 +273,20 @@ class Sighting extends AppModel
             }
 
             // rearrange it to match the event format of fetchevent
-            $result = [
-                'Sighting' => $sighting['Sighting'],
-            ];
-            $result['Sighting']['Attribute'] = $sightingAttribute['Attribute'];
-            if ($withEvent) {
-                $sightingAttribute['Event']['Orgc']['name'] = $this->getOrganisationById($sightingAttribute['Event']['orgc_id'])['name'];
-                $result['Sighting']['Event'] = $sightingAttribute['Event'];
+            $result = $sighting['Sighting'];
+            $result['value'] = $sightingAttribute['Attribute']['value'];
+            if ($includeUuid) {
+                $result['attribute_uuid'] = $sightingAttribute['Attribute']['uuid'];
+                $result['event_uuid'] = $sightingAttribute['Event']['uuid'];
             }
-            $results[] = $result;
+            if ($includeAttribute) {
+                $result['Attribute'] = $sightingAttribute['Attribute'];
+            }
+            if ($includeEvent) {
+                $sightingAttribute['Event']['Orgc']['name'] = $this->getOrganisationById($sightingAttribute['Event']['orgc_id'])['name'];
+                $result['Event'] = $sightingAttribute['Event'];
+            }
+            $results[] = ['Sighting' => $result];
         }
 
         return $results;
@@ -961,12 +978,12 @@ class Sighting extends AppModel
      * @return TmpFileTool
      * @throws Exception
      */
-    public function restSearch($user, $returnFormat, $filters)
+    public function restSearch(array $user, $returnFormat, $filters)
     {
         $allowedContext = array('event', 'attribute');
         // validate context
         if (isset($filters['context']) && !in_array($filters['context'], $allowedContext, true)) {
-            throw new MethodNotAllowedException(__('Invalid context.'));
+            throw new MethodNotAllowedException(__('Invalid context %s.', $filters['context']));
         }
         // ensure that an id or uuid is provided if context is set
         if (!empty($filters['context']) && !(isset($filters['id']) || isset($filters['uuid'])) ) {
@@ -1051,6 +1068,8 @@ class Sighting extends AppModel
 
         $includeAttribute = isset($filters['includeAttribute']) && $filters['includeAttribute'];
         $includeEvent = isset($filters['includeEvent']) && $filters['includeEvent'];
+        $includeUuid = isset($filters['includeUuid']) && $filters['includeUuid'];
+
         $requestedAttributes = ['id', 'attribute_id', 'event_id', 'org_id', 'date_sighting', 'uuid', 'source', 'type'];
         if ($includeAttribute) {
             $requestedAttributes = array_merge($requestedAttributes, ['attribute_uuid', 'attribute_type', 'attribute_category', 'attribute_to_ids', 'attribute_value']);
@@ -1074,12 +1093,8 @@ class Sighting extends AppModel
 
         foreach (array_chunk($sightingIds, 500) as $chunk) {
             // fetch sightings with ACL checks and sighting policies
-            $sightings = $this->getSightings($user, $chunk, $includeEvent);
+            $sightings = $this->getSightings($user, $chunk, $includeEvent, $includeAttribute, $includeUuid);
             foreach ($sightings as $sighting) {
-                $sighting['Sighting']['value'] = $sighting['Sighting']['Attribute']['value'];
-                if (!$includeAttribute) {
-                    unset($sighting['Sighting']['Attribute']);
-                }
                 $tmpfile->writeWithSeparator($exportTool->handler($sighting, $exportToolParams), $separator);
             }
         }

From 77fd20a98f2b5f3fe9867de5f929370b0560f7f3 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 29 Oct 2022 10:26:06 +0200
Subject: [PATCH 04/11] chg: [sightings] Optimised fetching

---
 app/Model/Sighting.php | 50 ++++++++++++++++++++----------------------
 1 file changed, 24 insertions(+), 26 deletions(-)

diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php
index c5ee6b31c..d07a3e98d 100644
--- a/app/Model/Sighting.php
+++ b/app/Model/Sighting.php
@@ -227,28 +227,7 @@ class Sighting extends AppModel
         }
 
         // Create conditions for fetching just sightings user can see according to sightings policy
-        $sightingsPolicy = $this->sightingsPolicy();
-        $attributesById = [];
-        $conditions = [];
-        foreach ($attributes as $attribute) {
-            $attributesById[$attribute['Attribute']['id']] = $attribute;
-
-            $attributeConditions = ['Sighting.attribute_id' => $attribute['Attribute']['id']];
-            $ownEvent = $user['Role']['perm_site_admin'] || $attribute['Event']['org_id'] == $user['org_id'];
-            if (!$ownEvent) {
-                if ($sightingsPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
-                    $attributeConditions['Sighting.org_id'] = $user['org_id'];
-                } else if ($sightingsPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) {
-                    if (!$this->isReporter($attribute['Event']['id'], $user['org_id'])) {
-                        continue; // skip attribute
-                    }
-                } else if ($sightingsPolicy === self::SIGHTING_POLICY_HOST_ORG) {
-                    $attributeConditions['Sighting.org_id'] = [$user['org_id'], Configure::read('MISP.host_org_id')];
-                }
-            }
-            $conditions['OR'][] = $attributeConditions;
-        }
-
+        $conditions = $this->createConditionsByAttributes($user, $attributes);
         if (empty($conditions)) {
             return [];
         }
@@ -263,6 +242,11 @@ class Sighting extends AppModel
             return [];
         }
 
+        $attributesById = [];
+        foreach ($attributes as $attribute) {
+            $attributesById[$attribute['Attribute']['id']] = $attribute;
+        }
+
         $anonymise = Configure::read('Plugin.Sightings_anonymise');
         $results = [];
         foreach ($sightings as $sighting) {
@@ -387,12 +371,28 @@ class Sighting extends AppModel
             return ['data' => [], 'csv' => []];
         }
 
+        $conditions = $this->createConditionsByAttributes($user, $attributes);
+        $groupedSightings = $this->fetchGroupedSightings($conditions, $user);
+        return $this->generateStatistics($groupedSightings, $csvWithFalsePositive);
+    }
+
+    /**
+     * @param array $user
+     * @param array $attributes
+     * @return array
+     */
+    private function createConditionsByAttributes(array $user, array $attributes)
+    {
         $sightingsPolicy = $this->sightingsPolicy();
 
+        if ($sightingsPolicy === self::SIGHTING_POLICY_EVERYONE || $user['Role']['perm_site_admin']) {
+            return ['Sighting.attribute_id' => array_column(array_column($attributes, 'Attribute'), 'id')];
+        }
+
         $conditions = [];
         foreach ($attributes as $attribute) {
             $attributeConditions = ['Sighting.attribute_id' => $attribute['Attribute']['id']];
-            $ownEvent = $user['Role']['perm_site_admin'] || $attribute['Event']['org_id'] == $user['org_id'];
+            $ownEvent = $attribute['Event']['org_id'] == $user['org_id'];
             if (!$ownEvent) {
                 if ($sightingsPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
                     $attributeConditions['Sighting.org_id'] = $user['org_id'];
@@ -406,9 +406,7 @@ class Sighting extends AppModel
             }
             $conditions['OR'][] = $attributeConditions;
         }
-
-        $groupedSightings = $this->fetchGroupedSightings($conditions, $user);
-        return $this->generateStatistics($groupedSightings, $csvWithFalsePositive);
+        return $conditions;
     }
 
     /**

From 35b00b6c84a045c7bfca6c8c9005e1573e926d5b Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 29 Oct 2022 10:33:43 +0200
Subject: [PATCH 05/11] chg: [sighting] Include organisation in rest response

---
 app/Model/Sighting.php | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php
index d07a3e98d..006f41c6e 100644
--- a/app/Model/Sighting.php
+++ b/app/Model/Sighting.php
@@ -270,6 +270,9 @@ class Sighting extends AppModel
                 $sightingAttribute['Event']['Orgc']['name'] = $this->getOrganisationById($sightingAttribute['Event']['orgc_id'])['name'];
                 $result['Event'] = $sightingAttribute['Event'];
             }
+            if (isset($result['org_id']) && $result['org_id'] != 0) {
+                $result['Organisation'] = $this->getOrganisationById($result['org_id']);
+            }
             $results[] = ['Sighting' => $result];
         }
 
@@ -1543,7 +1546,7 @@ class Sighting extends AppModel
             $org = $org['Organisation'];
         }
         $this->orgCache[$orgId] = $org;
-        return $this->orgCache[$orgId];
+        return $org;
     }
 
     /**

From 338de3178cd6fc1c32eb7037e4f8884a5464e38c Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 29 Oct 2022 10:37:06 +0200
Subject: [PATCH 06/11] chg: [sync] Use new sighting pull for new MISP
 instances

---
 app/Lib/Tools/ServerSyncTool.php | 9 ++++++---
 app/Model/Sighting.php           | 6 +++++-
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/app/Lib/Tools/ServerSyncTool.php b/app/Lib/Tools/ServerSyncTool.php
index e5f92030a..8f9c1c014 100644
--- a/app/Lib/Tools/ServerSyncTool.php
+++ b/app/Lib/Tools/ServerSyncTool.php
@@ -13,7 +13,8 @@ class ServerSyncTool
         FEATURE_POST_TEST = 'post_test',
         FEATURE_EDIT_OF_GALAXY_CLUSTER = 'edit_of_galaxy_cluster',
         PERM_SYNC = 'perm_sync',
-        PERM_GALAXY_EDITOR = 'perm_galaxy_editor';
+        PERM_GALAXY_EDITOR = 'perm_galaxy_editor',
+        FEATURE_SIGHTING_REST_SEARCH = 'sighting_rest';
 
     /** @var array */
     private $server;
@@ -239,8 +240,7 @@ class ServerSyncTool
         return $this->post('/sightings/restSearch/event', [
             'returnFormat' => 'json',
             'last' => 0, // fetch all
-            'includeAttribute' => true,
-            'includeEvent' => true,
+            'includeUuid' => true,
             'uuid' => $eventUuids,
         ])->json()['response'];
     }
@@ -406,6 +406,9 @@ class ServerSyncTool
                 return isset($info['perm_sync']) && $info['perm_sync'];
             case self::PERM_GALAXY_EDITOR:
                 return isset($info['perm_galaxy_editor']) && $info['perm_galaxy_editor'];
+            case self::FEATURE_SIGHTING_REST_SEARCH:
+                $version = explode('.', $info['version']);
+                return $version[0] == 2 && $version[1] == 4 && $version[2] > 164;
             default:
                 throw new InvalidArgumentException("Invalid flag `$flag` provided");
         }
diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php
index 006f41c6e..23f1066ce 100644
--- a/app/Model/Sighting.php
+++ b/app/Model/Sighting.php
@@ -1323,7 +1323,11 @@ class Sighting extends AppModel
             return 0;
         }
 
-        return $this->pullSightingNewWay($user, $eventUuids, $serverSync);
+        if ($serverSync->isSupported(ServerSyncTool::FEATURE_SIGHTING_REST_SEARCH)) {
+            return $this->pullSightingNewWay($user, $eventUuids, $serverSync);
+        } else {
+            return $this->pullSightingOldWay($user, $eventUuids, $serverSync);
+        }
     }
 
     /**

From 1bc02930cf6af3311deb01d864e7042f5aa824f8 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 29 Oct 2022 10:46:31 +0200
Subject: [PATCH 07/11] fix: [sighting] Return just requested sighting

---
 app/Model/Sighting.php | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php
index 23f1066ce..a7c089c74 100644
--- a/app/Model/Sighting.php
+++ b/app/Model/Sighting.php
@@ -232,6 +232,7 @@ class Sighting extends AppModel
             return [];
         }
 
+        $conditions['Sighting.id'] = $ids;
         $sightings = $this->find('all', [
             'recursive' => -1,
             'conditions' => $conditions,
@@ -1508,7 +1509,7 @@ class Sighting extends AppModel
         }
         return $this->Organisation->find('list', [
             'fields' => ['Organisation.uuid', 'Organisation.id'],
-            'conditions' => ['Organisation.uuid' => array_unique(array_column($organisations, 'uuid'))],
+            'conditions' => ['Organisation.uuid' => array_unique(array_column($organisations, 'uuid'), SORT_REGULAR)],
         ]);
     }
 

From 4fe5a73386b82f23974610a4cfe9dc09b270d47d Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 29 Oct 2022 11:07:05 +0200
Subject: [PATCH 08/11] chg: [internal] Use subquery to sighting fetching

---
 app/Model/AppModel.php | 15 ++++++++++-----
 app/Model/Sighting.php | 43 +++++++++++++++++-------------------------
 2 files changed, 27 insertions(+), 31 deletions(-)

diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php
index 125e86f05..8465e5794 100644
--- a/app/Model/AppModel.php
+++ b/app/Model/AppModel.php
@@ -2983,7 +2983,15 @@ class AppModel extends Model
         return self::$loadedBackgroundJobsTool;
     }
 
-    // generate a generic subquery - options needs to include conditions
+    /**
+     * Generate a generic subquery - options needs to include conditions
+     *
+     * @param AppModel $model
+     * @param array $options
+     * @param string $lookupKey
+     * @param bool $negation
+     * @return string[]
+     */
     protected function subQueryGenerator(AppModel $model, array $options, $lookupKey, $negation = false)
     {
         $defaults = array(
@@ -3012,10 +3020,7 @@ class AppModel extends Model
         } else {
             $subQuery = $lookupKey . ' IN (' . $subQuery . ') ';
         }
-        $conditions = array(
-            $db->expression($subQuery)->value
-        );
-        return $conditions;
+        return [$subQuery];
     }
 
     // start a benchmark run for the given bench name
diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php
index a7c089c74..35bcaa7e1 100644
--- a/app/Model/Sighting.php
+++ b/app/Model/Sighting.php
@@ -186,16 +186,6 @@ class Sighting extends AppModel
      */
     private function getSightings(array $user, array $ids, $includeEvent = true, $includeAttribute = false, $includeUuid = false)
     {
-        // Fetch all attribute IDs that are connected to sightings
-        $attributesIds = $this->find('column', [
-           'fields' => ['Sighting.attribute_id'],
-           'conditions' => ['Sighting.id' => $ids],
-           'unique' => true,
-        ]);
-        if (empty($attributesIds)) {
-            return [];
-        }
-
         $eventFields = $includeEvent ? ['Event.id', 'Event.uuid', 'Event.orgc_id', 'Event.org_id', 'Event.info'] : ['Event.org_id'];
         if ($includeUuid && !$includeEvent) {
             $eventFields[] = 'Event.uuid';
@@ -208,7 +198,11 @@ class Sighting extends AppModel
 
         // Fetch all attributes that are connected to sightings and user can see them
         $attributeConditions = $this->Attribute->buildConditions($user);
-        $attributeConditions['Attribute.id'] = $attributesIds;
+        $subQueryOptions = [
+            'fields' => ['DISTINCT Sighting.attribute_id'],
+            'conditions' => ['Sighting.id' => $ids],
+        ];
+        $attributeConditions[] = $this->subQueryGenerator($this, $subQueryOptions, 'Attribute.id');
         $attributes = $this->Attribute->find('all', [
             'recursive' => -1,
             'conditions' => $attributeConditions,
@@ -226,7 +220,7 @@ class Sighting extends AppModel
             return [];
         }
 
-        // Create conditions for fetching just sightings user can see according to sightings policy
+        // Create conditions for fetching just sightings that user can see according to sightings policy
         $conditions = $this->createConditionsByAttributes($user, $attributes);
         if (empty($conditions)) {
             return [];
@@ -284,10 +278,9 @@ class Sighting extends AppModel
      * Return sighting without ACL checks
      *
      * @param int $id
-     * @param bool $withEvent
      * @return array
      */
-    public function getSighting($id, $withEvent = true)
+    public function getSighting($id)
     {
         $sighting = $this->find('first', array(
             'recursive' => -1,
@@ -296,7 +289,7 @@ class Sighting extends AppModel
                     'fields' => array('Attribute.value', 'Attribute.id', 'Attribute.uuid', 'Attribute.type', 'Attribute.category', 'Attribute.to_ids')
                 ),
                 'Event' => array(
-                    'fields' => $withEvent ? ['Event.id', 'Event.uuid', 'Event.orgc_id', 'Event.org_id', 'Event.info'] : ['Event.org_id'],
+                    'fields' => ['Event.id', 'Event.uuid', 'Event.orgc_id', 'Event.org_id', 'Event.info'],
                 )
             ),
             'conditions' => array('Sighting.id' => $id)
@@ -306,17 +299,13 @@ class Sighting extends AppModel
         }
 
         // Put event organisation name from cache
-        if ($withEvent) {
-            $sighting['Event']['Orgc']['name'] = $this->getOrganisationById($sighting['Event']['orgc_id'])['name'];
-        }
+        $sighting['Event']['Orgc']['name'] = $this->getOrganisationById($sighting['Event']['orgc_id'])['name'];
 
         // rearrange it to match the event format of fetchevent
         $result = array(
             'Sighting' => $sighting['Sighting']
         );
-        if ($withEvent) {
-            $result['Sighting']['Event'] = $sighting['Event'];
-        }
+        $result['Sighting']['Event'] = $sighting['Event'];
         $result['Sighting']['Attribute'] = $sighting['Attribute'];
         return $result;
     }
@@ -382,7 +371,7 @@ class Sighting extends AppModel
 
     /**
      * @param array $user
-     * @param array $attributes
+     * @param array $attributes Attributes with `Attribute.id`, `Event.id` and `Event.org_id` fields
      * @return array
      */
     private function createConditionsByAttributes(array $user, array $attributes)
@@ -393,19 +382,21 @@ class Sighting extends AppModel
             return ['Sighting.attribute_id' => array_column(array_column($attributes, 'Attribute'), 'id')];
         }
 
+        $hostOrgId = Configure::read('MISP.host_org_id');
+        $userOrgId = $user['org_id'];
         $conditions = [];
         foreach ($attributes as $attribute) {
             $attributeConditions = ['Sighting.attribute_id' => $attribute['Attribute']['id']];
-            $ownEvent = $attribute['Event']['org_id'] == $user['org_id'];
+            $ownEvent = $attribute['Event']['org_id'] == $userOrgId;
             if (!$ownEvent) {
                 if ($sightingsPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
-                    $attributeConditions['Sighting.org_id'] = $user['org_id'];
+                    $attributeConditions['Sighting.org_id'] = $userOrgId;
                 } else if ($sightingsPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) {
-                    if (!$this->isReporter($attribute['Event']['id'], $user['org_id'])) {
+                    if (!$this->isReporter($attribute['Event']['id'], $userOrgId)) {
                         continue; // skip attribute
                     }
                 } else if ($sightingsPolicy === self::SIGHTING_POLICY_HOST_ORG) {
-                    $attributeConditions['Sighting.org_id'] = [$user['org_id'], Configure::read('MISP.host_org_id')];
+                    $attributeConditions['Sighting.org_id'] = [$userOrgId, $hostOrgId];
                 }
             }
             $conditions['OR'][] = $attributeConditions;

From e23b470da41e3309a29a0ac791bfb6cf3dd14407 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 29 Oct 2022 17:38:06 +0200
Subject: [PATCH 09/11] new: [test] Sighting rest search test

---
 tests/testlive_security.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tests/testlive_security.py b/tests/testlive_security.py
index adf90cb27..d9f2d7a33 100644
--- a/tests/testlive_security.py
+++ b/tests/testlive_security.py
@@ -1536,7 +1536,8 @@ class TestSecurity(unittest.TestCase):
         event = user1.add_event(self.__generate_event())
         check_response(event)
         check_response(user1.add_sighting(s, event.Attribute[0]))
-        self.assertEqual(len(user1.sightings(event)), 1, "User should see hos own sighting")
+        self.assertEqual(len(user1.sightings(event)), 1, "User should see only own sighting")
+        self.assertEqual(len(user1.search_sightings('event', event.id)), 1)
 
         org = self.__create_org()
         user = self.__create_user(org.id, ROLE.USER)
@@ -1544,9 +1545,11 @@ class TestSecurity(unittest.TestCase):
         user2.global_pythonify = True
 
         self.assertEqual(len(user2.sightings(event)), 0, "User should not seen any sighting")
+        self.assertEqual(len(user2.search_sightings('event', event.id)), 0)
 
         with self.__setting({"MISP.host_org_id": self.test_org.id, "Plugin.Sightings_policy": 3}):
             self.assertEqual(len(user2.sightings(event)), 1, "User should see host org sighting")
+            self.assertEqual(len(user2.search_sightings('event', event.id)), 1)
 
         self.admin_misp_connector.delete_event(event)
         self.admin_misp_connector.delete_user(user)

From bd0dde5e37e2a56e4b485f1a229c0cffcf8b75cb Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 29 Oct 2022 18:25:44 +0200
Subject: [PATCH 10/11] chg: [API] Throw exception if invalid ID provided

---
 app/Model/Sighting.php | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php
index 35bcaa7e1..015a3cdc9 100644
--- a/app/Model/Sighting.php
+++ b/app/Model/Sighting.php
@@ -1034,6 +1034,16 @@ class Sighting extends AppModel
         }
 
         if (!empty($filters['id'])) {
+            if (is_array($filters['id'])) {
+                foreach ($filters['id'] as $id) {
+                    if (!is_int($id) || !is_numeric($id)) {
+                        throw new BadRequestException("Invalid ID `$id` provided.");
+                    }
+                }
+            } else if (!is_int($filters['id']) || !is_numeric($filters['id'])) {
+                throw new BadRequestException("Invalid ID `{$filters['id']}` provided.");
+            }
+
             if ($filters['context'] === 'attribute') {
                 $conditions['Sighting.attribute_id'] = $filters['id'];
             } elseif ($filters['context'] === 'event') {
@@ -1051,14 +1061,6 @@ class Sighting extends AppModel
             }
         }
 
-        // fetch sightings matching the query without ACL checks
-        $sightingIds = $this->find('column', [
-            'conditions' => $conditions,
-            'fields' => ['Sighting.id'],
-            'contain' => $contain,
-            'order' => 'Sighting.id',
-        ]);
-
         $includeAttribute = isset($filters['includeAttribute']) && $filters['includeAttribute'];
         $includeEvent = isset($filters['includeEvent']) && $filters['includeEvent'];
         $includeUuid = isset($filters['includeUuid']) && $filters['includeUuid'];
@@ -1084,6 +1086,14 @@ class Sighting extends AppModel
         $tmpfile->write($exportTool->header($exportToolParams));
         $separator = $exportTool->separator($exportToolParams);
 
+        // fetch sightings matching the query without ACL checks
+        $sightingIds = $this->find('column', [
+            'conditions' => $conditions,
+            'fields' => ['Sighting.id'],
+            'contain' => $contain,
+            'order' => 'Sighting.id',
+        ]);
+
         foreach (array_chunk($sightingIds, 500) as $chunk) {
             // fetch sightings with ACL checks and sighting policies
             $sightings = $this->getSightings($user, $chunk, $includeEvent, $includeAttribute, $includeUuid);

From a27e036c5b7cc25028b0059c0e905f5c21c3d575 Mon Sep 17 00:00:00 2001
From: Jakub Onderka <jakub.onderka@gmail.com>
Date: Sat, 29 Oct 2022 21:03:29 +0200
Subject: [PATCH 11/11] chg: [internal] Faster fetching event index

---
 app/Controller/EventsController.php |  64 ++++--------
 app/Model/CryptographicKey.php      | 153 +++++++++++++++++-----------
 2 files changed, 111 insertions(+), 106 deletions(-)

diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php
index 092a94b1d..d33f5c6a1 100644
--- a/app/Controller/EventsController.php
+++ b/app/Controller/EventsController.php
@@ -768,26 +768,25 @@ class EventsController extends AppController
      */
     private function __indexRestResponse(array $passedArgs)
     {
-        $isSync = $skipProtected = false;
-        if (!empty($this->request->header('misp-version'))) {
-            $isSync = true;
-            if (version_compare($this->request->header('misp-version'), '2.4.156') < 0) {
-                $skipProtected = true;
-            }
-        }
+        // We do not want to allow instances to pull our data that can't make sense of protected mode events
+        $skipProtected = (
+            !empty($this->request->header('misp-version')) &&
+            version_compare($this->request->header('misp-version'), '2.4.156') < 0
+        );
+
         $fieldNames = $this->Event->schema();
         $minimal = !empty($passedArgs['searchminimal']) || !empty($passedArgs['minimal']);
         if ($minimal) {
             $rules = [
                 'recursive' => -1,
                 'fields' => array('id', 'timestamp', 'sighting_timestamp', 'published', 'uuid', 'protected'),
-                'contain' => array('Orgc.uuid', 'CryptographicKey.fingerprint'),
+                'contain' => array('Orgc.uuid'),
             ];
         } else {
             // Remove user ID from fetched fields
             unset($fieldNames['user_id']);
             $rules = [
-                'contain' => ['EventTag', 'CryptographicKey.fingerprint'],
+                'contain' => ['EventTag'],
                 'fields' => array_keys($fieldNames),
             ];
         }
@@ -801,6 +800,9 @@ class EventsController extends AppController
         if (isset($this->paginate['conditions'])) {
             $rules['conditions'] = $this->paginate['conditions'];
         }
+        if ($skipProtected) {
+            $rules['conditions']['Event.protected'] = 0;
+        }
         $paginationRules = array('page', 'limit', 'sort', 'direction', 'order');
         foreach ($paginationRules as $paginationRule) {
             if (isset($passedArgs[$paginationRule])) {
@@ -837,12 +839,12 @@ class EventsController extends AppController
 
             $events = $absolute_total === 0 ? [] : $this->Event->find('all', $rules);
         }
+
         $isCsvResponse = $this->response->type() === 'text/csv';
-        try {
-            $instanceFingerprint = $this->Event->CryptographicKey->ingestInstanceKey();
-        } catch (Exception $e) {
-            $instanceFingerprint = null;
-        }
+
+        $protectedEventsByInstanceKey = $this->Event->CryptographicKey->protectedEventsByInstanceKey($events);
+        $protectedEventsByInstanceKey = array_flip($protectedEventsByInstanceKey);
+
         if (!$minimal) {
             // Collect all tag IDs that are events
             $tagIds = [];
@@ -889,19 +891,8 @@ class EventsController extends AppController
                 $orgIds[$event['Event']['org_id']] = true;
                 $orgIds[$event['Event']['orgc_id']] = true;
                 $sharingGroupIds[$event['Event']['sharing_group_id']] = true;
-                if ($event['Event']['protected']) {
-                    if ($skipProtected) {
-                        unset($events[$k]);
-                        continue;
-                    }
-                    foreach ($event['CryptographicKey'] as $cryptoKey) {
-                        if ($instanceFingerprint === $cryptoKey['fingerprint']) {
-
-                            continue 2;
-                        }
-                    }
+                if ($event['Event']['protected'] && !isset($protectedEventsByInstanceKey[$event['Event']['id']])) {
                     unset($events[$k]);
-                    continue;
                 }
             }
             $events = array_values($events);
@@ -946,26 +937,9 @@ class EventsController extends AppController
             if ($this->response->type() === 'application/xml') {
                 $events = array('Event' => $events);
             }
-        } else {
-            // We do not want to allow instances to pull our data that can't make sense of protected mode events
-            $skipProtected = (
-                !empty($this->request->header('misp-version')) &&
-                version_compare($this->request->header('misp-version'), '2.4.156') < 0
-            );
+        } else { // minimal
             foreach ($events as $key => $event) {
-                if ($event['Event']['protected']) {
-                    if ($skipProtected) {
-                        unset($events[$key]);
-                        continue;
-                    }
-                    foreach ($event['CryptographicKey'] as $cryptoKey) {
-                        if ($instanceFingerprint === $cryptoKey['fingerprint']) {
-                            $event['Event']['orgc_uuid'] = $event['Orgc']['uuid'];
-                            unset($event['Event']['protected']);
-                            $events[$key] = $event['Event'];
-                            continue 2;
-                        }
-                    }
+                if ($event['Event']['protected'] && !isset($protectedEventsByInstanceKey[$event['Event']['id']])) {
                     unset($events[$key]);
                     continue;
                 }
diff --git a/app/Model/CryptographicKey.php b/app/Model/CryptographicKey.php
index 6ffd0af54..0f7eb1d1d 100644
--- a/app/Model/CryptographicKey.php
+++ b/app/Model/CryptographicKey.php
@@ -27,57 +27,46 @@ class CryptographicKey extends AppModel
         ERROR_WRONG_KEY = 'Wrong key',
         ERROR_INVALID_KEY = 'Invalid key';
 
-    public $validTypes = [
+    const VALID_TYPES = [
         'pgp'
     ];
 
     public $error = false;
 
-    public $validate = [];
+    public $validate = [
+        'uuid' => [
+            'uuid' => [
+                'rule' => 'uuid',
+                'message' => 'Please provide a valid RFC 4122 UUID',
+            ],
+        ],
+        'type' => [
+            'rule' => ['inList', self::VALID_TYPES],
+            'message' => 'Invalid key type',
+            'required' => 'create'
+        ],
+        'key_data' => [
+            'notBlankKey' => [
+                'rule' => 'notBlank',
+                'message' => 'No key data received.',
+                'required' => 'create'
+            ],
+            'validKey' => [
+                'rule' => 'validateKey',
+                'message' => 'Invalid key.',
+                'required' => 'create'
+            ],
+            'uniqueKeyForElement' => [
+                'rule' => 'uniqueKeyForElement',
+                'message' => 'This key is already assigned to the target.',
+                'required' => 'create'
+            ]
+        ]
+    ];
 
     /** @var CryptGpgExtended|null */
     private $gpg;
 
-    public function __construct($id = false, $table = null, $ds = null)
-    {
-        parent::__construct($id, $table, $ds);
-        try {
-            $this->gpg = GpgTool::initializeGpg();
-        } catch (Exception $e) {
-            $this->gpg = null;
-        }
-        $this->validate = [
-            'uuid' => [
-                'uuid' => [
-                    'rule' => 'uuid',
-                    'message' => 'Please provide a valid RFC 4122 UUID',
-                ],
-            ],
-            'type' => [
-                'rule' => ['inList', $this->validTypes],
-                'message' => __('Invalid key type'),
-                'required' => 'create'
-            ],
-            'key_data' => [
-                'notBlankKey' => [
-                    'rule' => 'notBlank',
-                    'message' => __('No key data received.'),
-                    'required' => 'create'
-                ],
-                'validKey' => [
-                    'rule' => 'validateKey',
-                    'message' => __('Invalid key.'),
-                    'required' => 'create'
-                ],
-                'uniqueKeyForElement' => [
-                    'rule' => 'uniqueKeyForElement',
-                    'message' => __('This key is already assigned to the target.'),
-                    'required' => 'create'
-                ]
-            ]
-        ];
-    }
-
     public function beforeSave($options = array())
     {
         $this->data['CryptographicKey']['timestamp'] = time();
@@ -97,20 +86,17 @@ class CryptographicKey extends AppModel
     {
         // If instance key is stored just in GPG homedir, use that key.
         if (Configure::read('MISP.download_gpg_from_homedir')) {
-            if (!$this->gpg) {
-                throw new Exception("Could not initiate GPG");
-            }
             /** @var Crypt_GPG_Key[] $keys */
-            $keys = $this->gpg->getKeys(Configure::read('GnuPG.email'));
+            $keys = $this->getGpg()->getKeys(Configure::read('GnuPG.email'));
             if (empty($keys)) {
                 return false;
             }
-            $this->gpg->addSignKey($keys[0], Configure::read('GnuPG.password'));
+            $this->getGpg()->addSignKey($keys[0], Configure::read('GnuPG.password'));
             return $keys[0]->getPrimaryKey()->getFingerprint();
         }
 
         try {
-            $redis = $this->setupRedisWithException();
+            $redis = RedisTool::init();
         } catch (Exception $e) {
             $redis = false;
         }
@@ -124,33 +110,63 @@ class CryptographicKey extends AppModel
         if (empty($fingerprint)) {
             $file = new File(APP . '/webroot/gpg.asc');
             $instanceKey = $file->read();
-            if (!$this->gpg) {
-                throw new MethodNotAllowedException("Could not initiate GPG");
-            }
             try {
-                $this->gpg->importKey($instanceKey);
+                $this->getGpg()->importKey($instanceKey);
             } catch (Crypt_GPG_NoDataException $e) {
                 throw new MethodNotAllowedException("Could not import the instance key.");
             }
-            $fingerprint = $this->gpg->getFingerprint(Configure::read('GnuPG.email'));
+            $fingerprint = $this->getGpg()->getFingerprint(Configure::read('GnuPG.email'));
             if ($redis) {
                 $redis->setEx($redisKey, 300, $fingerprint);
             }
         }
-        if (!$this->gpg) {
-            throw new MethodNotAllowedException("Could not initiate GPG");
-        }
         try {
-            $this->gpg->addSignKey(Configure::read('GnuPG.email'), Configure::read('GnuPG.password'));
+            $this->getGpg()->addSignKey(Configure::read('GnuPG.email'), Configure::read('GnuPG.password'));
         } catch (Exception $e) {
             throw new NotFoundException('Could not add signing key.');
         }
         return $fingerprint;
     }
 
+    /**
+     * Check if given events are protected by instance key, returns array of Event IDs
+     * @param array $events
+     * @return array Event ID that is protected in key
+     */
+    public function protectedEventsByInstanceKey(array $events)
+    {
+        $eventIds = [];
+        foreach ($events as $event) {
+            if ($event['Event']['protected']) {
+                $eventIds[] = $event['Event']['id'];
+            }
+        }
+
+        if (empty($eventIds)) {
+            return [];
+        }
+
+        try {
+            $instanceKey = $this->ingestInstanceKey();
+        } catch (Exception $e) {
+            // could not fetch instance key
+            return [];
+        }
+
+        return $this->find('column', [
+            'conditions' => [
+                'CryptographicKey.parent_type' => 'Event',
+                'CryptographicKey.parent_id' => $eventIds,
+                'CryptographicKey.fingerprint' => $instanceKey,
+            ],
+            'fields' => ['CryptographicKey.parent_id'],
+            'recursive' => -1,
+        ]);
+    }
+
     /**
      * @param string $data
-     * @return false|string
+     * @return false|string Signature
      * @throws Crypt_GPG_BadPassphraseException
      * @throws Crypt_GPG_Exception
      * @throws Crypt_GPG_KeyNotFoundException
@@ -161,7 +177,7 @@ class CryptographicKey extends AppModel
             return false;
         }
         $data = preg_replace("/\s+/", "", $data);
-        $signature = $this->gpg->sign($data, Crypt_GPG::SIGN_MODE_DETACHED, Crypt_GPG::ARMOR_BINARY);
+        $signature = $this->getGpg()->sign($data, Crypt_GPG::SIGN_MODE_DETACHED, Crypt_GPG::ARMOR_BINARY);
         return $signature;
     }
 
@@ -181,7 +197,7 @@ class CryptographicKey extends AppModel
         }
         $data = preg_replace("/\s+/", "", $data);
         try {
-            $verifiedSignature = $this->gpg->verify($data, $signature);
+            $verifiedSignature = $this->getGpg()->verify($data, $signature);
         } catch (Exception $e) {
             $this->error = self::ERROR_WRONG_KEY;
             return false;
@@ -219,7 +235,7 @@ class CryptographicKey extends AppModel
     private function __extractPGPKeyData($data)
     {
         try {
-            $gpgTool = new GpgTool($this->gpg);
+            $gpgTool = new GpgTool($this->getGpg());
         } catch (Exception $e) {
             $this->logException("GPG couldn't be initialized, GPG encryption and signing will be not available.", $e, LOG_NOTICE);
             return false;
@@ -345,4 +361,19 @@ class CryptographicKey extends AppModel
         $this->deleteAll(['CryptographicKey.id' => $toRemove]);
         $this->loadLog()->createLogEntry($user, 'updateCryptoKeys', $cryptographicKey['parent_type'], $cryptographicKey['parent_id'], $message);
     }
+
+    /**
+     * Lazy load GPG
+     * @return CryptGpgExtended|null
+     * @throws Exception
+     */
+    private function getGpg()
+    {
+        if ($this->gpg) {
+            return $this->gpg;
+        }
+
+        $this->gpg = GpgTool::initializeGpg();
+        return $this->gpg;
+    }
 }