chg: [internal] Faster fetching event index

app/Controller/EventsController.php

@@ -768,26 +768,25 @@ class EventsController extends AppController
      */
     private function __indexRestResponse(array $passedArgs)
     {
-        $isSync = $skipProtected = false;
+        // We do not want to allow instances to pull our data that can't make sense of protected mode events
-        if (!empty($this->request->header('misp-version'))) {
+        $skipProtected = (
-            $isSync = true;
+            !empty($this->request->header('misp-version')) &&
-            if (version_compare($this->request->header('misp-version'), '2.4.156') < 0) {
+            version_compare($this->request->header('misp-version'), '2.4.156') < 0
-                $skipProtected = true;
+        );
-            }
+
-        }
         $fieldNames = $this->Event->schema();
         $minimal = !empty($passedArgs['searchminimal']) || !empty($passedArgs['minimal']);
         if ($minimal) {
             $rules = [
                 'recursive' => -1,
                 'fields' => array('id', 'timestamp', 'sighting_timestamp', 'published', 'uuid', 'protected'),
-                'contain' => array('Orgc.uuid', 'CryptographicKey.fingerprint'),
+                'contain' => array('Orgc.uuid'),
             ];
         } else {
             // Remove user ID from fetched fields
             unset($fieldNames['user_id']);
             $rules = [
-                'contain' => ['EventTag', 'CryptographicKey.fingerprint'],
+                'contain' => ['EventTag'],
                 'fields' => array_keys($fieldNames),
             ];
         }
@@ -801,6 +800,9 @@ class EventsController extends AppController
         if (isset($this->paginate['conditions'])) {
             $rules['conditions'] = $this->paginate['conditions'];
         }
+        if ($skipProtected) {
+            $rules['conditions']['Event.protected'] = 0;
+        }
         $paginationRules = array('page', 'limit', 'sort', 'direction', 'order');
         foreach ($paginationRules as $paginationRule) {
             if (isset($passedArgs[$paginationRule])) {
@@ -837,12 +839,12 @@ class EventsController extends AppController
 
             $events = $absolute_total === 0 ? [] : $this->Event->find('all', $rules);
         }
+
         $isCsvResponse = $this->response->type() === 'text/csv';
-        try {
+
-            $instanceFingerprint = $this->Event->CryptographicKey->ingestInstanceKey();
+        $protectedEventsByInstanceKey = $this->Event->CryptographicKey->protectedEventsByInstanceKey($events);
-        } catch (Exception $e) {
+        $protectedEventsByInstanceKey = array_flip($protectedEventsByInstanceKey);
-            $instanceFingerprint = null;
+
-        }
         if (!$minimal) {
             // Collect all tag IDs that are events
             $tagIds = [];
@@ -889,19 +891,8 @@ class EventsController extends AppController
                 $orgIds[$event['Event']['org_id']] = true;
                 $orgIds[$event['Event']['orgc_id']] = true;
                 $sharingGroupIds[$event['Event']['sharing_group_id']] = true;
-                if ($event['Event']['protected']) {
+                if ($event['Event']['protected'] && !isset($protectedEventsByInstanceKey[$event['Event']['id']])) {
-                    if ($skipProtected) {
-                        unset($events[$k]);
-                        continue;
-                    }
-                    foreach ($event['CryptographicKey'] as $cryptoKey) {
-                        if ($instanceFingerprint === $cryptoKey['fingerprint']) {
-
-                            continue 2;
-                        }
-                    }
                     unset($events[$k]);
-                    continue;
                 }
             }
             $events = array_values($events);
@@ -946,26 +937,9 @@ class EventsController extends AppController
             if ($this->response->type() === 'application/xml') {
                 $events = array('Event' => $events);
             }
-        } else {
+        } else { // minimal
-            // We do not want to allow instances to pull our data that can't make sense of protected mode events
-            $skipProtected = (
-                !empty($this->request->header('misp-version')) &&
-                version_compare($this->request->header('misp-version'), '2.4.156') < 0
-            );
             foreach ($events as $key => $event) {
-                if ($event['Event']['protected']) {
+                if ($event['Event']['protected'] && !isset($protectedEventsByInstanceKey[$event['Event']['id']])) {
-                    if ($skipProtected) {
-                        unset($events[$key]);
-                        continue;
-                    }
-                    foreach ($event['CryptographicKey'] as $cryptoKey) {
-                        if ($instanceFingerprint === $cryptoKey['fingerprint']) {
-                            $event['Event']['orgc_uuid'] = $event['Orgc']['uuid'];
-                            unset($event['Event']['protected']);
-                            $events[$key] = $event['Event'];
-                            continue 2;
-                        }
-                    }
                     unset($events[$key]);
                     continue;
                 }

app/Model/CryptographicKey.php

@@ -27,57 +27,46 @@ class CryptographicKey extends AppModel
         ERROR_WRONG_KEY = 'Wrong key',
         ERROR_INVALID_KEY = 'Invalid key';
 
-    public $validTypes = [
+    const VALID_TYPES = [
         'pgp'
     ];
 
     public $error = false;
 
-    public $validate = [];
+    public $validate = [
+        'uuid' => [
+            'uuid' => [
+                'rule' => 'uuid',
+                'message' => 'Please provide a valid RFC 4122 UUID',
+            ],
+        ],
+        'type' => [
+            'rule' => ['inList', self::VALID_TYPES],
+            'message' => 'Invalid key type',
+            'required' => 'create'
+        ],
+        'key_data' => [
+            'notBlankKey' => [
+                'rule' => 'notBlank',
+                'message' => 'No key data received.',
+                'required' => 'create'
+            ],
+            'validKey' => [
+                'rule' => 'validateKey',
+                'message' => 'Invalid key.',
+                'required' => 'create'
+            ],
+            'uniqueKeyForElement' => [
+                'rule' => 'uniqueKeyForElement',
+                'message' => 'This key is already assigned to the target.',
+                'required' => 'create'
+            ]
+        ]
+    ];
 
     /** @var CryptGpgExtended|null */
     private $gpg;
 
-    public function __construct($id = false, $table = null, $ds = null)
-    {
-        parent::__construct($id, $table, $ds);
-        try {
-            $this->gpg = GpgTool::initializeGpg();
-        } catch (Exception $e) {
-            $this->gpg = null;
-        }
-        $this->validate = [
-            'uuid' => [
-                'uuid' => [
-                    'rule' => 'uuid',
-                    'message' => 'Please provide a valid RFC 4122 UUID',
-                ],
-            ],
-            'type' => [
-                'rule' => ['inList', $this->validTypes],
-                'message' => __('Invalid key type'),
-                'required' => 'create'
-            ],
-            'key_data' => [
-                'notBlankKey' => [
-                    'rule' => 'notBlank',
-                    'message' => __('No key data received.'),
-                    'required' => 'create'
-                ],
-                'validKey' => [
-                    'rule' => 'validateKey',
-                    'message' => __('Invalid key.'),
-                    'required' => 'create'
-                ],
-                'uniqueKeyForElement' => [
-                    'rule' => 'uniqueKeyForElement',
-                    'message' => __('This key is already assigned to the target.'),
-                    'required' => 'create'
-                ]
-            ]
-        ];
-    }
-
     public function beforeSave($options = array())
     {
         $this->data['CryptographicKey']['timestamp'] = time();
@@ -97,20 +86,17 @@ class CryptographicKey extends AppModel
     {
         // If instance key is stored just in GPG homedir, use that key.
         if (Configure::read('MISP.download_gpg_from_homedir')) {
-            if (!$this->gpg) {
-                throw new Exception("Could not initiate GPG");
-            }
             /** @var Crypt_GPG_Key[] $keys */
-            $keys = $this->gpg->getKeys(Configure::read('GnuPG.email'));
+            $keys = $this->getGpg()->getKeys(Configure::read('GnuPG.email'));
             if (empty($keys)) {
                 return false;
             }
-            $this->gpg->addSignKey($keys[0], Configure::read('GnuPG.password'));
+            $this->getGpg()->addSignKey($keys[0], Configure::read('GnuPG.password'));
             return $keys[0]->getPrimaryKey()->getFingerprint();
         }
 
         try {
-            $redis = $this->setupRedisWithException();
+            $redis = RedisTool::init();
         } catch (Exception $e) {
             $redis = false;
         }
@@ -124,33 +110,63 @@ class CryptographicKey extends AppModel
         if (empty($fingerprint)) {
             $file = new File(APP . '/webroot/gpg.asc');
             $instanceKey = $file->read();
-            if (!$this->gpg) {
-                throw new MethodNotAllowedException("Could not initiate GPG");
-            }
             try {
-                $this->gpg->importKey($instanceKey);
+                $this->getGpg()->importKey($instanceKey);
             } catch (Crypt_GPG_NoDataException $e) {
                 throw new MethodNotAllowedException("Could not import the instance key.");
             }
-            $fingerprint = $this->gpg->getFingerprint(Configure::read('GnuPG.email'));
+            $fingerprint = $this->getGpg()->getFingerprint(Configure::read('GnuPG.email'));
             if ($redis) {
                 $redis->setEx($redisKey, 300, $fingerprint);
             }
         }
-        if (!$this->gpg) {
-            throw new MethodNotAllowedException("Could not initiate GPG");
-        }
         try {
-            $this->gpg->addSignKey(Configure::read('GnuPG.email'), Configure::read('GnuPG.password'));
+            $this->getGpg()->addSignKey(Configure::read('GnuPG.email'), Configure::read('GnuPG.password'));
         } catch (Exception $e) {
             throw new NotFoundException('Could not add signing key.');
         }
         return $fingerprint;
     }
 
+    /**
+     * Check if given events are protected by instance key, returns array of Event IDs
+     * @param array $events
+     * @return array Event ID that is protected in key
+     */
+    public function protectedEventsByInstanceKey(array $events)
+    {
+        $eventIds = [];
+        foreach ($events as $event) {
+            if ($event['Event']['protected']) {
+                $eventIds[] = $event['Event']['id'];
+            }
+        }
+
+        if (empty($eventIds)) {
+            return [];
+        }
+
+        try {
+            $instanceKey = $this->ingestInstanceKey();
+        } catch (Exception $e) {
+            // could not fetch instance key
+            return [];
+        }
+
+        return $this->find('column', [
+            'conditions' => [
+                'CryptographicKey.parent_type' => 'Event',
+                'CryptographicKey.parent_id' => $eventIds,
+                'CryptographicKey.fingerprint' => $instanceKey,
+            ],
+            'fields' => ['CryptographicKey.parent_id'],
+            'recursive' => -1,
+        ]);
+    }
+
     /**
      * @param string $data
-     * @return false|string
+     * @return false|string Signature
      * @throws Crypt_GPG_BadPassphraseException
      * @throws Crypt_GPG_Exception
      * @throws Crypt_GPG_KeyNotFoundException
@@ -161,7 +177,7 @@ class CryptographicKey extends AppModel
             return false;
         }
         $data = preg_replace("/\s+/", "", $data);
-        $signature = $this->gpg->sign($data, Crypt_GPG::SIGN_MODE_DETACHED, Crypt_GPG::ARMOR_BINARY);
+        $signature = $this->getGpg()->sign($data, Crypt_GPG::SIGN_MODE_DETACHED, Crypt_GPG::ARMOR_BINARY);
         return $signature;
     }
 
@@ -181,7 +197,7 @@ class CryptographicKey extends AppModel
         }
         $data = preg_replace("/\s+/", "", $data);
         try {
-            $verifiedSignature = $this->gpg->verify($data, $signature);
+            $verifiedSignature = $this->getGpg()->verify($data, $signature);
         } catch (Exception $e) {
             $this->error = self::ERROR_WRONG_KEY;
             return false;
@@ -219,7 +235,7 @@ class CryptographicKey extends AppModel
     private function __extractPGPKeyData($data)
     {
         try {
-            $gpgTool = new GpgTool($this->gpg);
+            $gpgTool = new GpgTool($this->getGpg());
         } catch (Exception $e) {
             $this->logException("GPG couldn't be initialized, GPG encryption and signing will be not available.", $e, LOG_NOTICE);
             return false;
@@ -345,4 +361,19 @@ class CryptographicKey extends AppModel
         $this->deleteAll(['CryptographicKey.id' => $toRemove]);
         $this->loadLog()->createLogEntry($user, 'updateCryptoKeys', $cryptographicKey['parent_type'], $cryptographicKey['parent_id'], $message);
     }
+
+    /**
+     * Lazy load GPG
+     * @return CryptGpgExtended|null
+     * @throws Exception
+     */
+    private function getGpg()
+    {
+        if ($this->gpg) {
+            return $this->gpg;
+        }
+
+        $this->gpg = GpgTool::initializeGpg();
+        return $this->gpg;
+    }
 }