MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [internal] Faster fetching event index

pull/8720/head
Jakub Onderka 2022-10-29 21:03:29 +02:00
parent bd0dde5e37
commit a27e036c5b
2 changed files with 111 additions and 106 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

64
app/Controller/EventsController.php
View File

@ -768,26 +768,25 @@ class EventsController extends AppController
*/
private function __indexRestResponse(array $passedArgs)
{
$isSync = $skipProtected = false;
if (!empty($this->request->header('misp-version'))) {
$isSync = true;
if (version_compare($this->request->header('misp-version'), '2.4.156') < 0) {
$skipProtected = true;
}
}
// We do not want to allow instances to pull our data that can't make sense of protected mode events
$skipProtected = (
!empty($this->request->header('misp-version')) &&
version_compare($this->request->header('misp-version'), '2.4.156') < 0
);
$fieldNames = $this->Event->schema();
$minimal = !empty($passedArgs['searchminimal']) || !empty($passedArgs['minimal']);
if ($minimal) {
$rules = [
'recursive' => -1,
'fields' => array('id', 'timestamp', 'sighting_timestamp', 'published', 'uuid', 'protected'),
'contain' => array('Orgc.uuid', 'CryptographicKey.fingerprint'),
'contain' => array('Orgc.uuid'),
];
} else {
// Remove user ID from fetched fields
unset($fieldNames['user_id']);
$rules = [
'contain' => ['EventTag', 'CryptographicKey.fingerprint'],
'contain' => ['EventTag'],
'fields' => array_keys($fieldNames),
];
}
@ -801,6 +800,9 @@ class EventsController extends AppController
if (isset($this->paginate['conditions'])) {
$rules['conditions'] = $this->paginate['conditions'];
}
if ($skipProtected) {
$rules['conditions']['Event.protected'] = 0;
}
$paginationRules = array('page', 'limit', 'sort', 'direction', 'order');
foreach ($paginationRules as $paginationRule) {
if (isset($passedArgs[$paginationRule])) {
@ -837,12 +839,12 @@ class EventsController extends AppController
$events = $absolute_total === 0 ? [] : $this->Event->find('all', $rules);
}
$isCsvResponse = $this->response->type() === 'text/csv';
try {
$instanceFingerprint = $this->Event->CryptographicKey->ingestInstanceKey();
} catch (Exception $e) {
$instanceFingerprint = null;
}
$protectedEventsByInstanceKey = $this->Event->CryptographicKey->protectedEventsByInstanceKey($events);
$protectedEventsByInstanceKey = array_flip($protectedEventsByInstanceKey);
if (!$minimal) {
// Collect all tag IDs that are events
$tagIds = [];
@ -889,19 +891,8 @@ class EventsController extends AppController
$orgIds[$event['Event']['org_id']] = true;
$orgIds[$event['Event']['orgc_id']] = true;
$sharingGroupIds[$event['Event']['sharing_group_id']] = true;
if ($event['Event']['protected']) {
if ($skipProtected) {
unset($events[$k]);
continue;
}
foreach ($event['CryptographicKey'] as $cryptoKey) {
if ($instanceFingerprint === $cryptoKey['fingerprint']) {
continue 2;
}
}
if ($event['Event']['protected'] && !isset($protectedEventsByInstanceKey[$event['Event']['id']])) {
unset($events[$k]);
continue;
}
}
$events = array_values($events);
@ -946,26 +937,9 @@ class EventsController extends AppController
if ($this->response->type() === 'application/xml') {
$events = array('Event' => $events);
}
} else {
// We do not want to allow instances to pull our data that can't make sense of protected mode events
$skipProtected = (
!empty($this->request->header('misp-version')) &&
version_compare($this->request->header('misp-version'), '2.4.156') < 0
);
} else { // minimal
foreach ($events as $key => $event) {
if ($event['Event']['protected']) {
if ($skipProtected) {
unset($events[$key]);
continue;
}
foreach ($event['CryptographicKey'] as $cryptoKey) {
if ($instanceFingerprint === $cryptoKey['fingerprint']) {
$event['Event']['orgc_uuid'] = $event['Orgc']['uuid'];
unset($event['Event']['protected']);
$events[$key] = $event['Event'];
continue 2;
}
}
if ($event['Event']['protected'] && !isset($protectedEventsByInstanceKey[$event['Event']['id']])) {
unset($events[$key]);
continue;
}

153
app/Model/CryptographicKey.php
View File

@ -27,57 +27,46 @@ class CryptographicKey extends AppModel
ERROR_WRONG_KEY = 'Wrong key',
ERROR_INVALID_KEY = 'Invalid key';
public $validTypes = [
const VALID_TYPES = [
'pgp'
];
public $error = false;
public $validate = [];
public $validate = [
'uuid' => [
'uuid' => [
'rule' => 'uuid',
'message' => 'Please provide a valid RFC 4122 UUID',
],
],
'type' => [
'rule' => ['inList', self::VALID_TYPES],
'message' => 'Invalid key type',
'required' => 'create'
],
'key_data' => [
'notBlankKey' => [
'rule' => 'notBlank',
'message' => 'No key data received.',
'required' => 'create'
],
'validKey' => [
'rule' => 'validateKey',
'message' => 'Invalid key.',
'required' => 'create'
],
'uniqueKeyForElement' => [
'rule' => 'uniqueKeyForElement',
'message' => 'This key is already assigned to the target.',
'required' => 'create'
]
]
];
/** @var CryptGpgExtended|null */
private $gpg;
public function __construct($id = false, $table = null, $ds = null)
{
parent::__construct($id, $table, $ds);
try {
$this->gpg = GpgTool::initializeGpg();
} catch (Exception $e) {
$this->gpg = null;
}
$this->validate = [
'uuid' => [
'uuid' => [
'rule' => 'uuid',
'message' => 'Please provide a valid RFC 4122 UUID',
],
],
'type' => [
'rule' => ['inList', $this->validTypes],
'message' => __('Invalid key type'),
'required' => 'create'
],
'key_data' => [
'notBlankKey' => [
'rule' => 'notBlank',
'message' => __('No key data received.'),
'required' => 'create'
],
'validKey' => [
'rule' => 'validateKey',
'message' => __('Invalid key.'),
'required' => 'create'
],
'uniqueKeyForElement' => [
'rule' => 'uniqueKeyForElement',
'message' => __('This key is already assigned to the target.'),
'required' => 'create'
]
]
];
}
public function beforeSave($options = array())
{
$this->data['CryptographicKey']['timestamp'] = time();
@ -97,20 +86,17 @@ class CryptographicKey extends AppModel
{
// If instance key is stored just in GPG homedir, use that key.
if (Configure::read('MISP.download_gpg_from_homedir')) {
if (!$this->gpg) {
throw new Exception("Could not initiate GPG");
}
/** @var Crypt_GPG_Key[] $keys */
$keys = $this->gpg->getKeys(Configure::read('GnuPG.email'));
$keys = $this->getGpg()->getKeys(Configure::read('GnuPG.email'));
if (empty($keys)) {
return false;
}
$this->gpg->addSignKey($keys[0], Configure::read('GnuPG.password'));
$this->getGpg()->addSignKey($keys[0], Configure::read('GnuPG.password'));
return $keys[0]->getPrimaryKey()->getFingerprint();
}
try {
$redis = $this->setupRedisWithException();
$redis = RedisTool::init();
} catch (Exception $e) {
$redis = false;
}
@ -124,33 +110,63 @@ class CryptographicKey extends AppModel
if (empty($fingerprint)) {
$file = new File(APP . '/webroot/gpg.asc');
$instanceKey = $file->read();
if (!$this->gpg) {
throw new MethodNotAllowedException("Could not initiate GPG");
}
try {
$this->gpg->importKey($instanceKey);
$this->getGpg()->importKey($instanceKey);
} catch (Crypt_GPG_NoDataException $e) {
throw new MethodNotAllowedException("Could not import the instance key.");
}
$fingerprint = $this->gpg->getFingerprint(Configure::read('GnuPG.email'));
$fingerprint = $this->getGpg()->getFingerprint(Configure::read('GnuPG.email'));
if ($redis) {
$redis->setEx($redisKey, 300, $fingerprint);
}
}
if (!$this->gpg) {
throw new MethodNotAllowedException("Could not initiate GPG");
}
try {
$this->gpg->addSignKey(Configure::read('GnuPG.email'), Configure::read('GnuPG.password'));
$this->getGpg()->addSignKey(Configure::read('GnuPG.email'), Configure::read('GnuPG.password'));
} catch (Exception $e) {
throw new NotFoundException('Could not add signing key.');
}
return $fingerprint;
}
/**
* Check if given events are protected by instance key, returns array of Event IDs
* @param array $events
* @return array Event ID that is protected in key
*/
public function protectedEventsByInstanceKey(array $events)
{
$eventIds = [];
foreach ($events as $event) {
if ($event['Event']['protected']) {
$eventIds[] = $event['Event']['id'];
}
}
if (empty($eventIds)) {
return [];
}
try {
$instanceKey = $this->ingestInstanceKey();
} catch (Exception $e) {
// could not fetch instance key
return [];
}
return $this->find('column', [
'conditions' => [
'CryptographicKey.parent_type' => 'Event',
'CryptographicKey.parent_id' => $eventIds,
'CryptographicKey.fingerprint' => $instanceKey,
],
'fields' => ['CryptographicKey.parent_id'],
'recursive' => -1,
]);
}
/**
* @param string $data
* @return false|string
* @return false|string Signature
* @throws Crypt_GPG_BadPassphraseException
* @throws Crypt_GPG_Exception
* @throws Crypt_GPG_KeyNotFoundException
@ -161,7 +177,7 @@ class CryptographicKey extends AppModel
return false;
}
$data = preg_replace("/\s+/", "", $data);
$signature = $this->gpg->sign($data, Crypt_GPG::SIGN_MODE_DETACHED, Crypt_GPG::ARMOR_BINARY);
$signature = $this->getGpg()->sign($data, Crypt_GPG::SIGN_MODE_DETACHED, Crypt_GPG::ARMOR_BINARY);
return $signature;
}
@ -181,7 +197,7 @@ class CryptographicKey extends AppModel
}
$data = preg_replace("/\s+/", "", $data);
try {
$verifiedSignature = $this->gpg->verify($data, $signature);
$verifiedSignature = $this->getGpg()->verify($data, $signature);
} catch (Exception $e) {
$this->error = self::ERROR_WRONG_KEY;
return false;
@ -219,7 +235,7 @@ class CryptographicKey extends AppModel
private function __extractPGPKeyData($data)
{
try {
$gpgTool = new GpgTool($this->gpg);
$gpgTool = new GpgTool($this->getGpg());
} catch (Exception $e) {
$this->logException("GPG couldn't be initialized, GPG encryption and signing will be not available.", $e, LOG_NOTICE);
return false;
@ -345,4 +361,19 @@ class CryptographicKey extends AppModel
$this->deleteAll(['CryptographicKey.id' => $toRemove]);
$this->loadLog()->createLogEntry($user, 'updateCryptoKeys', $cryptographicKey['parent_type'], $cryptographicKey['parent_id'], $message);
}
/**
* Lazy load GPG
* @return CryptGpgExtended|null
* @throws Exception
*/
private function getGpg()
{
if ($this->gpg) {
return $this->gpg;
}
$this->gpg = GpgTool::initializeGpg();
return $this->gpg;
}
}