MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #8720 from JakubOnderka/sightings-rest-search 

Sightings rest search
pull/8719/head
Jakub Onderka 2022-11-01 11:34:07 +01:00 committed by GitHub
parent acd6732613 a27e036c5b
commit d182d99fde
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 436 additions and 204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
app/Controller/Component/RestResponseComponent.php
View File

@ -261,7 +261,7 @@ class RestResponseComponent extends Component
'restSearch' => array(
'description' => "Search MISP sightings using a list of filter parameters and return the data in the JSON format. The search is available on an event, attribute or instance level, just select the scope via the URL (/sighting/restSearch/event vs /sighting/restSearch/attribute vs /sighting/restSearch/). id or uuid MUST be provided if context is set.",
'mandatory' => array('returnFormat'),
'optional' => array('id', 'uuid', 'type', 'from', 'to', 'last', 'org_id', 'source', 'includeAttribute', 'includeEvent'),
'optional' => array('id', 'uuid', 'type', 'from', 'to', 'last', 'org_id', 'source', 'includeAttribute', 'includeEvent', 'includeUuid'),
'params' => array('context')
),
),
@ -1256,6 +1256,12 @@ class RestResponseComponent extends Component
'values' => array(1 => 'True', 0 => 'False' ),
'help' => 'Include all enabled decaying score'
),
'includeUuid' => array(
'input' => 'radio',
'type' => 'integer',
'values' => array(1 => 'True', 0 => 'False'),
'help' => __('Include matching event and attribute UUID to in the response'),
),
'includeEvent' => array(
'input' => 'radio',
'type' => 'integer',

64
app/Controller/EventsController.php
View File

@ -768,26 +768,25 @@ class EventsController extends AppController
*/
private function __indexRestResponse(array $passedArgs)
{
$isSync = $skipProtected = false;
if (!empty($this->request->header('misp-version'))) {
$isSync = true;
if (version_compare($this->request->header('misp-version'), '2.4.156') < 0) {
$skipProtected = true;
}
}
// We do not want to allow instances to pull our data that can't make sense of protected mode events
$skipProtected = (
!empty($this->request->header('misp-version')) &&
version_compare($this->request->header('misp-version'), '2.4.156') < 0
);
$fieldNames = $this->Event->schema();
$minimal = !empty($passedArgs['searchminimal']) || !empty($passedArgs['minimal']);
if ($minimal) {
$rules = [
'recursive' => -1,
'fields' => array('id', 'timestamp', 'sighting_timestamp', 'published', 'uuid', 'protected'),
'contain' => array('Orgc.uuid', 'CryptographicKey.fingerprint'),
'contain' => array('Orgc.uuid'),
];
} else {
// Remove user ID from fetched fields
unset($fieldNames['user_id']);
$rules = [
'contain' => ['EventTag', 'CryptographicKey.fingerprint'],
'contain' => ['EventTag'],
'fields' => array_keys($fieldNames),
];
}
@ -801,6 +800,9 @@ class EventsController extends AppController
if (isset($this->paginate['conditions'])) {
$rules['conditions'] = $this->paginate['conditions'];
}
if ($skipProtected) {
$rules['conditions']['Event.protected'] = 0;
}
$paginationRules = array('page', 'limit', 'sort', 'direction', 'order');
foreach ($paginationRules as $paginationRule) {
if (isset($passedArgs[$paginationRule])) {
@ -837,12 +839,12 @@ class EventsController extends AppController
$events = $absolute_total === 0 ? [] : $this->Event->find('all', $rules);
}
$isCsvResponse = $this->response->type() === 'text/csv';
try {
$instanceFingerprint = $this->Event->CryptographicKey->ingestInstanceKey();
} catch (Exception $e) {
$instanceFingerprint = null;
}
$protectedEventsByInstanceKey = $this->Event->CryptographicKey->protectedEventsByInstanceKey($events);
$protectedEventsByInstanceKey = array_flip($protectedEventsByInstanceKey);
if (!$minimal) {
// Collect all tag IDs that are events
$tagIds = [];
@ -889,19 +891,8 @@ class EventsController extends AppController
$orgIds[$event['Event']['org_id']] = true;
$orgIds[$event['Event']['orgc_id']] = true;
$sharingGroupIds[$event['Event']['sharing_group_id']] = true;
if ($event['Event']['protected']) {
if ($skipProtected) {
unset($events[$k]);
continue;
}
foreach ($event['CryptographicKey'] as $cryptoKey) {
if ($instanceFingerprint === $cryptoKey['fingerprint']) {
continue 2;
}
}
if ($event['Event']['protected'] && !isset($protectedEventsByInstanceKey[$event['Event']['id']])) {
unset($events[$k]);
continue;
}
}
$events = array_values($events);
@ -946,26 +937,9 @@ class EventsController extends AppController
if ($this->response->type() === 'application/xml') {
$events = array('Event' => $events);
}
} else {
// We do not want to allow instances to pull our data that can't make sense of protected mode events
$skipProtected = (
!empty($this->request->header('misp-version')) &&
version_compare($this->request->header('misp-version'), '2.4.156') < 0
);
} else { // minimal
foreach ($events as $key => $event) {
if ($event['Event']['protected']) {
if ($skipProtected) {
unset($events[$key]);
continue;
}
foreach ($event['CryptographicKey'] as $cryptoKey) {
if ($instanceFingerprint === $cryptoKey['fingerprint']) {
$event['Event']['orgc_uuid'] = $event['Orgc']['uuid'];
unset($event['Event']['protected']);
$events[$key] = $event['Event'];
continue 2;
}
}
if ($event['Event']['protected'] && !isset($protectedEventsByInstanceKey[$event['Event']['id']])) {
unset($events[$key]);
continue;
}

23
app/Lib/Tools/ServerSyncTool.php
View File

@ -13,7 +13,8 @@ class ServerSyncTool
FEATURE_POST_TEST = 'post_test',
FEATURE_EDIT_OF_GALAXY_CLUSTER = 'edit_of_galaxy_cluster',
PERM_SYNC = 'perm_sync',
PERM_GALAXY_EDITOR = 'perm_galaxy_editor';
PERM_GALAXY_EDITOR = 'perm_galaxy_editor',
FEATURE_SIGHTING_REST_SEARCH = 'sighting_rest';
/** @var array */
private $server;
@ -227,6 +228,23 @@ class ServerSyncTool
return $this->get($url);
}
/**
* @param array $eventUuids
* @return array
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
* @throws JsonException
*/
public function fetchSightingsForEvents(array $eventUuids)
{
return $this->post('/sightings/restSearch/event', [
'returnFormat' => 'json',
'last' => 0, // fetch all
'includeUuid' => true,
'uuid' => $eventUuids,
])->json()['response'];
}
/**
* @param array $event
* @param array $sightingUuids
@ -388,6 +406,9 @@ class ServerSyncTool
return isset($info['perm_sync']) && $info['perm_sync'];
case self::PERM_GALAXY_EDITOR:
return isset($info['perm_galaxy_editor']) && $info['perm_galaxy_editor'];
case self::FEATURE_SIGHTING_REST_SEARCH:
$version = explode('.', $info['version']);
return $version[0] == 2 && $version[1] == 4 && $version[2] > 164;
default:
throw new InvalidArgumentException("Invalid flag `$flag` provided");
}

15
app/Model/AppModel.php
View File

@ -2983,7 +2983,15 @@ class AppModel extends Model
return self::$loadedBackgroundJobsTool;
}
// generate a generic subquery - options needs to include conditions
/**
* Generate a generic subquery - options needs to include conditions
*
* @param AppModel $model
* @param array $options
* @param string $lookupKey
* @param bool $negation
* @return string[]
*/
protected function subQueryGenerator(AppModel $model, array $options, $lookupKey, $negation = false)
{
$defaults = array(
@ -3012,10 +3020,7 @@ class AppModel extends Model
} else {
$subQuery = $lookupKey . ' IN (' . $subQuery . ') ';
}
$conditions = array(
$db->expression($subQuery)->value
);
return $conditions;
return [$subQuery];
}
// start a benchmark run for the given bench name

153
app/Model/CryptographicKey.php
View File

@ -27,57 +27,46 @@ class CryptographicKey extends AppModel
ERROR_WRONG_KEY = 'Wrong key',
ERROR_INVALID_KEY = 'Invalid key';
public $validTypes = [
const VALID_TYPES = [
'pgp'
];
public $error = false;
public $validate = [];
public $validate = [
'uuid' => [
'uuid' => [
'rule' => 'uuid',
'message' => 'Please provide a valid RFC 4122 UUID',
],
],
'type' => [
'rule' => ['inList', self::VALID_TYPES],
'message' => 'Invalid key type',
'required' => 'create'
],
'key_data' => [
'notBlankKey' => [
'rule' => 'notBlank',
'message' => 'No key data received.',
'required' => 'create'
],
'validKey' => [
'rule' => 'validateKey',
'message' => 'Invalid key.',
'required' => 'create'
],
'uniqueKeyForElement' => [
'rule' => 'uniqueKeyForElement',
'message' => 'This key is already assigned to the target.',
'required' => 'create'
]
]
];
/** @var CryptGpgExtended|null */
private $gpg;
public function __construct($id = false, $table = null, $ds = null)
{
parent::__construct($id, $table, $ds);
try {
$this->gpg = GpgTool::initializeGpg();
} catch (Exception $e) {
$this->gpg = null;
}
$this->validate = [
'uuid' => [
'uuid' => [
'rule' => 'uuid',
'message' => 'Please provide a valid RFC 4122 UUID',
],
],
'type' => [
'rule' => ['inList', $this->validTypes],
'message' => __('Invalid key type'),
'required' => 'create'
],
'key_data' => [
'notBlankKey' => [
'rule' => 'notBlank',
'message' => __('No key data received.'),
'required' => 'create'
],
'validKey' => [
'rule' => 'validateKey',
'message' => __('Invalid key.'),
'required' => 'create'
],
'uniqueKeyForElement' => [
'rule' => 'uniqueKeyForElement',
'message' => __('This key is already assigned to the target.'),
'required' => 'create'
]
]
];
}
public function beforeSave($options = array())
{
$this->data['CryptographicKey']['timestamp'] = time();
@ -97,20 +86,17 @@ class CryptographicKey extends AppModel
{
// If instance key is stored just in GPG homedir, use that key.
if (Configure::read('MISP.download_gpg_from_homedir')) {
if (!$this->gpg) {
throw new Exception("Could not initiate GPG");
}
/** @var Crypt_GPG_Key[] $keys */
$keys = $this->gpg->getKeys(Configure::read('GnuPG.email'));
$keys = $this->getGpg()->getKeys(Configure::read('GnuPG.email'));
if (empty($keys)) {
return false;
}
$this->gpg->addSignKey($keys[0], Configure::read('GnuPG.password'));
$this->getGpg()->addSignKey($keys[0], Configure::read('GnuPG.password'));
return $keys[0]->getPrimaryKey()->getFingerprint();
}
try {
$redis = $this->setupRedisWithException();
$redis = RedisTool::init();
} catch (Exception $e) {
$redis = false;
}
@ -124,33 +110,63 @@ class CryptographicKey extends AppModel
if (empty($fingerprint)) {
$file = new File(APP . '/webroot/gpg.asc');
$instanceKey = $file->read();
if (!$this->gpg) {
throw new MethodNotAllowedException("Could not initiate GPG");
}
try {
$this->gpg->importKey($instanceKey);
$this->getGpg()->importKey($instanceKey);
} catch (Crypt_GPG_NoDataException $e) {
throw new MethodNotAllowedException("Could not import the instance key.");
}
$fingerprint = $this->gpg->getFingerprint(Configure::read('GnuPG.email'));
$fingerprint = $this->getGpg()->getFingerprint(Configure::read('GnuPG.email'));
if ($redis) {
$redis->setEx($redisKey, 300, $fingerprint);
}
}
if (!$this->gpg) {
throw new MethodNotAllowedException("Could not initiate GPG");
}
try {
$this->gpg->addSignKey(Configure::read('GnuPG.email'), Configure::read('GnuPG.password'));
$this->getGpg()->addSignKey(Configure::read('GnuPG.email'), Configure::read('GnuPG.password'));
} catch (Exception $e) {
throw new NotFoundException('Could not add signing key.');
}
return $fingerprint;
}
/**
* Check if given events are protected by instance key, returns array of Event IDs
* @param array $events
* @return array Event ID that is protected in key
*/
public function protectedEventsByInstanceKey(array $events)
{
$eventIds = [];
foreach ($events as $event) {
if ($event['Event']['protected']) {
$eventIds[] = $event['Event']['id'];
}
}
if (empty($eventIds)) {
return [];
}
try {
$instanceKey = $this->ingestInstanceKey();
} catch (Exception $e) {
// could not fetch instance key
return [];
}
return $this->find('column', [
'conditions' => [
'CryptographicKey.parent_type' => 'Event',
'CryptographicKey.parent_id' => $eventIds,
'CryptographicKey.fingerprint' => $instanceKey,
],
'fields' => ['CryptographicKey.parent_id'],
'recursive' => -1,
]);
}
/**
* @param string $data
* @return false|string
* @return false|string Signature
* @throws Crypt_GPG_BadPassphraseException
* @throws Crypt_GPG_Exception
* @throws Crypt_GPG_KeyNotFoundException
@ -161,7 +177,7 @@ class CryptographicKey extends AppModel
return false;
}
$data = preg_replace("/\s+/", "", $data);
$signature = $this->gpg->sign($data, Crypt_GPG::SIGN_MODE_DETACHED, Crypt_GPG::ARMOR_BINARY);
$signature = $this->getGpg()->sign($data, Crypt_GPG::SIGN_MODE_DETACHED, Crypt_GPG::ARMOR_BINARY);
return $signature;
}
@ -181,7 +197,7 @@ class CryptographicKey extends AppModel
}
$data = preg_replace("/\s+/", "", $data);
try {
$verifiedSignature = $this->gpg->verify($data, $signature);
$verifiedSignature = $this->getGpg()->verify($data, $signature);
} catch (Exception $e) {
$this->error = self::ERROR_WRONG_KEY;
return false;
@ -219,7 +235,7 @@ class CryptographicKey extends AppModel
private function __extractPGPKeyData($data)
{
try {
$gpgTool = new GpgTool($this->gpg);
$gpgTool = new GpgTool($this->getGpg());
} catch (Exception $e) {
$this->logException("GPG couldn't be initialized, GPG encryption and signing will be not available.", $e, LOG_NOTICE);
return false;
@ -345,4 +361,19 @@ class CryptographicKey extends AppModel
$this->deleteAll(['CryptographicKey.id' => $toRemove]);
$this->loadLog()->createLogEntry($user, 'updateCryptoKeys', $cryptographicKey['parent_type'], $cryptographicKey['parent_id'], $message);
}
/**
* Lazy load GPG
* @return CryptGpgExtended|null
* @throws Exception
*/
private function getGpg()
{
if ($this->gpg) {
return $this->gpg;
}
$this->gpg = GpgTool::initializeGpg();
return $this->gpg;
}
}

8
app/Model/Server.php
View File

@ -247,6 +247,14 @@ class Server extends AppModel
if (!empty($server['authkey']) && strlen($server['authkey']) === 40) {
$server['authkey'] = EncryptedValue::encryptIfEnabled($server['authkey']);
}
try {
// Clean caches when remote server setting changed
RedisTool::unlink(RedisTool::init(), ["misp:fetched_sightings:{$this->id}", "misp:event_index:{$this->id}"]);
} catch (Exception $e) {
// ignore
}
return true;
}

364
app/Model/Sighting.php
View File

@ -80,13 +80,7 @@ class Sighting extends AppModel
$pubToZmq = $this->pubToZmq('sighting');
$kafkaTopic = $this->kafkaTopic('sighting');
if ($pubToZmq || $kafkaTopic) {
$user = array(
'org_id' => -1,
'Role' => array(
'perm_site_admin' => 1
)
);
$sighting = $this->getSighting($this->id, $user);
$sighting = $this->getSighting($this->id);
if ($pubToZmq) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->sighting_save($sighting, 'add');
@ -104,13 +98,7 @@ class Sighting extends AppModel
$pubToZmq = $this->pubToZmq('sighting');
$kafkaTopic = $this->kafkaTopic('sighting');
if ($pubToZmq || $kafkaTopic) {
$user = array(
'org_id' => -1,
'Role' => array(
'perm_site_admin' => 1
)
);
$sighting = $this->getSighting($this->id, $user);
$sighting = $this->getSighting($this->id);
if ($pubToZmq) {
$pubSubTool = $this->getPubSubTool();
$pubSubTool->sighting_save($sighting, 'delete');
@ -187,12 +175,112 @@ class Sighting extends AppModel
}
/**
* @param int $id
* Fetch sightings with proper ACL checks
*
* @param array $user
* @param bool $withEvent
* @param array $ids Sightings IDs
* @param bool $includeEvent
* @param bool $includeAttribute
* @param bool $includeUuid Add attribute and event UUID to sighting
* @return array
*/
public function getSighting($id, array $user, $withEvent = true)
private function getSightings(array $user, array $ids, $includeEvent = true, $includeAttribute = false, $includeUuid = false)
{
$eventFields = $includeEvent ? ['Event.id', 'Event.uuid', 'Event.orgc_id', 'Event.org_id', 'Event.info'] : ['Event.org_id'];
if ($includeUuid && !$includeEvent) {
$eventFields[] = 'Event.uuid';
}
$attributeFields = $includeAttribute ? ['Attribute.id', 'Attribute.value','Attribute.uuid', 'Attribute.type', 'Attribute.category', 'Attribute.to_ids'] : ['Attribute.id', 'Attribute.value'];
if ($includeUuid && !$includeAttribute) {
$attributeFields[] = 'Attribute.uuid';
}
// Fetch all attributes that are connected to sightings and user can see them
$attributeConditions = $this->Attribute->buildConditions($user);
$subQueryOptions = [
'fields' => ['DISTINCT Sighting.attribute_id'],
'conditions' => ['Sighting.id' => $ids],
];
$attributeConditions[] = $this->subQueryGenerator($this, $subQueryOptions, 'Attribute.id');
$attributes = $this->Attribute->find('all', [
'recursive' => -1,
'conditions' => $attributeConditions,
'contain' => [
'Event' => [
'fields' => $eventFields,
],
'Object',
],
'fields' => $attributeFields,
'order' => [],
]);
if (empty($attributes)) {
return [];
}
// Create conditions for fetching just sightings that user can see according to sightings policy
$conditions = $this->createConditionsByAttributes($user, $attributes);
if (empty($conditions)) {
return [];
}
$conditions['Sighting.id'] = $ids;
$sightings = $this->find('all', [
'recursive' => -1,
'conditions' => $conditions,
'order' => 'Sighting.id',
]);
if (empty($sightings)) {
return [];
}
$attributesById = [];
foreach ($attributes as $attribute) {
$attributesById[$attribute['Attribute']['id']] = $attribute;
}
$anonymise = Configure::read('Plugin.Sightings_anonymise');
$results = [];
foreach ($sightings as $sighting) {
$sightingAttribute = $attributesById[$sighting['Sighting']['attribute_id']];
if ($anonymise && $sighting['Sighting']['org_id'] != $user['org_id']) {
unset($sighting['Sighting']['org_id']);
}
// rearrange it to match the event format of fetchevent
$result = $sighting['Sighting'];
$result['value'] = $sightingAttribute['Attribute']['value'];
if ($includeUuid) {
$result['attribute_uuid'] = $sightingAttribute['Attribute']['uuid'];
$result['event_uuid'] = $sightingAttribute['Event']['uuid'];
}
if ($includeAttribute) {
$result['Attribute'] = $sightingAttribute['Attribute'];
}
if ($includeEvent) {
$sightingAttribute['Event']['Orgc']['name'] = $this->getOrganisationById($sightingAttribute['Event']['orgc_id'])['name'];
$result['Event'] = $sightingAttribute['Event'];
}
if (isset($result['org_id']) && $result['org_id'] != 0) {
$result['Organisation'] = $this->getOrganisationById($result['org_id']);
}
$results[] = ['Sighting' => $result];
}
return $results;
}
/**
* Return sighting without ACL checks
*
* @param int $id
* @return array
*/
public function getSighting($id)
{
$sighting = $this->find('first', array(
'recursive' => -1,
@ -201,7 +289,7 @@ class Sighting extends AppModel
'fields' => array('Attribute.value', 'Attribute.id', 'Attribute.uuid', 'Attribute.type', 'Attribute.category', 'Attribute.to_ids')
),
'Event' => array(
'fields' => $withEvent ? ['Event.id', 'Event.uuid', 'Event.orgc_id', 'Event.org_id', 'Event.info'] : ['Event.org_id'],
'fields' => ['Event.id', 'Event.uuid', 'Event.orgc_id', 'Event.org_id', 'Event.info'],
)
),
'conditions' => array('Sighting.id' => $id)
@ -210,44 +298,14 @@ class Sighting extends AppModel
return array();
}
$ownEvent = $user['Role']['perm_site_admin'] || $sighting['Event']['org_id'] == $user['org_id'];
if (!$ownEvent) {
$sightingPolicy = $this->sightingsPolicy();
// if sighting policy == 0 then return false if the sighting doesn't belong to the user
if ($sightingPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
if ($sighting['Sighting']['org_id'] != $user['org_id']) {
return array();
}
}
// if sighting policy == 1, the user can only see the sighting if they've sighted something in the event once
else if ($sightingPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) {
if (!$this->isReporter($sighting['Sighting']['event_id'], $user['org_id'])) {
return array();
}
}
else if ($sightingPolicy === self::SIGHTING_POLICY_HOST_ORG) {
if ($sighting['Sighting']['org_id'] != $user['org_id'] || $sighting['Sighting']['org_id'] != Configure::read('MISP.host_org_id')) {
return array();
}
}
}
// Put event organisation name from cache
if ($withEvent) {
$sighting['Event']['Orgc']['name'] = $this->getOrganisationById($sighting['Event']['orgc_id'])['name'];
}
$sighting['Event']['Orgc']['name'] = $this->getOrganisationById($sighting['Event']['orgc_id'])['name'];
$anonymise = Configure::read('Plugin.Sightings_anonymise');
if ($anonymise && $sighting['Sighting']['org_id'] != $user['org_id']) {
unset($sighting['Sighting']['org_id']);
}
// rearrange it to match the event format of fetchevent
$result = array(
'Sighting' => $sighting['Sighting']
);
if ($withEvent) {
$result['Sighting']['Event'] = $sighting['Event'];
}
$result['Sighting']['Event'] = $sighting['Event'];
$result['Sighting']['Attribute'] = $sighting['Attribute'];
return $result;
}
@ -306,28 +364,44 @@ class Sighting extends AppModel
return ['data' => [], 'csv' => []];
}
$conditions = $this->createConditionsByAttributes($user, $attributes);
$groupedSightings = $this->fetchGroupedSightings($conditions, $user);
return $this->generateStatistics($groupedSightings, $csvWithFalsePositive);
}
/**
* @param array $user
* @param array $attributes Attributes with `Attribute.id`, `Event.id` and `Event.org_id` fields
* @return array
*/
private function createConditionsByAttributes(array $user, array $attributes)
{
$sightingsPolicy = $this->sightingsPolicy();
if ($sightingsPolicy === self::SIGHTING_POLICY_EVERYONE || $user['Role']['perm_site_admin']) {
return ['Sighting.attribute_id' => array_column(array_column($attributes, 'Attribute'), 'id')];
}
$hostOrgId = Configure::read('MISP.host_org_id');
$userOrgId = $user['org_id'];
$conditions = [];
foreach ($attributes as $attribute) {
$attributeConditions = ['Sighting.attribute_id' => $attribute['Attribute']['id']];
$ownEvent = $user['Role']['perm_site_admin'] || $attribute['Event']['org_id'] == $user['org_id'];
$ownEvent = $attribute['Event']['org_id'] == $userOrgId;
if (!$ownEvent) {
if ($sightingsPolicy === self::SIGHTING_POLICY_EVENT_OWNER) {
$attributeConditions['Sighting.org_id'] = $user['org_id'];
$attributeConditions['Sighting.org_id'] = $userOrgId;
} else if ($sightingsPolicy === self::SIGHTING_POLICY_SIGHTING_REPORTER) {
if (!$this->isReporter($attribute['Event']['id'], $user['org_id'])) {
if (!$this->isReporter($attribute['Event']['id'], $userOrgId)) {
continue; // skip attribute
}
} else if ($sightingsPolicy === self::SIGHTING_POLICY_HOST_ORG) {
$attributeConditions['Sighting.org_id'] = [$user['org_id'], Configure::read('MISP.host_org_id')];
$attributeConditions['Sighting.org_id'] = [$userOrgId, $hostOrgId];
}
}
$conditions['OR'][] = $attributeConditions;
}
$groupedSightings = $this->fetchGroupedSightings($conditions, $user);
return $this->generateStatistics($groupedSightings, $csvWithFalsePositive);
return $conditions;
}
/**
@ -897,12 +971,12 @@ class Sighting extends AppModel
* @return TmpFileTool
* @throws Exception
*/
public function restSearch($user, $returnFormat, $filters)
public function restSearch(array $user, $returnFormat, $filters)
{
$allowedContext = array('event', 'attribute');
// validate context
if (isset($filters['context']) && !in_array($filters['context'], $allowedContext, true)) {
throw new MethodNotAllowedException(__('Invalid context.'));
throw new MethodNotAllowedException(__('Invalid context %s.', $filters['context']));
}
// ensure that an id or uuid is provided if context is set
if (!empty($filters['context']) && !(isset($filters['id']) || isset($filters['uuid'])) ) {
@ -935,7 +1009,6 @@ class Sighting extends AppModel
}
if (isset($filters['org_id'])) {
$this->Organisation = ClassRegistry::init('Organisation');
if (!is_array($filters['org_id'])) {
$filters['org_id'] = array($filters['org_id']);
}
@ -961,6 +1034,16 @@ class Sighting extends AppModel
}
if (!empty($filters['id'])) {
if (is_array($filters['id'])) {
foreach ($filters['id'] as $id) {
if (!is_int($id) || !is_numeric($id)) {
throw new BadRequestException("Invalid ID `$id` provided.");
}
}
} else if (!is_int($filters['id']) || !is_numeric($filters['id'])) {
throw new BadRequestException("Invalid ID `{$filters['id']}` provided.");
}
if ($filters['context'] === 'attribute') {
$conditions['Sighting.attribute_id'] = $filters['id'];
} elseif ($filters['context'] === 'event') {
@ -978,15 +1061,10 @@ class Sighting extends AppModel
}
}
// fetch sightings matching the query
$sightingIds = $this->find('column', [
'conditions' => $conditions,
'fields' => ['Sighting.id'],
'contain' => $contain,
]);
$includeAttribute = isset($filters['includeAttribute']) && $filters['includeAttribute'];
$includeEvent = isset($filters['includeEvent']) && $filters['includeEvent'];
$includeUuid = isset($filters['includeUuid']) && $filters['includeUuid'];
$requestedAttributes = ['id', 'attribute_id', 'event_id', 'org_id', 'date_sighting', 'uuid', 'source', 'type'];
if ($includeAttribute) {
$requestedAttributes = array_merge($requestedAttributes, ['attribute_uuid', 'attribute_type', 'attribute_category', 'attribute_to_ids', 'attribute_value']);
@ -1008,14 +1086,18 @@ class Sighting extends AppModel
$tmpfile->write($exportTool->header($exportToolParams));
$separator = $exportTool->separator($exportToolParams);
foreach ($sightingIds as $sightingId) {
// apply ACL and sighting policies
$sighting = $this->getSighting($sightingId, $user, $includeEvent);
if (!empty($sighting)) {
$sighting['Sighting']['value'] = $sighting['Sighting']['Attribute']['value'];
if (!$includeAttribute) {
unset($sighting['Sighting']['Attribute']);
}
// fetch sightings matching the query without ACL checks
$sightingIds = $this->find('column', [
'conditions' => $conditions,
'fields' => ['Sighting.id'],
'contain' => $contain,
'order' => 'Sighting.id',
]);
foreach (array_chunk($sightingIds, 500) as $chunk) {
// fetch sightings with ACL checks and sighting policies
$sightings = $this->getSightings($user, $chunk, $includeEvent, $includeAttribute, $includeUuid);
foreach ($sightings as $sighting) {
$tmpfile->writeWithSeparator($exportTool->handler($sighting, $exportToolParams), $separator);
}
}
@ -1108,9 +1190,9 @@ class Sighting extends AppModel
$existingUuids = array_column($toSave, 'uuid');
$this->Event->publishSightingsRouter($event['Event']['id'], $user, $passAlong, $existingUuids);
return count($toSave);
} else {
return 0;
}
return 0;
}
/**
@ -1229,11 +1311,85 @@ class Sighting extends AppModel
$eventUuids = [];
foreach ($remoteEvents as $remoteEvent) {
if (isset($localEvents[$remoteEvent['uuid']]) && $localEvents[$remoteEvent['uuid']] < $remoteEvent['sighting_timestamp']) {
$eventUuids[] = $remoteEvent['uuid'];
$eventUuids[$remoteEvent['uuid']] = $remoteEvent['sighting_timestamp'];
}
}
unset($remoteEvents, $localEvents);
if (empty($eventUuids)) {
return 0;
}
$this->removeFetched($serverSync->serverId(), $eventUuids);
if (empty($eventUuids)) {
return 0;
}
if ($serverSync->isSupported(ServerSyncTool::FEATURE_SIGHTING_REST_SEARCH)) {
return $this->pullSightingNewWay($user, $eventUuids, $serverSync);
} else {
return $this->pullSightingOldWay($user, $eventUuids, $serverSync);
}
}
/**
* New way how to fetch sighting for events without fetching the whole event.
*
* @param array $user
* @param array $eventUuids With UUID in key and remote sighting_timestamp as value
* @param ServerSyncTool $serverSync
* @return int
* @throws Exception
*/
private function pullSightingNewWay(array $user, array $eventUuids, ServerSyncTool $serverSync)
{
$uuids = array_keys($eventUuids);
$saved = 0;
foreach (array_chunk($uuids, 100) as $chunk) {
try {
$sightings = $serverSync->fetchSightingsForEvents($chunk);
} catch (Exception $e) {
$this->logException("Failed downloading the sightings from {$serverSync->server()['Server']['name']}.", $e);
continue;
}
$sightingsToSave = [];
foreach ($sightings as $sighting) {
$sighting = $sighting['Sighting'];
$attributeUuid = $sighting['Attribute']['uuid'];
$eventUuid = $sighting['Event']['uuid'];
unset($sighting['Event'], $sighting['Attribute']);
$sighting['attribute_uuid'] = $attributeUuid;
$sightingsToSave[$eventUuid][] = $sighting;
}
$savedUuids = [];
foreach ($sightingsToSave as $eventUuid => $sightings) {
$savedForEvent = $this->bulkSaveSightings($eventUuid, $sightings, $user, $serverSync->serverId());
if ($savedForEvent) {
$saved += $savedForEvent;
$savedUuids[] = $eventUuid;
}
}
// Save to Redis that we fetched event sightings, that was not saved. This avoid fetching sightings for
// same event that has sightings not visible to user again and again.
foreach (array_diff($uuids, $savedUuids) as $notSavedUuid) {
$this->saveFetched($serverSync->serverId(), $notSavedUuid, $eventUuids[$notSavedUuid]);
}
}
return $saved;
}
/**
* @param array $user
* @param array $eventUuids With UUID in key and remote sighting_timestamp as value
* @param ServerSyncTool $serverSync
* @return int
* @throws Exception
*/
private function pullSightingOldWay(array $user, array $eventUuids, ServerSyncTool $serverSync)
{
$saved = 0;
// We don't need some of the event data, like correlations and event reports
$params = [
@ -1249,7 +1405,7 @@ class Sighting extends AppModel
];
// now process the $eventUuids to pull each of the events sequentially
// download each event and save sightings
foreach ($eventUuids as $eventUuid) {
foreach ($eventUuids as $eventUuid => $sightingTimestamp) {
try {
$event = $serverSync->fetchEvent($eventUuid, $params)->json();
} catch (Exception $e) {
@ -1279,12 +1435,45 @@ class Sighting extends AppModel
$result = $this->bulkSaveSightings($event['Event']['uuid'], $sightings, $user, $serverSync->serverId());
if (is_numeric($result)) {
$saved += $result;
} else {
$this->saveFetched($serverSync->serverId(), $eventUuid, $sightingTimestamp);
}
}
}
return $saved;
}
/**
* Remove from fetching events that was already fetched with the same sighting_timestamp
* @param int $serverId
* @param array $eventUuids
* @return void
* @throws RedisException
*/
private function removeFetched($serverId, array &$eventUuids)
{
$lastFetched = RedisTool::init()->hMGet("misp:fetched_sightings:$serverId", array_keys($eventUuids));
foreach ($lastFetched as $uuid => $savedTimestamp) {
if ($savedTimestamp == $eventUuids[$uuid]) {
unset($eventUuids[$uuid]); // event with the same sighting_timestamp was already fetched
}
}
}
/**
* @param int $serverId
* @param string $eventUuid
* @param int $sightingTimestamp
* @return void
* @throws RedisException
*/
private function saveFetched($serverId, $eventUuid, $sightingTimestamp)
{
$redis = RedisTool::init();
$redis->hSet("misp:fetched_sightings:$serverId", $eventUuid, $sightingTimestamp);
$redis->expire("misp:fetched_sightings:$serverId", 24 * 3600); // keep for one day
}
/**
* @return int Timestamp
*/
@ -1321,7 +1510,7 @@ class Sighting extends AppModel
}
return $this->Organisation->find('list', [
'fields' => ['Organisation.uuid', 'Organisation.id'],
'conditions' => ['Organisation.uuid' => array_unique(array_column($organisations, 'uuid'))],
'conditions' => ['Organisation.uuid' => array_unique(array_column($organisations, 'uuid'), SORT_REGULAR)],
]);
}
@ -1336,15 +1525,10 @@ class Sighting extends AppModel
*/
private function isReporter($eventId, $orgId)
{
return (bool)$this->find('first', array(
'recursive' => -1,
'callbacks' => false,
'fields' => ['Sighting.id'],
'conditions' => array(
'Sighting.event_id' => $eventId,
'Sighting.org_id' => $orgId,
)
));
return $this->hasAny([
'Sighting.event_id' => $eventId,
'Sighting.org_id' => $orgId,
]);
}
/**
@ -1368,7 +1552,7 @@ class Sighting extends AppModel
$org = $org['Organisation'];
}
$this->orgCache[$orgId] = $org;
return $this->orgCache[$orgId];
return $org;
}
/**

5
tests/testlive_security.py
View File

@ -1536,7 +1536,8 @@ class TestSecurity(unittest.TestCase):
event = user1.add_event(self.__generate_event())
check_response(event)
check_response(user1.add_sighting(s, event.Attribute[0]))
self.assertEqual(len(user1.sightings(event)), 1, "User should see hos own sighting")
self.assertEqual(len(user1.sightings(event)), 1, "User should see only own sighting")
self.assertEqual(len(user1.search_sightings('event', event.id)), 1)
org = self.__create_org()
user = self.__create_user(org.id, ROLE.USER)
@ -1544,9 +1545,11 @@ class TestSecurity(unittest.TestCase):
user2.global_pythonify = True
self.assertEqual(len(user2.sightings(event)), 0, "User should not seen any sighting")
self.assertEqual(len(user2.search_sightings('event', event.id)), 0)
with self.__setting({"MISP.host_org_id": self.test_org.id, "Plugin.Sightings_policy": 3}):
self.assertEqual(len(user2.sightings(event)), 1, "User should see host org sighting")
self.assertEqual(len(user2.search_sightings('event', event.id)), 1)
self.admin_misp_connector.delete_event(event)
self.admin_misp_connector.delete_user(user)