MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #7956 from JakubOnderka/fix-attr-count 

fix: [internal] Correctly count matched attributes
pull/7967/head
Jakub Onderka 2021-11-17 13:58:06 +01:00 committed by GitHub
parent f4a013a5f7 b081dda00e
commit 96f6b5cd4d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 166 additions and 140 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

280
app/Model/Attribute.php
View File

@ -2014,15 +2014,23 @@ class Attribute extends AppModel
));
}
// Method that fetches all attributes for the various exports
// very flexible, it's basically a replacement for find, with the addition that it restricts access based on user
// options:
// fields
// contain
// conditions
// order
// group
public function fetchAttributes($user, $options = array(), &$continue = true, &$result_count = 0)
/**
* Method that fetches all attributes for the various exports
* very flexible, it's basically a replacement for find, with the addition that it restricts access based on user
* options:
* - fields
* - contain
* - conditions
* - order
* - group
*
* @param array $user
* @param array $options
* @param int|false $result_count If false, count is not fetched
* @return array
* @throws Exception
*/
public function fetchAttributes(array $user, array $options = [], &$result_count = false)
{
$params = array(
'conditions' => $this->buildConditions($user),
@ -2039,16 +2047,13 @@ class Attribute extends AppModel
);
if (!empty($options['includeProposals'])) {
$this->bindModel(
array('hasMany' => array(
'ShadowAttribute' => array(
'className' => 'ShadowAttribute',
'foreignKey' => 'old_id',
'conditions' => array('ShadowAttribute.deleted' => 0)
)
)
$this->bindModel(['hasMany' => array(
'ShadowAttribute' => array(
'className' => 'ShadowAttribute',
'foreignKey' => 'old_id',
'conditions' => array('ShadowAttribute.deleted' => 0)
)
);
)]);
$params['contain']['ShadowAttribute'] = array('fields' => array(
"id",
"old_id",
@ -2097,24 +2102,21 @@ class Attribute extends AppModel
if (isset($options['limit'])) {
$params['limit'] = $options['limit'];
}
if (
!empty($options['allow_proposal_blocking']) &&
Configure::read('MISP.proposals_block_attributes')
) {
if (!empty($options['allow_proposal_blocking']) && Configure::read('MISP.proposals_block_attributes')) {
$this->bindModel(array('hasMany' => array('ShadowAttribute' => array('foreignKey' => 'old_id'))));
$proposalRestriction = array(
'ShadowAttribute' => array(
'conditions' => array(
'AND' => array(
'ShadowAttribute.deleted' => 0,
'OR' => array(
'ShadowAttribute.proposal_to_delete' => 1,
'ShadowAttribute.to_ids' => 0
)
)
),
'fields' => array('ShadowAttribute.id', 'ShadowAttribute.value', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.to_ids')
)
'ShadowAttribute' => array(
'conditions' => array(
'AND' => array(
'ShadowAttribute.deleted' => 0,
'OR' => array(
'ShadowAttribute.proposal_to_delete' => 1,
'ShadowAttribute.to_ids' => 0
)
)
),
'fields' => array('ShadowAttribute.id', 'ShadowAttribute.value', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.to_ids')
)
);
$params['contain'] = array_merge($params['contain'], $proposalRestriction);
}
@ -2127,12 +2129,10 @@ class Attribute extends AppModel
if (empty($options['flatten'])) {
$params['conditions']['AND'][] = array('Attribute.object_id' => 0);
}
if (isset($options['order'])) {
$params['order'] = $options['order'];
}
$params['order'] = isset($options['order']) ? $options['order'] : [];
if (!isset($options['withAttachments'])) {
$options['withAttachments'] = false;
} else ($params['order'] = array());
}
if (!isset($options['enforceWarninglist'])) {
$options['enforceWarninglist'] = false;
}
@ -2156,7 +2156,7 @@ class Attribute extends AppModel
} else {
$options['includeDecayScore'] = true;
}
//Add EventTags to attributes to take them into account when calculating decay score
// Add EventTags to attributes to take them into account when calculating decay score
if ($options['includeDecayScore']) {
$options['includeEventTags'] = true;
}
@ -2171,10 +2171,6 @@ class Attribute extends AppModel
if (isset($options['group'])) {
$params['group'] = !empty($options['group']) ? $options['group'] : false;
}
// Site admin can access even unpublished event attributes if `unpublishedprivate` option is enabled
if (!$user['Role']['perm_site_admin'] && Configure::read('MISP.unpublishedprivate')) {
$params['conditions']['AND'][] = array('OR' => array('Event.published' => 1, 'Event.orgc_id' => $user['org_id'], 'Event.org_id' => $user['org_id']));
}
if (!empty($options['list'])) {
if (!empty($options['event_ids'])) {
return $this->find('column', [
@ -2187,7 +2183,6 @@ class Attribute extends AppModel
} else {
return $this->find('list', array(
'conditions' => $params['conditions'],
'recursive' => -1,
'contain' => array('Event', 'Object'),
'fields' => array('Attribute.event_id'),
'order' => false
@ -2198,127 +2193,118 @@ class Attribute extends AppModel
if (($options['enforceWarninglist'] || $options['includeWarninglistHits']) && !isset($this->Warninglist)) {
$this->Warninglist = ClassRegistry::init('Warninglist');
}
// If no limit is provided, fetch attributes in bulk
if (empty($params['limit'])) {
$loopLimit = 50000;
$loop = true;
$params['limit'] = $loopLimit;
$params['page'] = 0;
$params['page'] = 1;
} else {
$loop = false;
}
$attributes = array();
if (!empty($options['includeEventTags'])) {
$eventTags = array();
// Do not fetch result count when `$result_count` is false
if ($result_count !== false) {
$find_params = $params;
unset($find_params['limit']);
$result_count = $this->find('count', $find_params);
if ($result_count === 0) { // skip early
return [];
}
}
$find_params = $params;
unset($find_params['limit']);
$result_count = $this->find('count', $find_params);
while ($continue) {
if ($loop) {
$params['page'] = $params['page'] + 1;
if (isset($results) && count($results) < $loopLimit) {
$continue = false;
continue;
}
}
$eventTags = []; // tag cache
$attributes = [];
do {
$results = $this->find('all', $params);
if (empty($results)) {
break;
}
if (!empty($options['includeContext']) && !empty($results)) {
if (!empty($options['includeContext'])) {
$eventIds = [];
foreach ($results as $result) {
$eventIds[$result['Attribute']['event_id']] = true; // deduplicate
}
$eventsById = $this->__fetchEventsForAttributeContext($user, array_keys($eventIds), !empty($options['includeAllTags']));
foreach ($results as &$result) {
$result['Event'] = $eventsById[$result['Attribute']['event_id']];
}
unset($eventsById, $result); // unset result is important, because it is reference
unset($eventIds);
}
$this->attachTagsToAttributes($results, $options);
$proposals_block_attributes = Configure::read('MISP.proposals_block_attributes');
foreach ($results as $k => $result) {
foreach ($results as &$attribute) {
if (!empty($options['includeContext'])) {
$attribute['Event'] = $eventsById[$attribute['Attribute']['event_id']];
}
if (!empty($options['includeSightings'])) {
$temp = $result['Attribute'];
$temp['Event'] = $result['Event'];
$results[$k]['Attribute']['Sighting'] = $this->Sighting->attachToEvent($temp, $user, $temp['id']);
$temp = $attribute['Attribute'];
$temp['Event'] = $attribute['Event'];
$attribute['Attribute']['Sighting'] = $this->Sighting->attachToEvent($temp, $user, $temp['id']);
}
if (!empty($options['includeCorrelations'])) {
$attributeFields = array('id', 'event_id', 'object_id', 'object_relation', 'category', 'type', 'value', 'uuid', 'timestamp', 'distribution', 'sharing_group_id', 'to_ids', 'comment');
$results[$k]['Attribute']['RelatedAttribute'] = ($this->getRelatedAttributes($user, $results[$k]['Attribute'], $attributeFields, true));
$attribute['Attribute']['RelatedAttribute'] = $this->getRelatedAttributes($user, $attribute['Attribute'], $attributeFields, true);
}
}
if (!$loop) {
if (!empty($params['limit']) && count($results) < $params['limit']) {
$continue = false;
}
$break = true;
}
// return false if we're paginating
if (isset($options['limit']) && empty($results)) {
return array();
}
$results = array_values($results);
$proposals_block_attributes = Configure::read('MISP.proposals_block_attributes');
foreach ($results as $key => $attribute) {
if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttribute($attribute['Attribute'])) {
unset($results[$key]); // Remove attribute that match any enabled warninglists
continue;
}
if (!empty($options['includeEventTags'])) {
$results = $this->__attachEventTagsToAttributes($eventTags, $results, $key, $options);
$attribute = $this->__attachEventTagsToAttributes($eventTags, $attribute, $options);
}
if ($options['includeWarninglistHits']) {
$results[$key]['Attribute'] = $this->Warninglist->checkForWarning($results[$key]['Attribute']);
$attribute['Attribute'] = $this->Warninglist->checkForWarning($attribute['Attribute']);
}
if (!empty($options['includeAttributeUuid']) || !empty($options['includeEventUuid'])) {
$results[$key]['Attribute']['event_uuid'] = $results[$key]['Event']['uuid'];
$attribute['Attribute']['event_uuid'] = $attribute['Event']['uuid'];
}
if ($proposals_block_attributes) {
$this->__blockAttributeViaProposal($results, $key);
}
if ($options['withAttachments']) {
if ($this->typeIsAttachment($attribute['Attribute']['type'])) {
$encodedFile = $this->base64EncodeAttachment($attribute['Attribute']);
$results[$key]['Attribute']['data'] = $encodedFile;
if ($this->__blockAttributeViaProposal($attribute)) {
continue;
}
unset($attribute['ShadowAttribute']);
}
if ($options['withAttachments'] && $this->typeIsAttachment($attribute['Attribute']['type'])) {
$encodedFile = $this->base64EncodeAttachment($attribute['Attribute']);
$attribute['Attribute']['data'] = $encodedFile;
}
if ($options['includeDecayScore']) {
$this->DecayingModel = ClassRegistry::init('DecayingModel');
$include_full_model = isset($options['includeFullModel']) && $options['includeFullModel'] ? 1 : 0;
if (empty($results[$key]['Attribute']['AttributeTag'])) {
$results[$key]['Attribute']['AttributeTag'] = isset($results[$key]['AttributeTag']) ? $results[$key]['AttributeTag'] : array();
$results[$key]['Attribute']['EventTag'] = isset($results[$key]['EventTag']) ? $results[$key]['EventTag'] : array();
if (empty($attribute['Attribute']['AttributeTag'])) {
$attribute['Attribute']['AttributeTag'] = isset($attribute['AttributeTag']) ? $attribute['AttributeTag'] : array();
$attribute['Attribute']['EventTag'] = isset($attribute['EventTag']) ? $attribute['EventTag'] : array();
}
$results[$key]['Attribute'] = $this->DecayingModel->attachScoresToAttribute($user, $results[$key]['Attribute'], $options['decayingModel'], $options['modelOverrides'], $include_full_model);
unset($results[$key]['Attribute']['AttributeTag']);
unset($results[$key]['Attribute']['EventTag']);
if ($options['excludeDecayed'] && !empty($results[$key]['Attribute']['decay_score'])) { // filter out decayed attribute
$attribute['Attribute'] = $this->DecayingModel->attachScoresToAttribute($user, $attribute['Attribute'], $options['decayingModel'], $options['modelOverrides'], $include_full_model);
unset($attribute['Attribute']['AttributeTag']);
unset($attribute['Attribute']['EventTag']);
if ($options['excludeDecayed'] && !empty($attribute['Attribute']['decay_score'])) { // filter out decayed attribute
$decayed_flag = true;
foreach ($results[$key]['Attribute']['decay_score'] as $decayResult) { // remove attribute if ALL score results in a decay
foreach ($attribute['Attribute']['decay_score'] as $decayResult) { // remove attribute if ALL score results in a decay
$decayed_flag = $decayed_flag && $decayResult['decayed'];
}
if ($decayed_flag) {
unset($results[$key]);
continue;
}
}
}
if (!empty($results[$key])) {
if (!empty($options['includeGalaxy'])) {
$massaged_attribute = $this->Event->massageTags($user, $results[$key], 'Attribute');
$massaged_event = $this->Event->massageTags($user, $results[$key], 'Event');
$massaged_attribute['Galaxy'] = array_merge_recursive($massaged_attribute['Galaxy'], $massaged_event['Galaxy']);
$results[$key] = $massaged_attribute;
}
$attributes[] = $results[$key];
if (!empty($options['includeGalaxy'])) {
$massaged_attribute = $this->Event->massageTags($user, $attribute, 'Attribute');
$massaged_event = $this->Event->massageTags($user, $attribute, 'Event');
$massaged_attribute['Galaxy'] = array_merge_recursive($massaged_attribute['Galaxy'], $massaged_event['Galaxy']);
$attribute = $massaged_attribute;
}
$attributes[] = $attribute;
}
if (!empty($break)) {
break;
unset($attribute);
if ($loop) {
if (count($results) < $loopLimit) { // we fetched less results than limit, so we can skip next query
break;
}
$params['page']++;
}
}
} while ($loop);
return $attributes;
}
@ -2419,48 +2405,58 @@ class Attribute extends AppModel
}
}
private function __attachEventTagsToAttributes($eventTags, &$results, $key, $options)
/**
* @param array $eventTags
* @param array $attribute
* @param array $options
* @return array
*/
private function __attachEventTagsToAttributes(&$eventTags, $attribute, $options)
{
if (!isset($eventTags[$results[$key]['Event']['id']])) {
$tagConditions = array('EventTag.event_id' => $results[$key]['Event']['id']);
$eventId = $attribute['Event']['id'];
if (!isset($eventTags[$eventId])) {
$tagConditions = array('EventTag.event_id' => $eventId);
if (empty($options['includeAllTags'])) {
$tagConditions['Tag.exportable'] = 1;
}
$temp = $this->Event->EventTag->find('all', array(
'recursive' => -1,
'contain' => array('Tag'),
'conditions' => $tagConditions
'conditions' => $tagConditions,
));
foreach ($temp as $tag) {
$tag['EventTag']['Tag'] = $tag['Tag'];
unset($tag['Tag']);
$eventTags[$results[$key]['Event']['id']][] = $tag;
if (empty($temp)) {
$eventTags[$eventId] = [];
} else {
foreach ($temp as $tag) {
$tag['EventTag']['Tag'] = $tag['Tag'];
unset($tag['Tag']);
$eventTags[$eventId][] = $tag['EventTag'];
}
}
}
if (!empty($eventTags)) {
foreach ($eventTags[$results[$key]['Event']['id']] as $eventTag) {
$results[$key]['EventTag'][] = $eventTag['EventTag'];
foreach ($eventTags[$eventId] as $eventTag) {
$attribute['EventTag'][] = $eventTag;
}
}
return $results;
return $attribute;
}
private function __blockAttributeViaProposal(&$attributes, $k)
private function __blockAttributeViaProposal($attribute)
{
if (!empty($attributes[$k]['ShadowAttribute'])) {
foreach ($attributes[$k]['ShadowAttribute'] as $sa) {
if ($sa['value'] === $attributes[$k]['Attribute']['value'] &&
$sa['type'] === $attributes[$k]['Attribute']['type'] &&
$sa['category'] === $attributes[$k]['Attribute']['category'] &&
if (!empty($attribute['ShadowAttribute'])) {
foreach ($attribute['ShadowAttribute'] as $sa) {
if ($sa['value'] === $attribute['Attribute']['value'] &&
$sa['type'] === $attribute['Attribute']['type'] &&
$sa['category'] === $attribute['Attribute']['category'] &&
($sa['to_ids'] == 0 || $sa['to_ids'] == '') &&
$attributes[$k]['Attribute']['to_ids'] == 1
$attribute['Attribute']['to_ids'] == 1
) {
unset($attributes[$k]);
return true;
}
}
} else {
unset($attributes[$k]['ShadowAttribute']);
}
return false;
}
// Method gets and converts the contents of a file passed along as a base64 encoded string with the original filename into a zip archive
@ -3349,7 +3345,7 @@ class Attribute extends AppModel
* @param TmpFileTool $tmpfile
* @param object $exportTool
* @param array $exportToolParams
* @return int Number of attributes
* @return int Number of all attributes that matches given conditions
* @throws Exception
*/
private function __iteratedFetch(array $user, array $params, $loop, TmpFileTool $tmpfile, $exportTool, array $exportToolParams)
@ -3357,9 +3353,10 @@ class Attribute extends AppModel
$this->Allowedlist = ClassRegistry::init('Allowedlist');
$separator = $exportTool->separator($exportToolParams);
$elementCounter = 0;
$continue = true;
do {
$results = $this->fetchAttributes($user, $params, $continue, $elementCounter);
$results = $this->fetchAttributes($user, $params, $elementCounter);
$totalCount = $elementCounter;
$elementCounter = false; // do not call `count` again
if (empty($results)) {
break; // nothing found, skip rest
}
@ -3374,10 +3371,13 @@ class Attribute extends AppModel
$tmpfile->writeWithSeparator($handlerResult, $separator);
}
}
if ($loop && count($results) < $params['limit']) {
break; // do not continue if we received less results than limit
}
$params['page'] += 1;
} while ($loop && $continue);
} while ($loop);
return $elementCounter;
return $totalCount;
}
public function set_filter_uuid(&$params, $conditions, $options)

26
tests/testlive_security.py
View File

@ -1234,6 +1234,32 @@ class TestSecurity(unittest.TestCase):
self.admin_misp_connector.delete_user(publisher_user)
self.admin_misp_connector.delete_organisation(different_org)
def test_unpublished_private(self):
with self.__setting("MISP.unpublishedprivate", True):
created_event = self.admin_misp_connector.add_event(self.__generate_event())
self.assertIsInstance(created_event, MISPEvent, "Admin user should be able to create event")
logged_in = PyMISP(url, self.test_usr.authkey)
# Event is not published, so normal user should not see that event
self.assertFalse(logged_in.event_exists(created_event.uuid))
fetched_event = logged_in.get_event(created_event.uuid)
self.assertEqual(fetched_event["errors"][0], 404)
attributes = logged_in.search(controller='attributes', uuid=created_event.uuid)
self.assertEqual(len(attributes["Attribute"]), 0, attributes)
# Publish
self.assertSuccessfulResponse(self.admin_misp_connector.publish(created_event))
# Event is published, so normal user should see that event
self.assertTrue(logged_in.event_exists(created_event.uuid))
fetched_event = logged_in.get_event(created_event.uuid)
self.assertSuccessfulResponse(fetched_event, "User should be able to see published event")
attributes = logged_in.search(controller='attributes', uuid=created_event.uuid)
self.assertEqual(len(attributes["Attribute"]), 1, attributes)
# Cleanup
self.admin_misp_connector.delete_event(created_event)
def test_sg_index_user_cannot_see(self):
org = self.__create_org()
hidden_sg = self.__create_sharing_group()