MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [internal] Simplify Attribute::fetchAttributes

pull/7956/head
Jakub Onderka 2021-11-15 16:32:41 +01:00
parent dbbc4599bf
commit 1bad9a008b
1 changed files with 99 additions and 113 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

212
app/Model/Attribute.php
View File

@ -2026,12 +2026,11 @@ class Attribute extends AppModel
* *
* @param array $user * @param array $user
* @param array $options * @param array $options
* @param bool $continue
* @param int|false $result_count If false, count is not fetched * @param int|false $result_count If false, count is not fetched
* @return array|int|null * @return array|int|null
* @throws Exception * @throws Exception
*/ */
public function fetchAttributes(array $user, array $options = [], &$continue = true, &$result_count = false) public function fetchAttributes(array $user, array $options = [], &$result_count = false)
{ {
$params = array( $params = array(
'conditions' => $this->buildConditions($user), 'conditions' => $this->buildConditions($user),
@ -2048,16 +2047,13 @@ class Attribute extends AppModel
); );
if (!empty($options['includeProposals'])) { if (!empty($options['includeProposals'])) {
$this->bindModel( $this->bindModel(['hasMany' => array(
array('hasMany' => array( 'ShadowAttribute' => array(
'ShadowAttribute' => array( 'className' => 'ShadowAttribute',
'className' => 'ShadowAttribute', 'foreignKey' => 'old_id',
'foreignKey' => 'old_id', 'conditions' => array('ShadowAttribute.deleted' => 0)
'conditions' => array('ShadowAttribute.deleted' => 0)
)
)
) )
); )]);
$params['contain']['ShadowAttribute'] = array('fields' => array( $params['contain']['ShadowAttribute'] = array('fields' => array(
"id", "id",
"old_id", "old_id",
@ -2106,10 +2102,7 @@ class Attribute extends AppModel
if (isset($options['limit'])) { if (isset($options['limit'])) {
$params['limit'] = $options['limit']; $params['limit'] = $options['limit'];
} }
if ( if (!empty($options['allow_proposal_blocking']) && Configure::read('MISP.proposals_block_attributes')) {
!empty($options['allow_proposal_blocking']) &&
Configure::read('MISP.proposals_block_attributes')
) {
$this->bindModel(array('hasMany' => array('ShadowAttribute' => array('foreignKey' => 'old_id')))); $this->bindModel(array('hasMany' => array('ShadowAttribute' => array('foreignKey' => 'old_id'))));
$proposalRestriction = array( $proposalRestriction = array(
'ShadowAttribute' => array( 'ShadowAttribute' => array(
@ -2136,12 +2129,10 @@ class Attribute extends AppModel
if (empty($options['flatten'])) { if (empty($options['flatten'])) {
$params['conditions']['AND'][] = array('Attribute.object_id' => 0); $params['conditions']['AND'][] = array('Attribute.object_id' => 0);
} }
if (isset($options['order'])) { $params['order'] = isset($options['order']) ? $options['order'] : [];
$params['order'] = $options['order'];
}
if (!isset($options['withAttachments'])) { if (!isset($options['withAttachments'])) {
$options['withAttachments'] = false; $options['withAttachments'] = false;
} else ($params['order'] = array()); }
if (!isset($options['enforceWarninglist'])) { if (!isset($options['enforceWarninglist'])) {
$options['enforceWarninglist'] = false; $options['enforceWarninglist'] = false;
} }
@ -2165,7 +2156,7 @@ class Attribute extends AppModel
} else { } else {
$options['includeDecayScore'] = true; $options['includeDecayScore'] = true;
} }
//Add EventTags to attributes to take them into account when calculating decay score // Add EventTags to attributes to take them into account when calculating decay score
if ($options['includeDecayScore']) { if ($options['includeDecayScore']) {
$options['includeEventTags'] = true; $options['includeEventTags'] = true;
} }
@ -2196,7 +2187,6 @@ class Attribute extends AppModel
} else { } else {
return $this->find('list', array( return $this->find('list', array(
'conditions' => $params['conditions'], 'conditions' => $params['conditions'],
'recursive' => -1,
'contain' => array('Event', 'Object'), 'contain' => array('Event', 'Object'),
'fields' => array('Attribute.event_id'), 'fields' => array('Attribute.event_id'),
'order' => false 'order' => false
@ -2207,18 +2197,15 @@ class Attribute extends AppModel
if (($options['enforceWarninglist'] || $options['includeWarninglistHits']) && !isset($this->Warninglist)) { if (($options['enforceWarninglist'] || $options['includeWarninglistHits']) && !isset($this->Warninglist)) {
$this->Warninglist = ClassRegistry::init('Warninglist'); $this->Warninglist = ClassRegistry::init('Warninglist');
} }
// If no limit is provided, fetch attributes in bulk
if (empty($params['limit'])) { if (empty($params['limit'])) {
$loopLimit = 50000; $loopLimit = 50000;
$loop = true; $loop = true;
$params['limit'] = $loopLimit; $params['limit'] = $loopLimit;
$params['page'] = 0; $params['page'] = 1;
} else { } else {
$loop = false; $loop = false;
} }
$attributes = array();
if (!empty($options['includeEventTags'])) {
$eventTags = array();
}
// Do not fetch result count when `$result_count` is false // Do not fetch result count when `$result_count` is false
if ($result_count !== false) { if ($result_count !== false) {
@ -2226,115 +2213,102 @@ class Attribute extends AppModel
unset($find_params['limit']); unset($find_params['limit']);
$result_count = $this->find('count', $find_params); $result_count = $this->find('count', $find_params);
if ($result_count === 0) { // skip early if ($result_count === 0) { // skip early
$continue = false;
return []; return [];
} }
} }
while ($continue) { $eventTags = []; // tag cache
if ($loop) { $attributes = [];
$params['page'] = $params['page'] + 1; do {
if (isset($results) && count($results) < $loopLimit) {
$continue = false;
continue;
}
}
$results = $this->find('all', $params); $results = $this->find('all', $params);
if (empty($results)) {
break;
}
if (!empty($options['includeContext']) && !empty($results)) { if (!empty($options['includeContext'])) {
$eventIds = []; $eventIds = [];
foreach ($results as $result) { foreach ($results as $result) {
$eventIds[$result['Attribute']['event_id']] = true; // deduplicate $eventIds[$result['Attribute']['event_id']] = true; // deduplicate
} }
$eventsById = $this->__fetchEventsForAttributeContext($user, array_keys($eventIds), !empty($options['includeAllTags'])); $eventsById = $this->__fetchEventsForAttributeContext($user, array_keys($eventIds), !empty($options['includeAllTags']));
foreach ($results as &$result) { unset($eventIds);
$result['Event'] = $eventsById[$result['Attribute']['event_id']];
}
unset($eventsById, $result); // unset result is important, because it is reference
} }
$this->attachTagsToAttributes($results, $options); $this->attachTagsToAttributes($results, $options);
$proposals_block_attributes = Configure::read('MISP.proposals_block_attributes');
foreach ($results as $k => $result) { foreach ($results as &$attribute) {
if (!empty($options['includeContext'])) {
$attribute['Event'] = $eventsById[$attribute['Attribute']['event_id']];
}
if (!empty($options['includeSightings'])) { if (!empty($options['includeSightings'])) {
$temp = $result['Attribute']; $temp = $attribute['Attribute'];
$temp['Event'] = $result['Event']; $temp['Event'] = $attribute['Event'];
$results[$k]['Attribute']['Sighting'] = $this->Sighting->attachToEvent($temp, $user, $temp['id']); $attribute['Attribute']['Sighting'] = $this->Sighting->attachToEvent($temp, $user, $temp['id']);
} }
if (!empty($options['includeCorrelations'])) { if (!empty($options['includeCorrelations'])) {
$attributeFields = array('id', 'event_id', 'object_id', 'object_relation', 'category', 'type', 'value', 'uuid', 'timestamp', 'distribution', 'sharing_group_id', 'to_ids', 'comment'); $attributeFields = array('id', 'event_id', 'object_id', 'object_relation', 'category', 'type', 'value', 'uuid', 'timestamp', 'distribution', 'sharing_group_id', 'to_ids', 'comment');
$results[$k]['Attribute']['RelatedAttribute'] = ($this->getRelatedAttributes($user, $results[$k]['Attribute'], $attributeFields, true)); $attribute['Attribute']['RelatedAttribute'] = $this->getRelatedAttributes($user, $attribute['Attribute'], $attributeFields, true);
} }
}
if (!$loop) {
if (!empty($params['limit']) && count($results) < $params['limit']) {
$continue = false;
}
$break = true;
}
// return false if we're paginating
if (isset($options['limit']) && empty($results)) {
return array();
}
$results = array_values($results);
$proposals_block_attributes = Configure::read('MISP.proposals_block_attributes');
foreach ($results as $key => $attribute) {
if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttribute($attribute['Attribute'])) { if ($options['enforceWarninglist'] && !$this->Warninglist->filterWarninglistAttribute($attribute['Attribute'])) {
unset($results[$key]); // Remove attribute that match any enabled warninglists
continue; continue;
} }
if (!empty($options['includeEventTags'])) { if (!empty($options['includeEventTags'])) {
$results = $this->__attachEventTagsToAttributes($eventTags, $results, $key, $options); $attribute = $this->__attachEventTagsToAttributes($eventTags, $attribute, $options);
} }
if ($options['includeWarninglistHits']) { if ($options['includeWarninglistHits']) {
$results[$key]['Attribute'] = $this->Warninglist->checkForWarning($results[$key]['Attribute']); $attribute['Attribute'] = $this->Warninglist->checkForWarning($attribute['Attribute']);
} }
if (!empty($options['includeAttributeUuid']) || !empty($options['includeEventUuid'])) { if (!empty($options['includeAttributeUuid']) || !empty($options['includeEventUuid'])) {
$results[$key]['Attribute']['event_uuid'] = $results[$key]['Event']['uuid']; $attribute['Attribute']['event_uuid'] = $attribute['Event']['uuid'];
} }
if ($proposals_block_attributes) { if ($proposals_block_attributes) {
$this->__blockAttributeViaProposal($results, $key); if ($this->__blockAttributeViaProposal($attribute)) {
} continue;
if ($options['withAttachments']) {
if ($this->typeIsAttachment($attribute['Attribute']['type'])) {
$encodedFile = $this->base64EncodeAttachment($attribute['Attribute']);
$results[$key]['Attribute']['data'] = $encodedFile;
} }
unset($attribute['ShadowAttribute']);
}
if ($options['withAttachments'] && $this->typeIsAttachment($attribute['Attribute']['type'])) {
$encodedFile = $this->base64EncodeAttachment($attribute['Attribute']);
$attribute['Attribute']['data'] = $encodedFile;
} }
if ($options['includeDecayScore']) { if ($options['includeDecayScore']) {
$this->DecayingModel = ClassRegistry::init('DecayingModel'); $this->DecayingModel = ClassRegistry::init('DecayingModel');
$include_full_model = isset($options['includeFullModel']) && $options['includeFullModel'] ? 1 : 0; $include_full_model = isset($options['includeFullModel']) && $options['includeFullModel'] ? 1 : 0;
if (empty($results[$key]['Attribute']['AttributeTag'])) { if (empty($attribute['Attribute']['AttributeTag'])) {
$results[$key]['Attribute']['AttributeTag'] = isset($results[$key]['AttributeTag']) ? $results[$key]['AttributeTag'] : array(); $attribute['Attribute']['AttributeTag'] = isset($attribute['AttributeTag']) ? $attribute['AttributeTag'] : array();
$results[$key]['Attribute']['EventTag'] = isset($results[$key]['EventTag']) ? $results[$key]['EventTag'] : array(); $attribute['Attribute']['EventTag'] = isset($attribute['EventTag']) ? $attribute['EventTag'] : array();
} }
$results[$key]['Attribute'] = $this->DecayingModel->attachScoresToAttribute($user, $results[$key]['Attribute'], $options['decayingModel'], $options['modelOverrides'], $include_full_model); $attribute['Attribute'] = $this->DecayingModel->attachScoresToAttribute($user, $attribute['Attribute'], $options['decayingModel'], $options['modelOverrides'], $include_full_model);
unset($results[$key]['Attribute']['AttributeTag']); unset($attribute['Attribute']['AttributeTag']);
unset($results[$key]['Attribute']['EventTag']); unset($attribute['Attribute']['EventTag']);
if ($options['excludeDecayed'] && !empty($results[$key]['Attribute']['decay_score'])) { // filter out decayed attribute if ($options['excludeDecayed'] && !empty($attribute['Attribute']['decay_score'])) { // filter out decayed attribute
$decayed_flag = true; $decayed_flag = true;
foreach ($results[$key]['Attribute']['decay_score'] as $decayResult) { // remove attribute if ALL score results in a decay foreach ($attribute['Attribute']['decay_score'] as $decayResult) { // remove attribute if ALL score results in a decay
$decayed_flag = $decayed_flag && $decayResult['decayed']; $decayed_flag = $decayed_flag && $decayResult['decayed'];
} }
if ($decayed_flag) { if ($decayed_flag) {
unset($results[$key]); continue;
} }
} }
} }
if (!empty($results[$key])) { if (!empty($options['includeGalaxy'])) {
if (!empty($options['includeGalaxy'])) { $massaged_attribute = $this->Event->massageTags($user, $attribute, 'Attribute');
$massaged_attribute = $this->Event->massageTags($user, $results[$key], 'Attribute'); $massaged_event = $this->Event->massageTags($user, $attribute, 'Event');
$massaged_event = $this->Event->massageTags($user, $results[$key], 'Event'); $massaged_attribute['Galaxy'] = array_merge_recursive($massaged_attribute['Galaxy'], $massaged_event['Galaxy']);
$massaged_attribute['Galaxy'] = array_merge_recursive($massaged_attribute['Galaxy'], $massaged_event['Galaxy']); $attribute = $massaged_attribute;
$results[$key] = $massaged_attribute;
}
$attributes[] = $results[$key];
} }
$attributes[] = $attribute;
} }
if (!empty($break)) { unset($attribute);
break;
if ($loop) {
if (count($results) < $loopLimit) { // we fetched less results than limit, so we can skip next query
break;
}
$params['page']++;
} }
} } while ($loop);
return $attributes; return $attributes;
} }
@ -2435,48 +2409,58 @@ class Attribute extends AppModel
} }
} }
private function __attachEventTagsToAttributes($eventTags, &$results, $key, $options) /**
* @param array $eventTags
* @param array $attribute
* @param array $options
* @return array
*/
private function __attachEventTagsToAttributes(&$eventTags, $attribute, $options)
{ {
if (!isset($eventTags[$results[$key]['Event']['id']])) { $eventId = $attribute['Event']['id'];
$tagConditions = array('EventTag.event_id' => $results[$key]['Event']['id']); if (!isset($eventTags[$eventId])) {
$tagConditions = array('EventTag.event_id' => $eventId);
if (empty($options['includeAllTags'])) { if (empty($options['includeAllTags'])) {
$tagConditions['Tag.exportable'] = 1; $tagConditions['Tag.exportable'] = 1;
} }
$temp = $this->Event->EventTag->find('all', array( $temp = $this->Event->EventTag->find('all', array(
'recursive' => -1, 'recursive' => -1,
'contain' => array('Tag'), 'contain' => array('Tag'),
'conditions' => $tagConditions 'conditions' => $tagConditions,
)); ));
foreach ($temp as $tag) { if (empty($temp)) {
$tag['EventTag']['Tag'] = $tag['Tag']; $eventTags[$eventId] = [];
unset($tag['Tag']); } else {
$eventTags[$results[$key]['Event']['id']][] = $tag; foreach ($temp as $tag) {
$tag['EventTag']['Tag'] = $tag['Tag'];
unset($tag['Tag']);
$eventTags[$eventId][] = $tag['EventTag'];
}
} }
} }
if (!empty($eventTags)) { if (!empty($eventTags)) {
foreach ($eventTags[$results[$key]['Event']['id']] as $eventTag) { foreach ($eventTags[$eventId] as $eventTag) {
$results[$key]['EventTag'][] = $eventTag['EventTag']; $attribute['EventTag'][] = $eventTag;
} }
} }
return $results; return $attribute;
} }
private function __blockAttributeViaProposal(&$attributes, $k) private function __blockAttributeViaProposal($attribute)
{ {
if (!empty($attributes[$k]['ShadowAttribute'])) { if (!empty($attribute['ShadowAttribute'])) {
foreach ($attributes[$k]['ShadowAttribute'] as $sa) { foreach ($attribute['ShadowAttribute'] as $sa) {
if ($sa['value'] === $attributes[$k]['Attribute']['value'] && if ($sa['value'] === $attribute['Attribute']['value'] &&
$sa['type'] === $attributes[$k]['Attribute']['type'] && $sa['type'] === $attribute['Attribute']['type'] &&
$sa['category'] === $attributes[$k]['Attribute']['category'] && $sa['category'] === $attribute['Attribute']['category'] &&
($sa['to_ids'] == 0 || $sa['to_ids'] == '') && ($sa['to_ids'] == 0 || $sa['to_ids'] == '') &&
$attributes[$k]['Attribute']['to_ids'] == 1 $attribute['Attribute']['to_ids'] == 1
) { ) {
unset($attributes[$k]); return true;
} }
} }
} else {
unset($attributes[$k]['ShadowAttribute']);
} }
return false;
} }
// Method gets and converts the contents of a file passed along as a base64 encoded string with the original filename into a zip archive // Method gets and converts the contents of a file passed along as a base64 encoded string with the original filename into a zip archive
@ -3372,10 +3356,9 @@ class Attribute extends AppModel
{ {
$this->Allowedlist = ClassRegistry::init('Allowedlist'); $this->Allowedlist = ClassRegistry::init('Allowedlist');
$separator = $exportTool->separator($exportToolParams); $separator = $exportTool->separator($exportToolParams);
$continue = true;
$elementCounter = 0; $elementCounter = 0;
do { do {
$results = $this->fetchAttributes($user, $params, $continue, $elementCounter); $results = $this->fetchAttributes($user, $params, $elementCounter);
$totalCount = $elementCounter; $totalCount = $elementCounter;
$elementCounter = false; // do not call `count` again $elementCounter = false; // do not call `count` again
if (empty($results)) { if (empty($results)) {
@ -3392,8 +3375,11 @@ class Attribute extends AppModel
$tmpfile->writeWithSeparator($handlerResult, $separator); $tmpfile->writeWithSeparator($handlerResult, $separator);
} }
} }
if ($loop && count($results) < $params['limit']) {
break; // do not continue if we received less results than limit
}
$params['page'] += 1; $params['page'] += 1;
} while ($loop && $continue); } while ($loop);
return $totalCount; return $totalCount;
} }