MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [internal] Faster attaching references

pull/7881/head
Jakub Onderka 2021-10-24 12:53:27 +02:00
parent 11e8b09d6f
commit 81717d4c0d
1 changed files with 87 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
app/Model/Event.php
View File

@ -1959,7 +1959,7 @@ class Event extends AppModel
$conditions['AND'][] = array('Event.published' => 1);
$conditionsAttributes['AND'][] = array('Attribute.to_ids' => 1);
}
$softDeletables = array('Attribute', 'Object', 'ObjectReference', 'EventReport');
$softDeletables = array('Attribute', 'Object', 'EventReport');
if (isset($options['deleted'])) {
if (!is_array($options['deleted'])) {
$options['deleted'] = array($options['deleted']);
@ -2056,9 +2056,6 @@ class Event extends AppModel
'Object' => array(
'conditions' => $conditionsObjects,
'order' => false,
'ObjectReference' => array(
'order' => false
)
),
'ShadowAttribute' => array(
'fields' => $fieldsShadowAtt,
@ -2145,6 +2142,10 @@ class Event extends AppModel
$this->__attachAttributeTags($results, $options['excludeLocalTags']);
}
if (!$options['metadata'] && !$flatten) {
$this->__attachReferences($results);
}
foreach ($results as &$event) {
/*
// REMOVING THIS FOR NOW - users should see data they own, even if they're not in the sharing group.
@ -2170,7 +2171,6 @@ class Event extends AppModel
$this->Warninglist->attachWarninglistToAttributes($event['ShadowAttribute']);
$event['warnings'] = $eventWarnings;
}
$this->__attachReferences($event);
$this->__attachTags($event, $justExportableTags);
$this->__attachGalaxies($event, $user, $options['excludeGalaxy'], $options['fetchFullClusters']);
$event = $this->Orgc->attachOrgs($event, $fieldsOrg);
@ -6805,6 +6805,88 @@ class Event extends AppModel
return $this->processModuleResultsData($user, $resolved_data, $id, $default_comment);
}
/**
* Attach references to objects faster than CakePHP.
* @param array $events
*/
private function __attachReferences(array &$events)
{
$eventIds = [];
foreach ($events as $event) {
if (!empty($event['Object'])) {
$eventIds[] = $event['Event']['id']; // event contains objects
}
}
if (!empty($eventIds)) {
// Do not fetch fields that we already know to reduce memory usage
$schema = $this->Object->ObjectReference->schema();
unset($schema['event_id']);
unset($schema['source_uuid']);
$references = $this->Object->ObjectReference->find('all', [
'conditions' => ['ObjectReference.event_id' => $eventIds],
'fields' => array_keys($schema),
'recursive' => -1,
]);
}
if (empty($references)) {
// Assign empty object reference object
foreach ($events as &$event) {
foreach ($event['Object'] as &$object) {
$object['ObjectReference'] = [];
}
}
return;
}
$referencesForObject = [];
foreach ($references as $reference) {
$referencesForObject[$reference['ObjectReference']['object_id']][] = $reference['ObjectReference'];
}
$fieldsToCopy = array(
'common' => array('distribution', 'sharing_group_id', 'uuid'),
'Attribute' => array('value', 'type', 'category', 'to_ids'),
'Object' => array('name', 'meta-category')
);
foreach ($events as &$event) {
foreach ($event['Object'] as &$object) {
$objectReferences = $referencesForObject[$object['id']] ?? [];
foreach ($objectReferences as &$reference) {
$reference['event_id'] = $event['Event']['id'];
$reference['source_uuid'] = $object['uuid'];
// find referenced object in current event
$type = $reference['referenced_type'] == 0 ? 'Attribute' : 'Object';
$found = null;
foreach ($event[$type] as $o) {
if ($o['id'] == $reference['referenced_id']) {
$found = $o;
break;
}
}
if ($found) {
// copy requested fields
$copied = [];
foreach (array_merge($fieldsToCopy['common'], $fieldsToCopy[$type]) as $field) {
$copied[$field] = $found[$field];
}
$reference[$type] = $copied;
} else { // object / attribute might be from an extended event
$otherEventText = __('%s from another event', $type);
$reference[$type] = [
'name' => '',
'meta-category' => $otherEventText,
'category' => $otherEventText,
'type' => '',
'value' => '',
'uuid' => $reference['referenced_uuid']
];
}
}
$object['ObjectReference'] = $objectReferences;
}
}
}
/**
* Faster way how to attach tags to events that integrated in CakePHP.
* @param array $events
@ -6940,58 +7022,6 @@ class Event extends AppModel
}
}
/**
* Attach referenced object to ObjectReference. Since reference can be just to attribute or object in the same event,
* we just find proper element in event.
*
* @param array $event
*/
private function __attachReferences(array &$event)
{
if (!isset($event['Object'])) {
return;
}
$fieldsToCopy = array(
'common' => array('distribution', 'sharing_group_id', 'uuid'),
'Attribute' => array('value', 'type', 'category', 'to_ids'),
'Object' => array('name', 'meta-category')
);
foreach ($event['Object'] as $k => $object) {
foreach ($object['ObjectReference'] as $k2 => $reference) {
// find referenced object in current event
$type = $reference['referenced_type'] == 0 ? 'Attribute' : 'Object';
$found = null;
foreach ($event[$type] as $o) {
if ($o['id'] == $reference['referenced_id']) {
$found = $o;
break;
}
}
if ($found) {
// copy requested fields
$reference = [];
foreach (array_merge($fieldsToCopy['common'], $fieldsToCopy[$type]) as $field) {
$reference[$field] = $found[$field];
}
$event['Object'][$k]['ObjectReference'][$k2][$type] = $reference;
} else { // object / attribute might be from an extended event
$otherEventText = __('%s from another event', $reference['referenced_type'] == 0 ? 'Attribute' : 'Object');
$event['Object'][$k]['ObjectReference'][$k2][$type] = [
'name' => '',
'meta-category' => $otherEventText,
'category' => $otherEventText,
'type' => '',
'value' => '',
'uuid' => $reference['referenced_uuid']
];
}
}
}
}
/**
* @param array $user
* @param string $returnFormat