MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [internal] Faster attaching attribute tags

pull/7881/head
Jakub Onderka 2021-10-23 18:21:24 +02:00
parent d2426c2458
commit abdd526b2d
2 changed files with 71 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

58
app/Model/Event.php
View File

@ -2052,9 +2052,6 @@ class Event extends AppModel
'Attribute' => array(
'fields' => $fieldsAtt,
'conditions' => $conditionsAttributes,
'AttributeTag' => array(
'order' => false
),
'order' => false
),
'Object' => array(
@ -2084,9 +2081,6 @@ class Event extends AppModel
$params['contain']['EventTag']['conditions'] = array(
'EventTag.local' => 0
);
$params['contain']['Attribute']['AttributeTag']['conditions'] = array(
'AttributeTag.local' => 0
);
}
if ($flatten) {
unset($params['contain']['Object']);
@ -2148,6 +2142,10 @@ class Event extends AppModel
$options['allow_proposal_blocking'] = false; // proposal blocking is not enabled
}
if (!$options['metadata']) {
$this->__attachAttributeTags($results, $options['excludeLocalTags']);
}
foreach ($results as &$event) {
/*
// REMOVING THIS FOR NOW - users should see data they own, even if they're not in the sharing group.
@ -6808,6 +6806,42 @@ class Event extends AppModel
return $this->processModuleResultsData($user, $resolved_data, $id, $default_comment);
}
/**
* Faster way how to attach tags to events that integrated in CakePHP.
* @param array $events
* @param bool $excludeLocalTags
*/
private function __attachAttributeTags(array &$events, $excludeLocalTags = false)
{
$eventIds = array_column(array_column($events, 'Event'), 'id');
$conditions = ['AttributeTag.event_id' => $eventIds];
if ($excludeLocalTags) {
$conditions['AttributeTag.local'] = false;
}
$ats = $this->Attribute->AttributeTag->find('all', [
'conditions' => $conditions,
'fields' => ['AttributeTag.attribute_id', 'AttributeTag.tag_id', 'AttributeTag.local'],
'recursive' => -1,
]);
if (empty($ats)) {
foreach ($events as &$event) {
foreach ($event['Attribute'] as &$attribute) {
$attribute['AttributeTag'] = [];
}
}
return;
}
$atForAttributes = [];
foreach ($ats as $at) {
$atForAttributes[$at['AttributeTag']['attribute_id']][] = $at['AttributeTag'];
}
foreach ($events as &$event) {
foreach ($event['Attribute'] as &$attribute) {
$attribute['AttributeTag'] = $atForAttributes[$attribute['id']] ?? [];
}
}
}
/**
* Get tag from cache by given ID.
*
@ -6844,21 +6878,19 @@ class Event extends AppModel
if (!empty($event['Attribute'])) {
foreach ($event['Attribute'] as $attribute) {
if (!empty($attribute['AttributeTag'])) {
foreach ($attribute['AttributeTag'] as $attributeTag) {
$tagIds[$attributeTag['tag_id']] = true;
}
foreach ($attribute['AttributeTag'] as $attributeTag) {
$tagIds[$attributeTag['tag_id']] = true;
}
}
}
$notCachedTags = array_diff(array_keys($tagIds), isset($this->assetCache['tags']) ? array_keys($this->assetCache['tags']) : []);
$notCachedTags = array_diff_key($tagIds, isset($this->assetCache['tags']) ? $this->assetCache['tags'] : []);
if (empty($notCachedTags)) {
return;
}
$conditions = ['id' => $notCachedTags];
$conditions = ['Tag.id' => array_keys($notCachedTags)];
if ($justExportable) {
$conditions['exportable'] = 1;
$conditions['Tag.exportable'] = 1;
}
$tags = $this->EventTag->Tag->find('all', [
'recursive' => -1,

28
tests/testlive_comprehensive_local.py
View File

@ -62,6 +62,7 @@ class TestComprehensive(unittest.TestCase):
def setUp(self):
self.user_misp_connector.global_pythonify = True
self.admin_misp_connector.global_pythonify = True
def test_search_index(self):
# Search all events
@ -330,7 +331,7 @@ class TestComprehensive(unittest.TestCase):
# Create test event
event = create_simple_event()
event = self.user_misp_connector.add_event(event, pythonify=True)
event = self.user_misp_connector.add_event(event)
check_response(event)
# Delete event
@ -343,7 +344,7 @@ class TestComprehensive(unittest.TestCase):
event = create_simple_event()
event.add_attribute('text', "deleted", deleted=True)
event.add_attribute('text', "not-deleted")
event = self.user_misp_connector.add_event(event, pythonify=True)
event = self.user_misp_connector.add_event(event)
check_response(event)
# Not deleted
@ -373,6 +374,29 @@ class TestComprehensive(unittest.TestCase):
check_response(self.user_misp_connector.delete_event(event))
def test_view_event_exclude_local_tags(self):
event = create_simple_event()
event.add_tag({"name": "local", "local": 1})
event.add_tag({"name": "global", "local": 0})
event.attributes[0].add_tag({"name": "local", "local": 1})
event.attributes[0].add_tag({"name": "global", "local": 0})
event = self.admin_misp_connector.add_event(event)
check_response(event)
event_with_local_tags = self.admin_misp_connector.get_event(event)
check_response(event_with_local_tags)
self.assertEqual(len(event_with_local_tags.tags), 2)
self.assertEqual(len(event_with_local_tags.attributes[0].tags), 2)
event_without_local_tags = self.admin_misp_connector._check_json_response(self.admin_misp_connector._prepare_request('GET', f'events/view/{event.id}/excludeLocalTags:1'))
check_response(event_without_local_tags)
self.assertEqual(event_without_local_tags["Event"]["Tag"][0]["local"], 0, event_without_local_tags)
self.assertEqual(event_without_local_tags["Event"]["Attribute"][0]["Tag"][0]["local"], 0, event_without_local_tags)
check_response(self.admin_misp_connector.delete_event(event))
if __name__ == '__main__':
unittest.main()