MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

fix: [performance] load analyst data in bulk 

speeds up event loading dramatically
pull/9178/merge
iglocska 2024-03-15 08:41:55 +01:00
parent 0723035c02
commit 0fb58cff44
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
3 changed files with 83 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
app/Model/Behavior/AnalystDataBehavior.php
View File

@ -9,6 +9,8 @@ class AnalystDataBehavior extends ModelBehavior
private $__current_type = null;
private $__valid_sharing_groups = null;
public function setup(Model $Model, $settings = array()) {
// We want to know whether we're a Note, Opinion or Relationship
$this->__current_type = $Model->alias;
@ -22,7 +24,9 @@ class AnalystDataBehavior extends ModelBehavior
];
$type = $Model->current_type;
if (empty($user['Role']['perm_site_admin'])) {
$validSharingGroups = $Model->SharingGroup->authorizedIds($user, true);
if ($this->__valid_sharing_groups === null) {
$this->__valid_sharing_groups = $Model->SharingGroup->authorizedIds($user, true);
}
$conditions['AND'][] = [
'OR' => [
$type . '.orgc_uuid' => $user['Organisation']['uuid'],
@ -30,7 +34,7 @@ class AnalystDataBehavior extends ModelBehavior
$type . '.distribution IN' => [1, 2, 3],
'AND' => [
$type . '.distribution' => 4,
$type . '.sharing_group_id IN' => $validSharingGroups
$type . '.sharing_group_id IN' => $this->__valid_sharing_groups
]
]
];
@ -42,6 +46,41 @@ class AnalystDataBehavior extends ModelBehavior
]);
}
// Return the analystData of the current type for a given UUID (this only checks the ACL of the analystData, NOT of the parent.)
public function fetchForUuids(Model $Model, $uuids, $user = null)
{
$conditions = [
'object_uuid' => $uuids
];
$type = $Model->current_type;
if (empty($user['Role']['perm_site_admin'])) {
if ($this->__valid_sharing_groups === null) {
$this->__valid_sharing_groups = $Model->SharingGroup->authorizedIds($user, true);
}
$conditions['AND'][] = [
'OR' => [
$type . '.orgc_uuid' => $user['Organisation']['uuid'],
$type . '.org_uuid' => $user['Organisation']['uuid'],
$type . '.distribution IN' => [1, 2, 3],
'AND' => [
$type . '.distribution' => 4,
$type . '.sharing_group_id IN' => $this->__valid_sharing_groups
]
]
];
}
$temp = $Model->find('all', [
'recursive' => -1,
'conditions' => $conditions,
'contain' => ['Org', 'Orgc', 'SharingGroup'],
]);
$results = [];
foreach ($temp as $result) {
$results[$result[$type]['object_uuid']] = $result;
}
return $results;
}
public function checkACL()
{

38
app/Model/Behavior/AnalystDataParentBehavior.php
View File

@ -38,6 +38,44 @@ class AnalystDataParentBehavior extends ModelBehavior
return $data;
}
public function attachAnalystDataBulk(Model $model, array $objects, array $types = ['Note', 'Opinion', 'Relationship'])
{
$uuids = [];
$objects = array_chunk($objects, 10000);
if (empty($this->__currentUser)) {
$user_id = Configure::read('CurrentUserId');
$this->User = ClassRegistry::init('User');
if ($user_id) {
$this->__currentUser = $this->User->getAuthUser($user_id);
}
}
foreach ($objects as $chunk => $chunked_objects) {
foreach ($chunked_objects as $k => $object) {
if (!empty($object['uuid'])) {
$uuids[] = $object['uuid'];
}
}
// No uuids, nothing to attach
if (empty($uuids)) {
continue;
}
foreach ($types as $type) {
$this->{$type} = ClassRegistry::init($type);
$this->{$type}->fetchRecursive = !empty($model->includeAnalystDataRecursive);
$temp = $this->{$type}->fetchForUuids($uuids, $this->__currentUser);
if (!empty($temp)) {
foreach ($chunked_objects as $k => $object) {
if (!empty($temp[$object['uuid']])) {
$objects[$chunk][$k][$type][] = $temp[$object['uuid']][$type];
}
}
}
}
}
$objects = call_user_func_array('array_merge', $objects);
return $objects;
}
public function afterFind(Model $model, $results, $primary = false)
{
if (!empty($model->includeAnalystData)) {

14
app/Model/Event.php
View File

@ -2212,11 +2212,7 @@ class Event extends AppModel
$event['Attribute'] = $this->Attribute->Correlation->attachCorrelationExclusion($event['Attribute']);
}
if (!empty($options['includeAnalystData'])) {
foreach ($event['Attribute'] as $k => $attribute) {
$this->Attribute->includeAnalystDataRecursive = true;
$analyst_data = $this->Attribute->attachAnalystData($attribute);
$event['Attribute'][$k] = array_merge($event['Attribute'][$k], $analyst_data);
}
$event['Attribute'] = $this->Attribute->attachAnalystDataBulk($event['Attribute']);
}
// move all object attributes to a temporary container
@ -2269,11 +2265,9 @@ class Event extends AppModel
if (isset($tempObjectAttributeContainer[$objectValue['id']])) {
$objectValue['Attribute'] = $tempObjectAttributeContainer[$objectValue['id']];
}
if (!empty($options['includeAnalystData'])) {
$this->Object->includeAnalystDataRecursive = true;
$analyst_data = $this->Object->attachAnalystData($objectValue);
$objectValue = array_merge($objectValue, $analyst_data);
}
}
if (!empty($options['includeAnalystData'])) {
$event['Object'] = $this->Object->attachAnalystDataBulk($event['Object']);
}
unset($tempObjectAttributeContainer);
}