MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [sync] Optimise removing old evens when pulling

pull/8205/head
Jakub Onderka 2022-03-12 16:02:16 +01:00
parent 9de6069ed9
commit f7af3c4e4e
2 changed files with 26 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
app/Model/Event.php
View File

@ -4753,30 +4753,6 @@ class Event extends AppModel
return $xmlArray;
}
public function removeOlder(array &$events, $scope = 'events')
{
$field = $scope === 'sightings' ? 'sighting_timestamp' : 'timestamp';
$localEvents = $this->find('all', [
'recursive' => -1,
'fields' => ['Event.uuid', 'Event.' . $field, 'Event.locked'],
]);
$localEvents = array_column(array_column($localEvents, 'Event'), null, 'uuid');
foreach ($events as $k => $event) {
// remove all events for the sighting sync if the remote is not aware of the new field yet
if (!isset($event[$field])) {
unset($events[$k]);
} else {
$uuid = $event['uuid'];
if (isset($localEvents[$uuid])
&& ($localEvents[$uuid][$field] >= $event[$field]
|| ($scope === 'events' && !$localEvents[$uuid]['locked'])))
{
unset($events[$k]);
}
}
}
}
public function sharingGroupRequired($field)
{
if ($this->data[$this->alias]['distribution'] == 4) {

31
app/Model/Server.php
View File

@ -780,13 +780,35 @@ class Server extends AppModel
return $eventIndex;
}
/**
* @param array $events
* @return void
*/
private function removeOlderEvents(array &$events)
{
$conditions = (count($events) > 10000) ? [] : ['Event.uuid' => array_column($events, 'uuid')];
$this->Event = ClassRegistry::init('Event');
$localEvents = $this->Event->find('all', [
'recursive' => -1,
'conditions' => $conditions,
'fields' => ['Event.uuid', 'Event.timestamp', 'Event.locked'],
]);
$localEvents = array_column(array_column($localEvents, 'Event'), null, 'uuid');
foreach ($events as $k => $event) {
$uuid = $event['uuid'];
if (isset($localEvents[$uuid]) && ($localEvents[$uuid]['timestamp'] >= $event['timestamp'] || !$localEvents[$uuid]['locked'])) {
unset($events[$k]);
}
}
}
/**
* Get an array of event UUIDs that are present on the remote server.
*
* @param ServerSyncTool $serverSync
* @param bool $all
* @param bool $ignoreFilterRules
* @param bool $force
* @param bool $ignoreFilterRules Ignore defined server pull rules
* @param bool $force If true, returns all events regardless their update timestamp
* @return array Array of event UUIDs.
* @throws HttpSocketHttpException
* @throws HttpSocketJsonException
@ -816,8 +838,7 @@ class Server extends AppModel
}
}
if (!$force) {
$this->Event = ClassRegistry::init('Event');
$this->Event->removeOlder($eventArray);
$this->removeOlderEvents($eventArray);
}
return array_column($eventArray, 'uuid');
}
@ -1045,7 +1066,7 @@ class Server extends AppModel
if ($push['canPush'] || $push['canSight']) {
$this->Sighting = ClassRegistry::init('Sighting');
$sightingSuccesses =$this->Sighting->pushSightings($user, $serverSync);
$sightingSuccesses = $this->Sighting->pushSightings($user, $serverSync);
} else {
$sightingSuccesses = array();
}