mirror of https://github.com/MISP/MISP
fix: [sighting sync] speculative fix for critical sync issue
- pulls from an instance with extremely high numbers of sightings (~300M+) can lead to the pulled instance becoming unusable - This fix addresses multiple issues: - The use of last:0 as a sighting pull filter parameter lead to a search using an unindexed field - Internally searching for sighting IDs across 500 events in one shot can lead to massive data-sets - Internally searching for sighting IDs by Event.uuid on a joined table is extremely slow compared to searching on the sighting table alonenotes^2
parent
7a22d7c413
commit
c1638e0a9c
|
@ -2402,11 +2402,15 @@ class Attribute extends AppModel
|
|||
$timestamp[0] = $timestamp[1];
|
||||
$timestamp[1] = $temp;
|
||||
}
|
||||
$conditions['AND'][] = array($scope . ' >=' => $timestamp[0]);
|
||||
if ($timestamp[0] != 0) {
|
||||
$conditions['AND'][] = array($scope . ' >=' => $timestamp[0]);
|
||||
}
|
||||
$conditions['AND'][] = array($scope . ' <=' => $timestamp[1]);
|
||||
} else {
|
||||
$timestamp = $this->resolveTimeDelta($timestamp);
|
||||
$conditions['AND'][] = array($scope . ' >=' => $timestamp);
|
||||
if ($timestamp !== 0) {
|
||||
$conditions['AND'][] = array($scope . ' >=' => $timestamp);
|
||||
}
|
||||
}
|
||||
if ($returnRaw) {
|
||||
return $timestamp;
|
||||
|
|
|
@ -1102,8 +1102,12 @@ class Sighting extends AppModel
|
|||
$conditions['Attribute.uuid'] = $filters['uuid'];
|
||||
$contain[] = 'Attribute';
|
||||
} elseif ($filters['context'] === 'event') {
|
||||
$conditions['Event.uuid'] = $filters['uuid'];
|
||||
$contain[] = 'Event';
|
||||
$temp = $this->Event->find('column', [
|
||||
'recursive' => -1,
|
||||
'fields' => ['Event.id'],
|
||||
'conditions' => ['Event.uuid IN' => $filters['uuid']]
|
||||
]);
|
||||
$conditions['Sighting.event_id'] = empty($temp) ? -1 : $temp;
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -1131,15 +1135,30 @@ class Sighting extends AppModel
|
|||
$tmpfile = new TmpFileTool();
|
||||
$tmpfile->write($exportTool->header($exportToolParams));
|
||||
$separator = $exportTool->separator($exportToolParams);
|
||||
|
||||
|
||||
// fetch sightings matching the query without ACL checks
|
||||
$sightingIds = $this->find('column', [
|
||||
'conditions' => $conditions,
|
||||
'fields' => ['Sighting.id'],
|
||||
'contain' => $contain,
|
||||
'order' => 'Sighting.id',
|
||||
]);
|
||||
|
||||
if (!empty($conditions['Sighting.event_id']) && is_array($conditions['Sighting.event_id'])) {
|
||||
$conditions_copy = $conditions;
|
||||
$sightingIds = [];
|
||||
foreach ($conditions['Sighting.event_id'] as $e_id) {
|
||||
$conditions_copy['Sighting.event_id'] = $e_id;
|
||||
$tempIds = $this->find('column', [
|
||||
'conditions' => $conditions,
|
||||
'fields' => ['Sighting.id'],
|
||||
'contain' => $contain
|
||||
]);
|
||||
if (!empty($tempIds)) {
|
||||
$sightingIds = array_merge($sightingIds, $tempIds);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
$sightingIds = $this->find('column', [
|
||||
'conditions' => $conditions,
|
||||
'fields' => ['Sighting.id'],
|
||||
'contain' => $contain
|
||||
]);
|
||||
}
|
||||
|
||||
foreach (array_chunk($sightingIds, 500) as $chunk) {
|
||||
// fetch sightings with ACL checks and sighting policies
|
||||
$sightings = $this->getSightings($user, $chunk, $includeEvent, $includeAttribute, $includeUuid);
|
||||
|
|
Loading…
Reference in New Issue