From c1638e0a9c58bbc36b3e733869331883ccdc017a Mon Sep 17 00:00:00 2001 From: iglocska Date: Fri, 16 Feb 2024 14:21:10 +0100 Subject: [PATCH 1/3] fix: [sighting sync] speculative fix for critical sync issue - pulls from an instance with extremely high numbers of sightings (~300M+) can lead to the pulled instance becoming unusable - This fix addresses multiple issues: - The use of last:0 as a sighting pull filter parameter lead to a search using an unindexed field - Internally searching for sighting IDs across 500 events in one shot can lead to massive data-sets - Internally searching for sighting IDs by Event.uuid on a joined table is extremely slow compared to searching on the sighting table alone --- app/Model/Attribute.php | 8 ++++++-- app/Model/Sighting.php | 39 +++++++++++++++++++++++++++++---------- 2 files changed, 35 insertions(+), 12 deletions(-) diff --git a/app/Model/Attribute.php b/app/Model/Attribute.php index 2d82063d1..c9327f452 100644 --- a/app/Model/Attribute.php +++ b/app/Model/Attribute.php @@ -2402,11 +2402,15 @@ class Attribute extends AppModel $timestamp[0] = $timestamp[1]; $timestamp[1] = $temp; } - $conditions['AND'][] = array($scope . ' >=' => $timestamp[0]); + if ($timestamp[0] != 0) { + $conditions['AND'][] = array($scope . ' >=' => $timestamp[0]); + } $conditions['AND'][] = array($scope . ' <=' => $timestamp[1]); } else { $timestamp = $this->resolveTimeDelta($timestamp); - $conditions['AND'][] = array($scope . ' >=' => $timestamp); + if ($timestamp !== 0) { + $conditions['AND'][] = array($scope . ' >=' => $timestamp); + } } if ($returnRaw) { return $timestamp; diff --git a/app/Model/Sighting.php b/app/Model/Sighting.php index 7384869b9..6f91868ba 100644 --- a/app/Model/Sighting.php +++ b/app/Model/Sighting.php @@ -1102,8 +1102,12 @@ class Sighting extends AppModel $conditions['Attribute.uuid'] = $filters['uuid']; $contain[] = 'Attribute'; } elseif ($filters['context'] === 'event') { - $conditions['Event.uuid'] = $filters['uuid']; - $contain[] = 'Event'; + $temp = $this->Event->find('column', [ + 'recursive' => -1, + 'fields' => ['Event.id'], + 'conditions' => ['Event.uuid IN' => $filters['uuid']] + ]); + $conditions['Sighting.event_id'] = empty($temp) ? -1 : $temp; } } @@ -1131,15 +1135,30 @@ class Sighting extends AppModel $tmpfile = new TmpFileTool(); $tmpfile->write($exportTool->header($exportToolParams)); $separator = $exportTool->separator($exportToolParams); - + // fetch sightings matching the query without ACL checks - $sightingIds = $this->find('column', [ - 'conditions' => $conditions, - 'fields' => ['Sighting.id'], - 'contain' => $contain, - 'order' => 'Sighting.id', - ]); - + if (!empty($conditions['Sighting.event_id']) && is_array($conditions['Sighting.event_id'])) { + $conditions_copy = $conditions; + $sightingIds = []; + foreach ($conditions['Sighting.event_id'] as $e_id) { + $conditions_copy['Sighting.event_id'] = $e_id; + $tempIds = $this->find('column', [ + 'conditions' => $conditions, + 'fields' => ['Sighting.id'], + 'contain' => $contain + ]); + if (!empty($tempIds)) { + $sightingIds = array_merge($sightingIds, $tempIds); + } + } + } else { + $sightingIds = $this->find('column', [ + 'conditions' => $conditions, + 'fields' => ['Sighting.id'], + 'contain' => $contain + ]); + } + foreach (array_chunk($sightingIds, 500) as $chunk) { // fetch sightings with ACL checks and sighting policies $sightings = $this->getSightings($user, $chunk, $includeEvent, $includeAttribute, $includeUuid); From 0f7b55a1df92b9ee9b2e84d0117c2847428a6a23 Mon Sep 17 00:00:00 2001 From: iglocska Date: Fri, 16 Feb 2024 14:41:19 +0100 Subject: [PATCH 2/3] fix: [db_schema] bump --- db_schema.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db_schema.json b/db_schema.json index b32833fed..a0a318251 100644 --- a/db_schema.json +++ b/db_schema.json @@ -9702,5 +9702,5 @@ "uuid": false } }, - "db_version": "119" -} \ No newline at end of file + "db_version": "120" +} From e4e6f1625ae15444671cef6fe802d0b3b41346f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rapha=C3=ABl=20Vinot?= Date: Fri, 16 Feb 2024 14:50:31 +0100 Subject: [PATCH 3/3] chg: [PyMISP] Bump version --- PyMISP | 2 +- app/Controller/AppController.php | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/PyMISP b/PyMISP index 05beec393..492cfba2d 160000 --- a/PyMISP +++ b/PyMISP @@ -1 +1 @@ -Subproject commit 05beec393c5613c352dafdaeaf5dcf983bfb2d51 +Subproject commit 492cfba2d2ad015d3fcda6e16c221fdefd93eca2 diff --git a/app/Controller/AppController.php b/app/Controller/AppController.php index 15fbc997f..8997cf30c 100755 --- a/app/Controller/AppController.php +++ b/app/Controller/AppController.php @@ -34,7 +34,7 @@ class AppController extends Controller public $helpers = array('OrgImg', 'FontAwesome', 'UserName'); private $__queryVersion = '158'; - public $pyMispVersion = '2.4.184'; + public $pyMispVersion = '2.4.185'; public $phpmin = '7.2'; public $phprec = '7.4'; public $phptoonew = '8.0'; diff --git a/requirements.txt b/requirements.txt index 5953ff55d..1fbc05deb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,7 +6,7 @@ misp-lib-stix2>=3.0.1.1 mixbox>=1.0.5 plyara>=2.1.1 pydeep2>=0.5.1 -pymisp==2.4.184 +pymisp==2.4.185 python-magic>=0.4.27 pyzmq>=25.1.1 redis>=5.0.1