From c46db527f68137675cef5aa1601139cfa561f189 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 10 Oct 2020 14:01:00 +0200 Subject: [PATCH 1/3] chg: [internal] Optimise event fetching --- app/Controller/EventsController.php | 3 +- app/Model/Event.php | 43 ++++++++++++++++++----------- 2 files changed, 29 insertions(+), 17 deletions(-) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 822717977..7e1a9ce69 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -1504,7 +1504,8 @@ class EventsController extends AppController $this->set('advancedFilteringActive', $advancedFiltering['active'] ? 1 : 0); $this->set('advancedFilteringActiveRules', $advancedFiltering['activeRules']); $this->set('defaultFilteringRules', $this->defaultFilteringRules); - $this->set('mitreAttackGalaxyId', $this->Event->GalaxyCluster->Galaxy->getMitreAttackGalaxyId()); + $this->loadModel('Galaxy'); + $this->set('mitreAttackGalaxyId', $this->Galaxy->getMitreAttackGalaxyId()); $this->set('modificationMapCSV', $modificationMapCSV); $this->set('title_for_layout', __('Event #%s', $event['Event']['id'])); } diff --git a/app/Model/Event.php b/app/Model/Event.php index a62e1d76f..7f87dfc8e 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -58,8 +58,6 @@ class Event extends AppModel 0 => 'Your organisation only', 1 => 'This community only', 2 => 'Connected communities', 3 => 'All communities', 4 => 'Sharing group' ); - private $__fTool = false; - public $shortDist = array(0 => 'Organisation', 1 => 'Community', 2 => 'Connected', 3 => 'All', 4 => ' sharing Group'); public $export_types = array( @@ -1962,20 +1960,16 @@ class Event extends AppModel 'includeFeedCorrelations', 'includeServerCorrelations', 'includeWarninglistHits', + 'noEventReports', // do not include event report in event data + 'noShadowAttributes', // do not fetch proposals ); if (!isset($options['excludeLocalTags']) && !empty($user['Role']['perm_sync']) && empty($user['Role']['perm_site_admin'])) { $options['excludeLocalTags'] = 1; } - if (!isset($options['excludeGalaxy']) || !$options['excludeGalaxy']) { - $this->GalaxyCluster = ClassRegistry::init('GalaxyCluster'); - } - if (!empty($options['includeDecayScore'])) { - $this->DecayingModel = ClassRegistry::init('DecayingModel'); - } if (!isset($options['includeEventCorrelations'])) { $options['includeEventCorrelations'] = true; } - foreach ($possibleOptions as &$opt) { + foreach ($possibleOptions as $opt) { if (!isset($options[$opt])) { $options[$opt] = false; } @@ -2020,10 +2014,9 @@ class Event extends AppModel } $conditionsAttributes = array(); $conditionsObjects = array(); - $conditionsObjectReferences = array(); $conditionsEventReport = array(); - if (isset($options['flatten']) && $options['flatten']) { + if ($options['flatten']) { $flatten = true; } else { $flatten = false; @@ -2190,7 +2183,6 @@ class Event extends AppModel $fieldsShadowAtt = array('ShadowAttribute.id', 'ShadowAttribute.type', 'ShadowAttribute.category', 'ShadowAttribute.value', 'ShadowAttribute.to_ids', 'ShadowAttribute.uuid', 'ShadowAttribute.event_uuid', 'ShadowAttribute.event_id', 'ShadowAttribute.old_id', 'ShadowAttribute.comment', 'ShadowAttribute.org_id', 'ShadowAttribute.proposal_to_delete', 'ShadowAttribute.timestamp', 'ShadowAttribute.first_seen', 'ShadowAttribute.last_seen'); $fieldsOrg = array('id', 'name', 'uuid', 'local'); $fieldsEventReport = array('*'); - $sharingGroupData = $this->__cacheSharingGroupData($user, $useCache); $params = array( 'conditions' => $conditions, 'recursive' => 0, @@ -2242,6 +2234,12 @@ class Event extends AppModel if ($flatten) { unset($params['contain']['Object']); } + if ($options['noEventReports']) { + unset($params['contain']['EventReport']); + } + if ($options['noShadowAttributes']) { + unset($params['contain']['ShadowAttribute']); + } if ($options['metadata']) { unset($params['contain']['Attribute']); unset($params['contain']['ShadowAttribute']); @@ -2253,6 +2251,15 @@ class Event extends AppModel return array(); } + $sharingGroupData = $this->__cacheSharingGroupData($user, $useCache); + + // Initialize classes that will be necessary during event fetching + if ((empty($options['metadata']) && empty($options['noSightings'])) && !isset($this->Sighting)) { + $this->Sighting = ClassRegistry::init('Sighting'); + } + if (!empty($options['includeDecayScore']) && !isset($this->DecayingModel)) { + $this->DecayingModel = ClassRegistry::init('DecayingModel'); + } if (($options['includeFeedCorrelations'] || $options['includeServerCorrelations']) && !isset($this->Feed)) { $this->Feed = ClassRegistry::init('Feed'); } @@ -2456,7 +2463,6 @@ class Event extends AppModel } } if (empty($options['metadata']) && empty($options['noSightings'])) { - $this->Sighting = ClassRegistry::init('Sighting'); $event['Sighting'] = $this->Sighting->attachToEvent($event, $user); } if ($options['includeSightingdb']) { @@ -2571,6 +2577,10 @@ class Event extends AppModel 'eventsExtendingUuid' => $event['Event']['uuid'], 'includeEventCorrelations' => $options['includeEventCorrelations'], 'includeWarninglistHits' => $options['includeWarninglistHits'], + 'noShadowAttributes' => $options['noShadowAttributes'], + 'noEventReports' => $options['noEventReports'], + 'noSightings' => isset($options['noSightings']) ? $options['noSightings'] : null, + 'sgReferenceOnly' => $options['sgReferenceOnly'], ]); foreach ($extensions as $extensionEvent) { $eventMeta = array( @@ -6174,11 +6184,12 @@ class Event extends AppModel public function massageTags($data, $dataType = 'Event', $excludeGalaxy = false, $cullGalaxyTags = false) { $data['Galaxy'] = array(); - if (empty($this->GalaxyCluster)) { - $this->GalaxyCluster = ClassRegistry::init('GalaxyCluster'); - } + // unset empty event tags that got added because the tag wasn't exportable if (!empty($data[$dataType . 'Tag'])) { + if (!isset($this->GalaxyCluster)) { + $this->GalaxyCluster = ClassRegistry::init('GalaxyCluster'); + } foreach ($data[$dataType . 'Tag'] as $k => &$dataTag) { if (empty($dataTag['Tag'])) { unset($data[$dataType . 'Tag'][$k]); From 150ccfcd73b792374ac67e5b338c7a6efad3aef6 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 10 Oct 2020 14:17:20 +0200 Subject: [PATCH 2/3] chg: [internal] Merge EventReports for extended view --- app/Model/Event.php | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/Model/Event.php b/app/Model/Event.php index 7f87dfc8e..0e0ffabbb 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -2597,6 +2597,10 @@ class Event extends AppModel foreach ($thingsToMerge as $thingToMerge) { $event[$thingToMerge] = array_merge($event[$thingToMerge], $extensionEvent[$thingToMerge]); } + // Merge event reports if requested + if (!$options['noEventReports']) { + $event['EventReport'] = array_merge($event['EventReport'], $extensionEvent['EventReport']); + } // Merge just tags that are not already in main event foreach ($extensionEvent['EventTag'] as $eventTag) { foreach ($event['EventTag'] as $eT) { From 897c2dc2255d7722e5a8c054afedfa727f611ce0 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Sat, 10 Oct 2020 14:19:39 +0200 Subject: [PATCH 3/3] chg: [internal] Do not fetch event reports for view --- app/Controller/EventsController.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/Controller/EventsController.php b/app/Controller/EventsController.php index 7e1a9ce69..9200d52df 100644 --- a/app/Controller/EventsController.php +++ b/app/Controller/EventsController.php @@ -1082,6 +1082,7 @@ class EventsController extends AppController $conditions['includeAllTags'] = true; $conditions['includeGranularCorrelations'] = 1; $conditions['includeEventCorrelations'] = false; + $conditions['noEventReports'] = true; // event reports for view are loaded dynamically if (!empty($filters['includeRelatedTags'])) { $this->set('includeRelatedTags', 1); $conditions['includeRelatedTags'] = 1; @@ -1524,6 +1525,7 @@ class EventsController extends AppController $conditions['includeAttachments'] = true; } else { $conditions['includeAllTags'] = true; + $conditions['noEventReports'] = true; // event reports for view are loaded dynamically } $deleted = 0; if (isset($this->params['named']['deleted'])) {