MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #7055 from JakubOnderka/fast-event-galaxies 

chg: [internal] Faster fetching galaxy clusters when fetching event
pull/7062/head
Jakub Onderka 2021-02-22 10:07:29 +01:00 committed by GitHub
parent 7e0516cb7a 28b6a9066b
commit f959907c4e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 147 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
app/Controller/EventsController.php
View File

@ -1025,7 +1025,17 @@ class EventsController extends AppController
if (isset($filters['focus'])) {
$this->set('focus', $filters['focus']);
}
$conditions = array('eventid' => $id);
$conditions = [
'eventid' => $id,
'includeFeedCorrelations' => true,
'includeWarninglistHits' => true,
'fetchFullClusters' => false,
'includeAllTags' => true,
'includeGranularCorrelations' => true,
'includeEventCorrelations' => false,
'noEventReports' => true, // event reports for view are loaded dynamically
'noSightings' => true,
];
if (isset($filters['extended'])) {
$conditions['extended'] = 1;
$this->set('extended', 1);
@ -1048,21 +1058,11 @@ class EventsController extends AppController
if (isset($filters['toIDS']) && $filters['toIDS'] != 0) {
$conditions['to_ids'] = $filters['toIDS'] == 2 ? 0 : 1;
}
$conditions['includeFeedCorrelations'] = true;
$conditions['includeWarninglistHits'] = true;
if (!isset($filters['includeServerCorrelations'])) {
$conditions['includeServerCorrelations'] = 1;
if ($this->_isRest()) {
$conditions['includeServerCorrelations'] = 0;
}
} else {
$conditions['includeServerCorrelations'] = $filters['includeServerCorrelations'];
}
$conditions['includeAllTags'] = true;
$conditions['includeGranularCorrelations'] = 1;
$conditions['includeEventCorrelations'] = false;
$conditions['noEventReports'] = true; // event reports for view are loaded dynamically
$conditions['noSightings'] = true;
if (!empty($filters['includeRelatedTags'])) {
$this->set('includeRelatedTags', 1);
$conditions['includeRelatedTags'] = 1;
@ -1512,6 +1512,7 @@ class EventsController extends AppController
$conditions['includeAllTags'] = true;
$conditions['noEventReports'] = true; // event reports for view are loaded dynamically
$conditions['noSightings'] = true;
$conditions['fetchFullClusters'] = false;
}
$deleted = 0;
if (isset($this->params['named']['deleted'])) {

1
app/Lib/Tools/DistributionGraphTool.php
View File

@ -116,6 +116,7 @@ class DistributionGraphTool
'noShadowAttributes' => true,
'noEventReports' => true,
'noSightings' => true,
'excludeGalaxy' => true,
'includeEventCorrelations' => false,
'extended' => $this->__extended_view,
));

114
app/Model/Event.php
View File

@ -1843,7 +1843,7 @@ class Event extends AppModel
'eventsExtendingUuid',
'extended',
'excludeGalaxy',
'includeCustomGalaxyCluster',
'includeCustomGalaxyCluster', // not used
'includeRelatedTags',
'excludeLocalTags',
'includeDecayScore',
@ -1863,6 +1863,9 @@ class Event extends AppModel
if (!isset($options['includeEventCorrelations'])) {
$options['includeEventCorrelations'] = true;
}
if (!isset($options['fetchFullClusters'])) {
$options['fetchFullClusters'] = true;
}
foreach ($possibleOptions as $opt) {
if (!isset($options[$opt])) {
$options[$opt] = false;
@ -2209,6 +2212,7 @@ class Event extends AppModel
}
$this->__attachReferences($event, $fields);
$this->__attachTags($event, $justExportableTags);
$this->__attachGalaxies($event, $user, $options['excludeGalaxy'], $options['fetchFullClusters']);
$event = $this->Orgc->attachOrgs($event, $fieldsOrg);
if (!$options['sgReferenceOnly'] && $event['Event']['sharing_group_id']) {
$event['SharingGroup'] = $sharingGroupData[$event['Event']['sharing_group_id']]['SharingGroup'];
@ -2230,7 +2234,6 @@ class Event extends AppModel
}
$event['User']['email'] = $userEmail;
}
$event = $this->massageTags($user, $event, 'Event', $options['excludeGalaxy']);
// Let's find all the related events and attach it to the event itself
if ($options['includeEventCorrelations']) {
$results[$eventKey]['RelatedEvent'] = $this->getRelatedEvents($user, $event['Event']['id'], $sgids);
@ -2279,7 +2282,6 @@ class Event extends AppModel
unset($event['Attribute'][$key]);
continue;
}
$event['Attribute'][$key] = $this->massageTags($user, $event['Attribute'][$key], 'Attribute', $options['excludeGalaxy']);
if ($attribute['category'] === 'Financial fraud') {
$event['Attribute'][$key] = $this->Attribute->attachValidationWarnings($event['Attribute'][$key]);
}
@ -2381,6 +2383,104 @@ class Event extends AppModel
return $results;
}
/**
* Attach galaxy clusters to event and attributes.
* @param array $event
* @param array $user
* @param bool $excludeGalaxy
* @param bool $fetchFullCluster
*/
private function __attachGalaxies(array &$event, array $user, $excludeGalaxy, $fetchFullCluster)
{
$galaxyTags = [];
$event['Galaxy'] = [];
if (!$excludeGalaxy && isset($event['EventTag'])) {
foreach ($event['EventTag'] as $eventTag) {
if ($eventTag['Tag']['is_galaxy']) {
$galaxyTags[$eventTag['Tag']['id']] = $eventTag['Tag']['name'];
}
}
}
if (isset($event['Attribute'])) {
foreach ($event['Attribute'] as &$attribute) {
$attribute['Galaxy'] = [];
if (!$excludeGalaxy && isset($attribute['AttributeTag'])) {
foreach ($attribute['AttributeTag'] as $attributeTag) {
if ($attributeTag['Tag']['is_galaxy']) {
$galaxyTags[$attributeTag['Tag']['id']] = $attributeTag['Tag']['name'];
}
}
}
}
}
if ($excludeGalaxy || empty($galaxyTags)) {
return;
}
$this->GalaxyCluster = ClassRegistry::init('GalaxyCluster');
$clusters = $this->GalaxyCluster->getClusters($galaxyTags, $user, true, $fetchFullCluster);
if (empty($clusters)) {
return;
}
$clustersByTagNames = [];
foreach ($clusters as $cluster) {
$clustersByTagNames[mb_strtolower($cluster['GalaxyCluster']['tag_name'])] = $cluster['GalaxyCluster'];
}
unset($clusters);
if (isset($event['EventTag'])) {
foreach ($event['EventTag'] as $etk => $eventTag) {
if (!$eventTag['Tag']['is_galaxy']) {
continue;
}
$tagName = mb_strtolower($eventTag['Tag']['name']);
if (isset($clustersByTagNames[$tagName])) {
$cluster = $clustersByTagNames[$tagName];
$galaxyId = $cluster['Galaxy']['id'];
$cluster['local'] = isset($eventTag['local']) ? $eventTag['local'] : false;
if (isset($event['Galaxy'][$galaxyId])) {
unset($cluster['Galaxy']);
$event['Galaxy'][$galaxyId]['GalaxyCluster'][] = $cluster;
} else {
$event['Galaxy'][$galaxyId] = $cluster['Galaxy'];
unset($cluster['Galaxy']);
$event['Galaxy'][$galaxyId]['GalaxyCluster'] = [$cluster];
}
}
}
$event['Galaxy'] = array_values($event['Galaxy']);
}
if (isset($event['Attribute'])) {
foreach ($event['Attribute'] as &$attribute) {
if (isset($attribute['AttributeTag'])) {
foreach ($attribute['AttributeTag'] as $attributeTag) {
if (!$attributeTag['Tag']['is_galaxy']) {
continue;
}
$tagName = mb_strtolower($attributeTag['Tag']['name']);
if (isset($clustersByTagNames[$tagName])) {
$cluster = $clustersByTagNames[$tagName];
$galaxyId = $cluster['Galaxy']['id'];
$cluster['local'] = isset($attributeTag['local']) ? $attributeTag['local'] : false;
if (isset($attribute['Galaxy'][$galaxyId])) {
unset($cluster['Galaxy']);
$attribute['Galaxy'][$galaxyId]['GalaxyCluster'][] = $cluster;
} else {
$attribute['Galaxy'][$galaxyId] = $cluster['Galaxy'];
unset($cluster['Galaxy']);
$attribute['Galaxy'][$galaxyId]['GalaxyCluster'] = [$cluster];
}
}
}
$attribute['Galaxy'] = array_values($attribute['Galaxy']);
}
}
}
}
private function __cacheRelatedEventTags($eventTagCache, array $relatedAttribute, $excludeLocalTags)
{
if (!isset($eventTagCache[$relatedAttribute['id']])) {
@ -6795,9 +6895,13 @@ class Event extends AppModel
foreach ($event['EventTag'] as $etk => $eventTag) {
$tag = $this->__getCachedTag($eventTag['tag_id'], $justExportable);
if ($tag !== null) {
$tag['local'] = empty($eventTag['local']) ? 0 : 1;
$event['EventTag'][$etk]['Tag'] = $tag;
} else {
unset($event['EventTag'][$etk]);
}
}
$event['EventTag'] = array_values($event['EventTag']);
}
if (!empty($event['Attribute'])) {
foreach ($event['Attribute'] as $ak => $attribute) {
@ -6805,9 +6909,13 @@ class Event extends AppModel
foreach ($attribute['AttributeTag'] as $atk => $attributeTag) {
$tag = $this->__getCachedTag($attributeTag['tag_id'], $justExportable);
if ($tag !== null) {
$tag['local'] = empty($attributeTag['local']) ? 0 : 1;
$event['Attribute'][$ak]['AttributeTag'][$atk]['Tag'] = $tag;
} else {
unset($event['Attribute'][$ak]['AttributeTag'][$atk]);
}
}
$event['Attribute'][$ak]['AttributeTag'] = array_values($event['Attribute'][$ak]['AttributeTag']);
}
}
}

31
app/Model/GalaxyCluster.php
View File

@ -2,6 +2,9 @@
App::uses('AppModel', 'Model');
App::uses('TmpFileTool', 'Tools');
/**
* @property Tag $Tag
*/
class GalaxyCluster extends AppModel
{
public $useTable = 'galaxy_clusters';
@ -905,21 +908,33 @@ class GalaxyCluster extends AppModel
* @param bool $fetchFullCluster
* @return array
*/
public function getClusters(array $namesOrIds, array $user, $postProcess = true)
public function getClusters(array $namesOrIds, array $user, $postProcess = true, $fetchFullCluster = true)
{
$conditions = array();
if (count(array_filter($namesOrIds, 'is_numeric')) === count($namesOrIds)) { // all elements are numeric
$conditions[] = array('GalaxyCluster.id' => $namesOrIds);
$conditions = array('GalaxyCluster.id' => $namesOrIds);
} else {
$conditions[] = array('LOWER(GalaxyCluster.tag_name)' => array_map('strtolower', $namesOrIds));
$conditions = array('LOWER(GalaxyCluster.tag_name)' => array_map('strtolower', $namesOrIds));
}
$clusters = $this->fetchGalaxyClusters($user, array(
'conditions' => $conditions,
), true);
$options = ['conditions' => $conditions];
if (!$fetchFullCluster) {
$options['contain'] = ['Galaxy'];
}
$clusters = $this->fetchGalaxyClusters($user, $options, $fetchFullCluster);
if (!empty($clusters) && $postProcess) {
$tagNames = array_map('strtolower', array_column(array_column($clusters, 'GalaxyCluster'), 'tag_name'));
$tagIds = $this->Tag->find('list', [
'conditions' => ['LOWER(Tag.name)' => $tagNames],
'recursive' => -1,
'fields' => array('Tag.name', 'Tag.id'),
]);
$tagIds = array_change_key_case($tagIds);
foreach ($clusters as $k => $cluster) {
$clusters[$k] = $this->postprocess($cluster);
$tagName = strtolower($cluster['GalaxyCluster']['tag_name']);
$clusters[$k] = $this->postprocess($cluster, isset($tagIds[$tagName]) ? $tagIds[$tagName] : null);
}
}

1
app/Model/GalaxyClusterRelation.php
View File

@ -85,7 +85,6 @@ class GalaxyClusterRelation extends AppModel
if (!$user['Role']['perm_site_admin']) {
$alias = $this->alias;
$sgids = $this->Event->cacheSgids($user, true);
$gcids = $this->SourceCluster->cacheGalaxyClusterIDs($user);
$gcOwnerIds = $this->SourceCluster->cacheGalaxyClusterOwnerIDs($user);
$conditionsRelations['AND']['OR'] = [
"${alias}.galaxy_cluster_id" => $gcOwnerIds,