MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [internal] Reduce memory usage when generating correlations vol. 4

pull/8356/head
Jakub Onderka 2022-05-13 09:27:15 +02:00
parent 9735b2a5fc
commit 5b3aa10fc8
2 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
app/Model/Attribute.php
View File

@ -1628,6 +1628,7 @@ class Attribute extends AppModel
$this->Correlation->afterSaveCorrelation($attribute['Attribute'], $full, $event);
}
$fetchedAttributes = count($attributes);
unset($attributes);
$attributeCount += $fetchedAttributes;
if ($fetchedAttributes === 5000) { // maximum number of attributes fetched, continue in next loop
$query['conditions']['Attribute.id >'] = $attribute['Attribute']['id'];
@ -1635,7 +1636,7 @@ class Attribute extends AppModel
break;
}
} while (true);
// Generating correlations can take long time, so clear CIDR cache after each event
// Generating correlations can take long time, so clear CIDR cache after each event to refresh cache
$this->Correlation->clearCidrCache();
}
if ($jobId) {

7
app/Model/Correlation.php
View File

@ -186,7 +186,7 @@ class Correlation extends AppModel
'value' => $value,
'1_event_id' => $a['Event']['id'],
'1_attribute_id' => $a['Attribute']['id'],
'event_id' => $b['Attribute']['event_id'],
'event_id' => $b['Event']['id'],
'attribute_id' => $b['Attribute']['id'],
'org_id' => $b['Event']['org_id'],
'distribution' => $b['Event']['distribution'],
@ -199,7 +199,7 @@ class Correlation extends AppModel
$value,
(int) $a['Event']['id'],
(int) $a['Attribute']['id'],
(int) $b['Attribute']['event_id'],
(int) $b['Event']['id'],
(int) $b['Attribute']['id'],
(int) $b['Event']['org_id'],
(int) $b['Event']['distribution'],
@ -395,7 +395,7 @@ class Correlation extends AppModel
'conditions' => $conditions,
'recursive' => -1,
'fields' => [
'Attribute.event_id', 'Attribute.id', 'Attribute.distribution', 'Attribute.sharing_group_id',
'Attribute.id', 'Attribute.distribution', 'Attribute.sharing_group_id',
'Attribute.value1', 'Attribute.value2'
],
'contain' => ['Event.id', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id'],
@ -424,6 +424,7 @@ class Correlation extends AppModel
if (empty($correlations)) {
return true;
}
unset($correlatingAttributes);
return $this->__saveCorrelations($correlations);
}