MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #9525 from JakubOnderka/speedup 

chg: [internal] Use Attribute::fetchAttributesInChunks for correlations
pull/9530/head
Jakub Onderka 2024-01-29 14:48:16 +01:00 committed by GitHub
parent de18832ec9 67eeb9df9d
commit 867d5281f0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 29 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

34
app/Model/Attribute.php
View File

@ -1246,27 +1246,34 @@ class Attribute extends AppModel
}
/**
* This method is useful if you want to iterate all attributes sorted by ID
* @param array $conditions
* @return Generator|void
* @param array $fields
* @param bool|string $callbacks
* @return Generator<array>|void
*/
private function fetchAttributesInChunks(array $conditions = [])
public function fetchAttributesInChunks(array $conditions = [], array $fields = [], $callbacks = true)
{
$query = [
'recursive' => -1,
'conditions' => $conditions,
'limit' => 500,
'order' => ['Attribute.id'],
'fields' => $fields,
'callbacks' => $callbacks,
];
while (true) {
$attributes = $this->find('all', [
'recursive' => -1,
'conditions' => $conditions,
'limit' => 500,
'order' => 'Attribute.id',
]);
if (empty($attributes)) {
return;
}
$attributes = $this->find('all', $query);
foreach ($attributes as $attribute) {
yield $attribute;
}
$count = count($attributes);
if ($count < 500) {
return;
}
$lastAttribute = $attributes[$count - 1];
$conditions['Attribute.id >'] = $lastAttribute['Attribute']['id'];
$query['conditions']['Attribute.id >'] = $lastAttribute['Attribute']['id'];
}
}
@ -3169,8 +3176,7 @@ class Attribute extends AppModel
$exportTool->additional_params
);
}
ClassRegistry::init('ConnectionManager');
$db = ConnectionManager::getDataSource('default');
$tmpfile = new TmpFileTool();
$tmpfile->write($exportTool->header($exportToolParams));
$loop = false;

34
app/Model/Correlation.php
View File

@ -153,7 +153,7 @@ class Correlation extends AppModel
if (!empty($eventIds)) {
$eventCount = count($eventIds);
foreach ($eventIds as $j => $currentEventId) {
$attributeCount += $this->__iteratedCorrelation(
$attributeCount += $this->iteratedCorrelation(
$jobId,
$full,
$attributeId,
@ -179,7 +179,7 @@ class Correlation extends AppModel
* @return int
* @throws Exception
*/
private function __iteratedCorrelation(
private function iteratedCorrelation(
$jobId = false,
$full = false,
$attributeId = null,
@ -215,30 +215,14 @@ class Correlation extends AppModel
if ($attributeId) {
$attributeConditions['Attribute.id'] = $attributeId;
}
$query = [
'recursive' => -1,
'conditions' => $attributeConditions,
// fetch just necessary fields to save memory
'fields' => $this->getFieldRules(),
'order' => 'Attribute.id',
'limit' => 5000,
'callbacks' => false, // memory leak fix
];
$attributes = $this->Attribute->fetchAttributesInChunks($attributeConditions, $this->getFieldRules(), false);
$attributeCount = 0;
do {
$attributes = $this->Attribute->find('all', $query);
foreach ($attributes as $attribute) {
$this->afterSaveCorrelation($attribute['Attribute'], $full, $event);
}
$fetchedAttributes = count($attributes);
unset($attributes);
$attributeCount += $fetchedAttributes;
if ($fetchedAttributes === 5000) { // maximum number of attributes fetched, continue in next loop
$query['conditions']['Attribute.id >'] = $attribute['Attribute']['id'];
} else {
break;
}
} while (true);
foreach ($attributes as $attribute) {
$this->afterSaveCorrelation($attribute['Attribute'], $full, $event);
++$attributeCount;
}
// Generating correlations can take long time, so clear caches after each event to refresh them
$this->cidrListCache = null;