MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [correlation] Faster saving correlations

pull/8572/head
Jakub Onderka 2022-09-07 18:09:47 +02:00
parent c367e2b3a1
commit 1daab04ce2
3 changed files with 70 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
app/Model/Behavior/DefaultCorrelationBehavior.php
View File

@ -58,12 +58,18 @@ class DefaultCorrelationBehavior extends ModelBehavior
return self::TABLE_NAME;
}
/**
* @param Model $Model
* @param string $value
* @param array $a
* @param array $b
* @return array
*/
public function createCorrelationEntry(Model $Model, $value, $a, $b)
{
$valueId = $this->Correlation->CorrelationValue->getValueId($value);
if ($this->deadlockAvoidance) {
return [
'value_id' => $valueId,
'value_id' => $value,
'1_event_id' => $a['Event']['id'],
'1_object_id' => $a['Attribute']['object_id'],
'1_attribute_id' => $a['Attribute']['id'],
@ -87,7 +93,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
];
} else {
return [
(int) $valueId,
$value,
(int) $a['Event']['id'],
(int) $a['Attribute']['object_id'],
(int) $a['Attribute']['id'],
@ -112,7 +118,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
}
}
public function saveCorrelations(Model $Model, $correlations)
public function saveCorrelations(Model $Model, array $correlations)
{
$fields = [
'value_id',
@ -138,14 +144,23 @@ class DefaultCorrelationBehavior extends ModelBehavior
'object_sharing_group_id'
];
// Replace value with value ID
$valueIndex = $this->deadlockAvoidance ? 'value_id' : 0;
$values = array_column($correlations, $valueIndex);
$valueIds = $this->Correlation->CorrelationValue->getIds($values);
foreach ($correlations as &$correlation) {
$correlation[$valueIndex] = $valueIds[$correlation[$valueIndex]];
}
if ($this->deadlockAvoidance) {
return $this->Correlation->saveMany($correlations, array(
return $this->Correlation->saveMany($correlations, [
'atomic' => false,
'callbacks' => false,
'deep' => false,
'validate' => false,
'fieldList' => $fields
));
'fieldList' => $fields,
]);
} else {
$db = $this->Correlation->getDataSource();
// Split to chunks datasource is is enabled

35
app/Model/Correlation.php
View File

@ -6,7 +6,8 @@ App::uses('AppModel', 'Model');
* @property Event $Event
* @property CorrelationValue $CorrelationValue
* @method saveCorrelations(array $correlations)
* @method runBeforeSaveCorrelation
* @method createCorrelationEntry(string $value, array $a, array $b)
* @method runBeforeSaveCorrelation(array $attribute)
* @method fetchRelatedEventIds(array $user, int $eventId, array $sgids)
* @method getFieldRules
* @method getContainRules($filter = null)
@ -183,17 +184,6 @@ class Correlation extends AppModel
return $correlatingAttributes;
}
/**
* @param string $value
* @param array $a Attribute A
* @param array $b Attribute B
* @return array
*/
private function __createCorrelationEntry($value, $a, $b)
{
return $this->createCorrelationEntry($value, $a, $b);
}
public function correlateValue($value, $jobId = false)
{
$correlatingAttributes = $this->__getMatchingAttributes($value);
@ -216,7 +206,7 @@ class Correlation extends AppModel
if ($correlatingAttribute['Attribute']['event_id'] === $correlatingAttribute2['Attribute']['event_id']) {
continue;
}
$correlations[] = $this->__createCorrelationEntry($value, $correlatingAttribute, $correlatingAttribute2);
$correlations[] = $this->createCorrelationEntry($value, $correlatingAttribute, $correlatingAttribute2);
}
$extraCorrelations = $this->__addAdvancedCorrelations($correlatingAttribute);
if (!empty($extraCorrelations)) {
@ -224,8 +214,8 @@ class Correlation extends AppModel
if ($correlatingAttribute['Attribute']['event_id'] === $extraCorrelation['Attribute']['event_id']) {
continue;
}
$correlations[] = $this->__createCorrelationEntry($value, $correlatingAttribute, $extraCorrelation);
//$correlations = $this->__createCorrelationEntry($value, $extraCorrelation, $correlatingAttribute, $correlations);
$correlations[] = $this->createCorrelationEntry($value, $correlatingAttribute, $extraCorrelation);
//$correlations = $this->createCorrelationEntry($value, $extraCorrelation, $correlatingAttribute, $correlations);
}
}
if ($jobId && $k % 100 === 0) {
@ -252,17 +242,16 @@ class Correlation extends AppModel
}
}
public function correlateAttribute(array $attribute)
{
$this->runBeforeSaveCorrelation($attribute);
$this->afterSaveCorrelation($attribute);
}
public function beforeSaveCorrelation(array $attribute)
{
$this->runBeforeSaveCorrelation($attribute);
}
/**
* @param string $scope
* @param int $id
* @return false|array
*/
private function __cachedGetContainData($scope, $id)
{
if (!empty($this->getContainRules($scope))) {
@ -390,9 +379,9 @@ class Correlation extends AppModel
$value = $cV;
}
if ($a['Attribute']['id'] > $b['Attribute']['id']) {
$correlations[] = $this->__createCorrelationEntry($value, $a, $b);
$correlations[] = $this->createCorrelationEntry($value, $a, $b);
} else {
$correlations[] = $this->__createCorrelationEntry($value, $b, $a);
$correlations[] = $this->createCorrelationEntry($value, $b, $a);
}
}
}

36
app/Model/CorrelationValue.php
View File

@ -5,6 +5,42 @@ class CorrelationValue extends AppModel
{
public $recursive = -1;
/**
* @param array $values
* @return array Value in key, value ID in value
*/
public function getIds(array $values)
{
foreach ($values as &$value) {
$value = mb_substr($value, 0, 191);
}
$values = array_unique($values, SORT_REGULAR); // Remove duplicate values
$existingValues = $this->find('list', [
'recursive' => -1,
'fields' => ['value', 'id'],
'conditions' => [
'value' => $values,
]
]);
$notExistValues = array_diff($values, array_keys($existingValues));
if (!empty($notExistValues)) {
$this->getDataSource()->begin();
foreach ($notExistValues as $notExistValue) {
$this->create();
try {
$this->save(['value' => $notExistValue]);
$existingValues[$notExistValue] = $this->id;
} catch (Exception $e) {
$existingValues[$notExistValue] = $this->getValueId($notExistValue);
}
}
$this->getDataSource()->commit();
}
return $existingValues;
}
/**
* @param string $value
* @return int