From 1daab04ce22ef9d08c59c9df6dd3082419ce0863 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Wed, 7 Sep 2022 18:09:47 +0200 Subject: [PATCH] chg: [correlation] Faster saving correlations --- .../Behavior/DefaultCorrelationBehavior.php | 29 +++++++++++---- app/Model/Correlation.php | 35 +++++++----------- app/Model/CorrelationValue.php | 36 +++++++++++++++++++ 3 files changed, 70 insertions(+), 30 deletions(-) diff --git a/app/Model/Behavior/DefaultCorrelationBehavior.php b/app/Model/Behavior/DefaultCorrelationBehavior.php index be0e83307..7a6375ba0 100644 --- a/app/Model/Behavior/DefaultCorrelationBehavior.php +++ b/app/Model/Behavior/DefaultCorrelationBehavior.php @@ -58,12 +58,18 @@ class DefaultCorrelationBehavior extends ModelBehavior return self::TABLE_NAME; } + /** + * @param Model $Model + * @param string $value + * @param array $a + * @param array $b + * @return array + */ public function createCorrelationEntry(Model $Model, $value, $a, $b) { - $valueId = $this->Correlation->CorrelationValue->getValueId($value); if ($this->deadlockAvoidance) { return [ - 'value_id' => $valueId, + 'value_id' => $value, '1_event_id' => $a['Event']['id'], '1_object_id' => $a['Attribute']['object_id'], '1_attribute_id' => $a['Attribute']['id'], @@ -87,7 +93,7 @@ class DefaultCorrelationBehavior extends ModelBehavior ]; } else { return [ - (int) $valueId, + $value, (int) $a['Event']['id'], (int) $a['Attribute']['object_id'], (int) $a['Attribute']['id'], @@ -112,7 +118,7 @@ class DefaultCorrelationBehavior extends ModelBehavior } } - public function saveCorrelations(Model $Model, $correlations) + public function saveCorrelations(Model $Model, array $correlations) { $fields = [ 'value_id', @@ -138,14 +144,23 @@ class DefaultCorrelationBehavior extends ModelBehavior 'object_sharing_group_id' ]; + // Replace value with value ID + $valueIndex = $this->deadlockAvoidance ? 'value_id' : 0; + $values = array_column($correlations, $valueIndex); + $valueIds = $this->Correlation->CorrelationValue->getIds($values); + + foreach ($correlations as &$correlation) { + $correlation[$valueIndex] = $valueIds[$correlation[$valueIndex]]; + } + if ($this->deadlockAvoidance) { - return $this->Correlation->saveMany($correlations, array( + return $this->Correlation->saveMany($correlations, [ 'atomic' => false, 'callbacks' => false, 'deep' => false, 'validate' => false, - 'fieldList' => $fields - )); + 'fieldList' => $fields, + ]); } else { $db = $this->Correlation->getDataSource(); // Split to chunks datasource is is enabled diff --git a/app/Model/Correlation.php b/app/Model/Correlation.php index 61f11af35..42c1da648 100644 --- a/app/Model/Correlation.php +++ b/app/Model/Correlation.php @@ -6,7 +6,8 @@ App::uses('AppModel', 'Model'); * @property Event $Event * @property CorrelationValue $CorrelationValue * @method saveCorrelations(array $correlations) - * @method runBeforeSaveCorrelation + * @method createCorrelationEntry(string $value, array $a, array $b) + * @method runBeforeSaveCorrelation(array $attribute) * @method fetchRelatedEventIds(array $user, int $eventId, array $sgids) * @method getFieldRules * @method getContainRules($filter = null) @@ -183,17 +184,6 @@ class Correlation extends AppModel return $correlatingAttributes; } - /** - * @param string $value - * @param array $a Attribute A - * @param array $b Attribute B - * @return array - */ - private function __createCorrelationEntry($value, $a, $b) - { - return $this->createCorrelationEntry($value, $a, $b); - } - public function correlateValue($value, $jobId = false) { $correlatingAttributes = $this->__getMatchingAttributes($value); @@ -216,7 +206,7 @@ class Correlation extends AppModel if ($correlatingAttribute['Attribute']['event_id'] === $correlatingAttribute2['Attribute']['event_id']) { continue; } - $correlations[] = $this->__createCorrelationEntry($value, $correlatingAttribute, $correlatingAttribute2); + $correlations[] = $this->createCorrelationEntry($value, $correlatingAttribute, $correlatingAttribute2); } $extraCorrelations = $this->__addAdvancedCorrelations($correlatingAttribute); if (!empty($extraCorrelations)) { @@ -224,8 +214,8 @@ class Correlation extends AppModel if ($correlatingAttribute['Attribute']['event_id'] === $extraCorrelation['Attribute']['event_id']) { continue; } - $correlations[] = $this->__createCorrelationEntry($value, $correlatingAttribute, $extraCorrelation); - //$correlations = $this->__createCorrelationEntry($value, $extraCorrelation, $correlatingAttribute, $correlations); + $correlations[] = $this->createCorrelationEntry($value, $correlatingAttribute, $extraCorrelation); + //$correlations = $this->createCorrelationEntry($value, $extraCorrelation, $correlatingAttribute, $correlations); } } if ($jobId && $k % 100 === 0) { @@ -252,17 +242,16 @@ class Correlation extends AppModel } } - public function correlateAttribute(array $attribute) - { - $this->runBeforeSaveCorrelation($attribute); - $this->afterSaveCorrelation($attribute); - } - public function beforeSaveCorrelation(array $attribute) { $this->runBeforeSaveCorrelation($attribute); } + /** + * @param string $scope + * @param int $id + * @return false|array + */ private function __cachedGetContainData($scope, $id) { if (!empty($this->getContainRules($scope))) { @@ -390,9 +379,9 @@ class Correlation extends AppModel $value = $cV; } if ($a['Attribute']['id'] > $b['Attribute']['id']) { - $correlations[] = $this->__createCorrelationEntry($value, $a, $b); + $correlations[] = $this->createCorrelationEntry($value, $a, $b); } else { - $correlations[] = $this->__createCorrelationEntry($value, $b, $a); + $correlations[] = $this->createCorrelationEntry($value, $b, $a); } } } diff --git a/app/Model/CorrelationValue.php b/app/Model/CorrelationValue.php index 442de2c93..adac4717b 100644 --- a/app/Model/CorrelationValue.php +++ b/app/Model/CorrelationValue.php @@ -5,6 +5,42 @@ class CorrelationValue extends AppModel { public $recursive = -1; + /** + * @param array $values + * @return array Value in key, value ID in value + */ + public function getIds(array $values) + { + foreach ($values as &$value) { + $value = mb_substr($value, 0, 191); + } + $values = array_unique($values, SORT_REGULAR); // Remove duplicate values + $existingValues = $this->find('list', [ + 'recursive' => -1, + 'fields' => ['value', 'id'], + 'conditions' => [ + 'value' => $values, + ] + ]); + + $notExistValues = array_diff($values, array_keys($existingValues)); + if (!empty($notExistValues)) { + $this->getDataSource()->begin(); + foreach ($notExistValues as $notExistValue) { + $this->create(); + try { + $this->save(['value' => $notExistValue]); + $existingValues[$notExistValue] = $this->id; + } catch (Exception $e) { + $existingValues[$notExistValue] = $this->getValueId($notExistValue); + } + } + $this->getDataSource()->commit(); + } + + return $existingValues; + } + /** * @param string $value * @return int