mirror of https://github.com/MISP/MISP
chg: [correlations] performance tuning
- added a new constraint to check for correlation uniqueness (a_id, 1_a_id, value_id) - stopped dropping correlations on a full recorrelation - only correlate "upwards" towards higher IDspull/8530/head
parent
b34a6d670a
commit
097ad78d54
|
@ -82,7 +82,7 @@ class AppModel extends Model
|
|||
69 => false, 70 => false, 71 => true, 72 => true, 73 => false, 74 => false,
|
||||
75 => false, 76 => true, 77 => false, 78 => false, 79 => false, 80 => false,
|
||||
81 => false, 82 => false, 83 => false, 84 => false, 85 => false, 86 => false,
|
||||
87 => false, 88 => false, 89 => false, 90 => false,
|
||||
87 => false, 88 => false, 89 => false, 90 => false, 91 => false
|
||||
);
|
||||
|
||||
const ADVANCED_UPDATES_DESCRIPTION = array(
|
||||
|
@ -237,6 +237,29 @@ class AppModel extends Model
|
|||
$this->Workflow = Classregistry::init('Workflow');
|
||||
$this->Workflow->enableDefaultModules();
|
||||
break;
|
||||
case 91:
|
||||
$existing_index = $this->query(
|
||||
"SHOW INDEX FROM default_correlations WHERE Key_name = 'unique_correlation';"
|
||||
);
|
||||
if (empty($existing_index)) {
|
||||
$this->query(
|
||||
"ALTER TABLE default_correlations
|
||||
ADD CONSTRAINT unique_correlation
|
||||
UNIQUE KEY(attribute_id, 1_attribute_id, value_id);"
|
||||
);
|
||||
}
|
||||
$existing_index = $this->query(
|
||||
"SHOW INDEX FROM no_acl_correlations WHERE Key_name = 'unique_correlation';"
|
||||
);
|
||||
if (empty($existing_index)) {
|
||||
$this->query(
|
||||
"ALTER TABLE no_acl_correlations
|
||||
ADD CONSTRAINT unique_correlation
|
||||
UNIQUE KEY(attribute_id, 1_attribute_id, value_id);"
|
||||
);
|
||||
}
|
||||
$dbUpdateSuccess = true;
|
||||
break;
|
||||
default:
|
||||
$dbUpdateSuccess = $this->updateDatabase($command);
|
||||
break;
|
||||
|
|
|
@ -1601,9 +1601,6 @@ class Attribute extends AppModel
|
|||
$attributes = $this->find('all', $query);
|
||||
foreach ($attributes as $attribute) {
|
||||
$attribute['Attribute']['event_id'] = $eventId;
|
||||
if ($full) {
|
||||
$this->Correlation->beforeSaveCorrelation($attribute['Attribute']);
|
||||
}
|
||||
$this->Correlation->afterSaveCorrelation($attribute['Attribute'], $full);
|
||||
}
|
||||
$fetchedAttributes = count($attributes);
|
||||
|
|
|
@ -247,7 +247,12 @@ class Correlation extends AppModel
|
|||
*/
|
||||
private function __saveCorrelations(array $correlations)
|
||||
{
|
||||
return $this->saveCorrelations($correlations);
|
||||
try {
|
||||
return $this->saveCorrelations($correlations);
|
||||
} catch (Exception $e) {
|
||||
// Correlations may fail for different reasons, such as the correlation already existing. We don't care and don't want to break the process
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
public function correlateAttribute(array $attribute)
|
||||
|
@ -375,6 +380,10 @@ class Correlation extends AppModel
|
|||
$this->OverCorrelatingValue->unblock($cV);
|
||||
}
|
||||
foreach ($correlatingAttributes as $b) {
|
||||
// On a full correlation, only correlate with attributes that have a higher ID to avoid duplicate correlations
|
||||
if ($full && $b['Attribute']['id'] < $b['Attribute']['id']) {
|
||||
continue;
|
||||
}
|
||||
if (isset($b['Attribute']['value1'])) {
|
||||
// TODO: Currently it is hard to check if value1 or value2 correlated, so we check value2 and if not, it is value1
|
||||
$value = $cV === $b['Attribute']['value2'] ? $b['Attribute']['value2'] : $b['Attribute']['value1'];
|
||||
|
|
Loading…
Reference in New Issue