From 6642d8bf7b50231c503482911ee900ba903cf043 Mon Sep 17 00:00:00 2001 From: Jakub Onderka Date: Thu, 29 Apr 2021 11:47:38 +0200 Subject: [PATCH] chg: [correlation] Allow to drop Correlation.{date,info} columns --- INSTALL/MYSQL.sql | 2 -- app/Model/Correlation.php | 50 ++++++++++++++++++++++++++++++++------- db_schema.json | 22 ----------------- 3 files changed, 41 insertions(+), 33 deletions(-) diff --git a/INSTALL/MYSQL.sql b/INSTALL/MYSQL.sql index 3291fd67e..574e15405 100644 --- a/INSTALL/MYSQL.sql +++ b/INSTALL/MYSQL.sql @@ -150,8 +150,6 @@ CREATE TABLE IF NOT EXISTS `correlations` ( `a_distribution` tinyint(4) NOT NULL, `sharing_group_id` int(11) NOT NULL, `a_sharing_group_id` int(11) NOT NULL, - `date` date NOT NULL, - `info` text COLLATE utf8_bin NOT NULL, PRIMARY KEY (`id`), INDEX `event_id` (`event_id`), INDEX `1_event_id` (`1_event_id`), diff --git a/app/Model/Correlation.php b/app/Model/Correlation.php index baaf98579..db441bb43 100644 --- a/app/Model/Correlation.php +++ b/app/Model/Correlation.php @@ -21,8 +21,21 @@ class Correlation extends AppModel ) ); + /** @var array */ private $exclusions; + /** + * Use old schema with `date` and `info` fields. + * @var bool + */ + private $oldSchema; + + public function __construct($id = false, $table = null, $ds = null) + { + parent::__construct($id, $table, $ds); + $this->oldSchema = $this->schema('date') !== null; + } + public function correlateValueRouter($value) { if (Configure::read('MISP.background_jobs')) { @@ -99,7 +112,7 @@ class Correlation extends AppModel ], 'contain' => [ 'Event' => [ - 'fields' => ['Event.id', 'Event.date', 'Event.info', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id', 'Event.disable_correlation'] + 'fields' => ['Event.id', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id', 'Event.disable_correlation'] ] ], 'order' => [], @@ -137,7 +150,7 @@ class Correlation extends AppModel ], 'contain' => [ 'Event' => [ - 'fields' => ['Event.id', 'Event.date', 'Event.info', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id', 'Event.disable_correlation'] + 'fields' => ['Event.id', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id', 'Event.disable_correlation'] ] ], 'order' => [], @@ -162,8 +175,6 @@ class Correlation extends AppModel 'a_distribution' => $b['Attribute']['distribution'], 'sharing_group_id' => $b['Event']['sharing_group_id'], 'a_sharing_group_id' => $b['Attribute']['sharing_group_id'], - 'date' => $b['Event']['date'], - 'info' => $b['Event']['info'] ]; } else { $correlations[] = [ @@ -177,8 +188,6 @@ class Correlation extends AppModel $b['Attribute']['distribution'], $b['Event']['sharing_group_id'], $b['Attribute']['sharing_group_id'], - $b['Event']['date'], - $b['Event']['info'] ]; } } @@ -228,9 +237,26 @@ class Correlation extends AppModel $fields = [ 'value', '1_event_id', '1_attribute_id', 'event_id', 'attribute_id', 'org_id', 'distribution', 'a_distribution', 'sharing_group_id', 'a_sharing_group_id', - 'date', 'info' ]; + + // In older MISP instances, correlations table contains also date and info columns, that stores information + // about correlated event title and date. But because this information can be fetched directly from Event table, + // it is not necessary to keep them there. The problem is that these columns are marked as not null, so they must + // be filled with value and removing these columns can take long time for big instances. So for new installation + // these columns doesn't exists anymore and we don't need to save dummy value into them. Also feel free to remove + // them from your instance. + if ($this->oldSchema) { + $fields[] = 'date'; + $fields[] = 'info'; + } + if (Configure::read('MISP.deadlock_avoidance')) { + if ($this->oldSchema) { + foreach ($correlations as &$correlation) { + $correlation['date'] = '1000-01-01'; // Dummy value + $correlation['info'] = ''; // Dummy value + } + } return $this->saveMany($correlations, array( 'atomic' => false, 'callbacks' => false, @@ -239,6 +265,12 @@ class Correlation extends AppModel 'fieldList' => $fields )); } else { + if ($this->oldSchema) { + foreach ($correlations as &$correlation) { + $correlation[] = '1000-01-01'; // Dummy value + $correlation[] = ''; // Dummy value + } + } $db = $this->getDataSource(); return $db->insertMulti('correlations', $fields, $correlations); } @@ -278,7 +310,7 @@ class Correlation extends AppModel if (!$event) { $event = $this->Attribute->Event->find('first', array( 'recursive' => -1, - 'fields' => array('Event.distribution', 'Event.id', 'Event.info', 'Event.org_id', 'Event.date', 'Event.sharing_group_id', 'Event.disable_correlation'), + 'fields' => array('Event.distribution', 'Event.id', 'Event.org_id', 'Event.sharing_group_id', 'Event.disable_correlation'), 'conditions' => array('id' => $a['event_id']), 'order' => array(), )); @@ -325,7 +357,7 @@ class Correlation extends AppModel 'Attribute.event_id', 'Attribute.id', 'Attribute.distribution', 'Attribute.sharing_group_id', 'Attribute.value1', 'Attribute.value2' ], - 'contain' => ['Event.id', 'Event.date', 'Event.info', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id'], + 'contain' => ['Event.id', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id'], 'order' => [] )); } diff --git a/db_schema.json b/db_schema.json index b031fec8e..708d71446 100644 --- a/db_schema.json +++ b/db_schema.json @@ -1010,28 +1010,6 @@ "column_type": "int(11)", "column_default": null, "extra": "" - }, - { - "column_name": "date", - "is_nullable": "NO", - "data_type": "date", - "character_maximum_length": null, - "numeric_precision": null, - "collation_name": null, - "column_type": "date", - "column_default": null, - "extra": "" - }, - { - "column_name": "info", - "is_nullable": "NO", - "data_type": "text", - "character_maximum_length": "65535", - "numeric_precision": null, - "collation_name": "utf8_bin", - "column_type": "text", - "column_default": null, - "extra": "" } ], "correlation_exclusions": [