MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

chg: [correlation] Allow to drop Correlation.{date,info} columns

pull/6021/head
Jakub Onderka 2021-04-29 11:47:38 +02:00
parent db1ff2496b
commit 6642d8bf7b
3 changed files with 41 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
INSTALL/MYSQL.sql
View File

@ -150,8 +150,6 @@ CREATE TABLE IF NOT EXISTS `correlations` (
`a_distribution` tinyint(4) NOT NULL,
`sharing_group_id` int(11) NOT NULL,
`a_sharing_group_id` int(11) NOT NULL,
`date` date NOT NULL,
`info` text COLLATE utf8_bin NOT NULL,
PRIMARY KEY (`id`),
INDEX `event_id` (`event_id`),
INDEX `1_event_id` (`1_event_id`),

50
app/Model/Correlation.php
View File

@ -21,8 +21,21 @@ class Correlation extends AppModel
)
);
/** @var array */
private $exclusions;
/**
* Use old schema with `date` and `info` fields.
* @var bool
*/
private $oldSchema;
public function __construct($id = false, $table = null, $ds = null)
{
parent::__construct($id, $table, $ds);
$this->oldSchema = $this->schema('date') !== null;
}
public function correlateValueRouter($value)
{
if (Configure::read('MISP.background_jobs')) {
@ -99,7 +112,7 @@ class Correlation extends AppModel
],
'contain' => [
'Event' => [
'fields' => ['Event.id', 'Event.date', 'Event.info', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id', 'Event.disable_correlation']
'fields' => ['Event.id', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id', 'Event.disable_correlation']
]
],
'order' => [],
@ -137,7 +150,7 @@ class Correlation extends AppModel
],
'contain' => [
'Event' => [
'fields' => ['Event.id', 'Event.date', 'Event.info', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id', 'Event.disable_correlation']
'fields' => ['Event.id', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id', 'Event.disable_correlation']
]
],
'order' => [],
@ -162,8 +175,6 @@ class Correlation extends AppModel
'a_distribution' => $b['Attribute']['distribution'],
'sharing_group_id' => $b['Event']['sharing_group_id'],
'a_sharing_group_id' => $b['Attribute']['sharing_group_id'],
'date' => $b['Event']['date'],
'info' => $b['Event']['info']
];
} else {
$correlations[] = [
@ -177,8 +188,6 @@ class Correlation extends AppModel
$b['Attribute']['distribution'],
$b['Event']['sharing_group_id'],
$b['Attribute']['sharing_group_id'],
$b['Event']['date'],
$b['Event']['info']
];
}
}
@ -228,9 +237,26 @@ class Correlation extends AppModel
$fields = [
'value', '1_event_id', '1_attribute_id', 'event_id', 'attribute_id', 'org_id',
'distribution', 'a_distribution', 'sharing_group_id', 'a_sharing_group_id',
'date', 'info'
];
// In older MISP instances, correlations table contains also date and info columns, that stores information
// about correlated event title and date. But because this information can be fetched directly from Event table,
// it is not necessary to keep them there. The problem is that these columns are marked as not null, so they must
// be filled with value and removing these columns can take long time for big instances. So for new installation
// these columns doesn't exists anymore and we don't need to save dummy value into them. Also feel free to remove
// them from your instance.
if ($this->oldSchema) {
$fields[] = 'date';
$fields[] = 'info';
}
if (Configure::read('MISP.deadlock_avoidance')) {
if ($this->oldSchema) {
foreach ($correlations as &$correlation) {
$correlation['date'] = '1000-01-01'; // Dummy value
$correlation['info'] = ''; // Dummy value
}
}
return $this->saveMany($correlations, array(
'atomic' => false,
'callbacks' => false,
@ -239,6 +265,12 @@ class Correlation extends AppModel
'fieldList' => $fields
));
} else {
if ($this->oldSchema) {
foreach ($correlations as &$correlation) {
$correlation[] = '1000-01-01'; // Dummy value
$correlation[] = ''; // Dummy value
}
}
$db = $this->getDataSource();
return $db->insertMulti('correlations', $fields, $correlations);
}
@ -278,7 +310,7 @@ class Correlation extends AppModel
if (!$event) {
$event = $this->Attribute->Event->find('first', array(
'recursive' => -1,
'fields' => array('Event.distribution', 'Event.id', 'Event.info', 'Event.org_id', 'Event.date', 'Event.sharing_group_id', 'Event.disable_correlation'),
'fields' => array('Event.distribution', 'Event.id', 'Event.org_id', 'Event.sharing_group_id', 'Event.disable_correlation'),
'conditions' => array('id' => $a['event_id']),
'order' => array(),
));
@ -325,7 +357,7 @@ class Correlation extends AppModel
'Attribute.event_id', 'Attribute.id', 'Attribute.distribution', 'Attribute.sharing_group_id',
'Attribute.value1', 'Attribute.value2'
],
'contain' => ['Event.id', 'Event.date', 'Event.info', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id'],
'contain' => ['Event.id', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id'],
'order' => []
));
}

22
db_schema.json
View File

@ -1010,28 +1010,6 @@
"column_type": "int(11)",
"column_default": null,
"extra": ""
},
{
"column_name": "date",
"is_nullable": "NO",
"data_type": "date",
"character_maximum_length": null,
"numeric_precision": null,
"collation_name": null,
"column_type": "date",
"column_default": null,
"extra": ""
},
{
"column_name": "info",
"is_nullable": "NO",
"data_type": "text",
"character_maximum_length": "65535",
"numeric_precision": null,
"collation_name": "utf8_bin",
"column_type": "text",
"column_default": null,
"extra": ""
}
],
"correlation_exclusions": [