MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch '2.4' into develop 

Following discussion with Luciano and the finding of a logic for
the format string. We decided to go for Unicode ;-)
pull/8568/head
Alexandre Dulaunoy 2022-09-01 09:12:34 +02:00
parent 8b373570ad 7ce21f0425
commit 1e73697da2
No known key found for this signature in database
GPG Key ID: 09E2CD4944E6CBCD
5 changed files with 72 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
app/Model/AppModel.php
View File

@ -83,7 +83,7 @@ class AppModel extends Model
75 => false, 76 => true, 77 => false, 78 => false, 79 => false, 80 => false,
81 => false, 82 => false, 83 => false, 84 => false, 85 => false, 86 => false,
87 => false, 88 => false, 89 => false, 90 => false, 91 => false, 92 => false,
93 => false,
93 => false, 94 => false,
);
const ADVANCED_UPDATES_DESCRIPTION = array(
@ -243,6 +243,9 @@ class AppModel extends Model
"SHOW INDEX FROM default_correlations WHERE Key_name = 'unique_correlation';"
);
if (empty($existing_index)) {
// If there are duplicate entries, the query creating the `unique_correlation` index will result in an integrity constraint violation.
// The query below cleans up potential duplicates before creating the constraint.
$this->removeDuplicateCorrelationEntries('default_correlations');
$this->query(
"ALTER TABLE default_correlations
ADD CONSTRAINT unique_correlation
@ -253,6 +256,7 @@ class AppModel extends Model
"SHOW INDEX FROM no_acl_correlations WHERE Key_name = 'unique_correlation';"
);
if (empty($existing_index)) {
$this->removeDuplicateCorrelationEntries('no_acl_correlations');
$this->query(
"ALTER TABLE no_acl_correlations
ADD CONSTRAINT unique_correlation
@ -1851,6 +1855,10 @@ class AppModel extends Model
$this->__dropIndex('default_correlations', '1_event_sharing_group_id');
$this->__dropIndex('default_correlations', '1_org_id');
break;
case 94:
$sqlArray[] = "UPDATE `over_correlating_values` SET `value` = SUBSTR(`value`, 1, 191);"; // truncate then migrate
$sqlArray[] = "ALTER TABLE `over_correlating_values` MODIFY `value` varchar(191) NOT NULL;";
break;
case 'fixNonEmptySharingGroupID':
$sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
$sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
@ -3684,4 +3692,24 @@ class AppModel extends Model
);
}
}
public function removeDuplicateCorrelationEntries($table_name = 'default_correlations')
{
// If there are duplicate entries, the query creating the `unique_correlation` index will result in an integrity constraint violation.
// The query below cleans up potential duplicates before creating the constraint.
return $this->query("
DELETE FROM `$table_name` WHERE id in (
SELECT m_id FROM (
SELECT MAX(corr_a.id) as m_id, CONCAT(corr_a.attribute_id, \" - \", corr_a.1_attribute_id, \" - \", corr_a.value_id) as uniq FROM `$table_name` corr_a
INNER JOIN `$table_name` corr_b on corr_a.attribute_id = corr_b.attribute_id
WHERE
corr_a.attribute_id = corr_b.attribute_id AND
corr_a.1_attribute_id = corr_b.1_attribute_id AND
corr_a.value_id = corr_b.value_id AND
corr_a.id <> corr_b.id
GROUP BY uniq
) as c
);
");
}
}

6
app/Model/Correlation.php
View File

@ -899,10 +899,7 @@ class Correlation extends AppModel
}
}
$overCorrelatingValues = array_flip($this->OverCorrelatingValue->find('column', [
'conditions' => ['value' => array_keys($valuesToCheck)],
'fields' => ['value'],
]));
$overCorrelatingValues = array_flip($this->OverCorrelatingValue->findOverCorrelatingValues(array_keys($valuesToCheck)));
unset($valuesToCheck);
foreach ($attributes as &$attribute) {
@ -911,6 +908,7 @@ class Correlation extends AppModel
} else {
$values = [$attribute['value']];
}
$values = $this->OverCorrelatingValue->truncateValues($values);
if (isset($overCorrelatingValues[$values[0]])) {
$attribute['over_correlation'] = true;

41
app/Model/OverCorrelatingValue.php
View File

@ -9,6 +9,29 @@ class OverCorrelatingValue extends AppModel
'Containable'
);
public function beforeValidate($options = array())
{
$this->data['OverCorrelatingValue']['value'] = self::truncate($this->data['OverCorrelatingValue']['value']);
return true;
}
public function beforeSave($options = array())
{
$this->data['OverCorrelatingValue']['value'] = self::truncate($this->data['OverCorrelatingValue']['value']);
return true;
}
public static function truncate(string $value): string
{
return mb_substr($value, 0, 191);
}
public static function truncateValues(array $values): array
{
return array_map(function(string $value) {
return self::truncate($value);
}, $values);
}
/**
* @param string $value
@ -36,7 +59,7 @@ class OverCorrelatingValue extends AppModel
{
$this->deleteAll(
[
'OverCorrelatingValue.value' => $value
'OverCorrelatingValue.value' => self::truncate($value)
],
false
);
@ -66,7 +89,17 @@ class OverCorrelatingValue extends AppModel
public function checkValue($value)
{
return $this->hasAny(['value' => $value]);
return $this->hasAny(['value' => self::truncate($value)]);
}
public function findOverCorrelatingValues(array $values_to_check): array
{
$values_to_check_truncated = array_unique(self::truncateValues($values_to_check));
$overCorrelatingValues = $this->find('column', [
'conditions' => ['value' => $values_to_check_truncated],
'fields' => ['value'],
]);
return $overCorrelatingValues;
}
public function generateOccurrencesRouter()
@ -110,8 +143,8 @@ class OverCorrelatingValue extends AppModel
'recursive' => -1,
'conditions' => [
'OR' => [
'Attribute.value1' => $overCorrelation['OverCorrelatingValue']['value'],
'Attribute.value2' => $overCorrelation['OverCorrelatingValue']['value']
'Attribute.value1 LIKE' => $overCorrelation['OverCorrelatingValue']['value'] . '%',
'Attribute.value2 LIKE' => $overCorrelation['OverCorrelatingValue']['value'] . '%'
]
]
]);

2
app/webroot/doc/openapi.yaml
View File

@ -4524,7 +4524,7 @@ components:
properties:
input:
type: string
example: "cd $(git rev-parse --show-toplevel) && git checkout app\/composer.json 2>&1"
example: "cd $(git rev-parse --show-toplevel) && git checkout app/composer.json 2>&1"
output:
type: array
items:

6
db_schema.json
View File

@ -4868,9 +4868,9 @@
"data_type": "varchar",
"character_maximum_length": "191",
"numeric_precision": null,
"collation_name": "utf8mb4_general_ci",
"collation_name": "utf8mb4_unicode_ci",
"column_type": "varchar(191)",
"column_default": "NULL",
"column_default": null,
"extra": ""
},
{
@ -9106,4 +9106,4 @@
}
},
"db_version": "94"
}
}