From 5d4f73fcbbef577e39390cba0e38d7281de6cc4e Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Mon, 22 Aug 2022 16:07:40 +0200 Subject: [PATCH 1/7] fix: event block rules not working with tags filters, see issue #8551 --- app/Model/Event.php | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/Model/Event.php b/app/Model/Event.php index 6793a5d2f..aef88ac32 100755 --- a/app/Model/Event.php +++ b/app/Model/Event.php @@ -3343,16 +3343,16 @@ class Event extends AppModel if (empty($this->eventBlockRule)) { return true; } - if (!empty($rules['tags'])) { - if (!is_array($rules['tags'])) { - $rules['tags'] = [$rules['tags']]; + if (!empty($this->eventBlockRule['tags'])) { + if (!is_array($this->eventBlockRule['tags'])) { + $this->eventBlockRule['tags'] = [$this->eventBlockRule['tags']]; } $eventTags = Hash::extract($event, 'Event.Tag.{n}.name'); if (empty($eventTags)) { $eventTags = Hash::extract($event, 'Event.EventTag.{n}.Tag.name'); } if (!empty($eventTags)) { - foreach ($rules['tags'] as $blockTag) { + foreach ($this->eventBlockRule['tags'] as $blockTag) { if (in_array($blockTag, $eventTags)) { return false; } From 914e8bde8484f3b45e06a75174bf19763d371f7c Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Wed, 24 Aug 2022 09:43:05 +0200 Subject: [PATCH 2/7] chg: [misp-galaxy] updated to the latest version --- app/files/misp-galaxy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/files/misp-galaxy b/app/files/misp-galaxy index bfda561f5..b0ffb843b 160000 --- a/app/files/misp-galaxy +++ b/app/files/misp-galaxy @@ -1 +1 @@ -Subproject commit bfda561f5f29a8cca573e22789fb58252ad36c93 +Subproject commit b0ffb843b0bb69ea94d3ce9318f5123612b4ccc9 From 962754dd3bb43ccc7622a21b28ab3fb9297a4a70 Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 29 Aug 2022 10:50:59 +0200 Subject: [PATCH 3/7] chg: [overCorrelatingValue] Truncated the `value` column - We keep the unique constraint on the table - Correlating values over the max. allowed size are truncated to fit the size requirement. That means large correlating values might be marked as over-correlating despite the fact they are not (as only the starting portion of the value is evaluated). --- app/Model/AppModel.php | 6 ++++- app/Model/Correlation.php | 6 ++--- app/Model/OverCorrelatingValue.php | 41 +++++++++++++++++++++++++++--- 3 files changed, 44 insertions(+), 9 deletions(-) diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 018979088..2b86213a6 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -83,7 +83,7 @@ class AppModel extends Model 75 => false, 76 => true, 77 => false, 78 => false, 79 => false, 80 => false, 81 => false, 82 => false, 83 => false, 84 => false, 85 => false, 86 => false, 87 => false, 88 => false, 89 => false, 90 => false, 91 => false, 92 => false, - 93 => false, + 93 => false, 94 => false, ); const ADVANCED_UPDATES_DESCRIPTION = array( @@ -1851,6 +1851,10 @@ class AppModel extends Model $this->__dropIndex('default_correlations', '1_event_sharing_group_id'); $this->__dropIndex('default_correlations', '1_org_id'); break; + case 94: + $sqlArray[] = "UPDATE `over_correlating_values` SET `value` = SUBSTR(`value`, 1, 191);"; // truncate then migrate + $sqlArray[] = "ALTER TABLE `over_correlating_values` MODIFY `value` varchar(191) NOT NULL;"; + break; case 'fixNonEmptySharingGroupID': $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; diff --git a/app/Model/Correlation.php b/app/Model/Correlation.php index 29071b16e..61f11af35 100644 --- a/app/Model/Correlation.php +++ b/app/Model/Correlation.php @@ -899,10 +899,7 @@ class Correlation extends AppModel } } - $overCorrelatingValues = array_flip($this->OverCorrelatingValue->find('column', [ - 'conditions' => ['value' => array_keys($valuesToCheck)], - 'fields' => ['value'], - ])); + $overCorrelatingValues = array_flip($this->OverCorrelatingValue->findOverCorrelatingValues(array_keys($valuesToCheck))); unset($valuesToCheck); foreach ($attributes as &$attribute) { @@ -911,6 +908,7 @@ class Correlation extends AppModel } else { $values = [$attribute['value']]; } + $values = $this->OverCorrelatingValue->truncateValues($values); if (isset($overCorrelatingValues[$values[0]])) { $attribute['over_correlation'] = true; diff --git a/app/Model/OverCorrelatingValue.php b/app/Model/OverCorrelatingValue.php index 066076a7b..7bf75b29a 100644 --- a/app/Model/OverCorrelatingValue.php +++ b/app/Model/OverCorrelatingValue.php @@ -9,6 +9,29 @@ class OverCorrelatingValue extends AppModel 'Containable' ); + public function beforeValidate($options = array()) + { + $this->data['OverCorrelatingValue']['value'] = self::truncate($this->data['OverCorrelatingValue']['value']); + return true; + } + + public function beforeSave($options = array()) + { + $this->data['OverCorrelatingValue']['value'] = self::truncate($this->data['OverCorrelatingValue']['value']); + return true; + } + + public static function truncate(string $value): string + { + return mb_substr($value, 0, 191); + } + + public static function truncateValues(array $values): array + { + return array_map(function(string $value) { + return self::truncate($value); + }, $values); + } /** * @param string $value @@ -36,7 +59,7 @@ class OverCorrelatingValue extends AppModel { $this->deleteAll( [ - 'OverCorrelatingValue.value' => $value + 'OverCorrelatingValue.value' => self::truncate($value) ], false ); @@ -66,7 +89,17 @@ class OverCorrelatingValue extends AppModel public function checkValue($value) { - return $this->hasAny(['value' => $value]); + return $this->hasAny(['value' => self::truncate($value)]); + } + + public function findOverCorrelatingValues(array $values_to_check): array + { + $values_to_check_truncated = array_unique(self::truncateValues($values_to_check)); + $overCorrelatingValues = $this->find('column', [ + 'conditions' => ['value' => $values_to_check_truncated], + 'fields' => ['value'], + ]); + return $overCorrelatingValues; } public function generateOccurrencesRouter() @@ -110,8 +143,8 @@ class OverCorrelatingValue extends AppModel 'recursive' => -1, 'conditions' => [ 'OR' => [ - 'Attribute.value1' => $overCorrelation['OverCorrelatingValue']['value'], - 'Attribute.value2' => $overCorrelation['OverCorrelatingValue']['value'] + 'Attribute.value1 LIKE' => $overCorrelation['OverCorrelatingValue']['value'] . '%', + 'Attribute.value2 LIKE' => $overCorrelation['OverCorrelatingValue']['value'] . '%' ] ] ]); From 05ad1057506efa07c454b5ff00c92e13f46f16ff Mon Sep 17 00:00:00 2001 From: Sami Mokaddem Date: Mon, 29 Aug 2022 12:12:39 +0200 Subject: [PATCH 4/7] fix: [update-91] Remove duplicates before creating the constraint --- app/Model/AppModel.php | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 018979088..e2827f6d6 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -243,6 +243,9 @@ class AppModel extends Model "SHOW INDEX FROM default_correlations WHERE Key_name = 'unique_correlation';" ); if (empty($existing_index)) { + // If there are duplicate entries, the query creating the `unique_correlation` index will result in an integrity constraint violation. + // The query below cleans up potential duplicates before creating the constraint. + $this->removeDuplicateCorrelationEntries('default_correlations'); $this->query( "ALTER TABLE default_correlations ADD CONSTRAINT unique_correlation @@ -253,6 +256,7 @@ class AppModel extends Model "SHOW INDEX FROM no_acl_correlations WHERE Key_name = 'unique_correlation';" ); if (empty($existing_index)) { + $this->removeDuplicateCorrelationEntries('no_acl_correlations'); $this->query( "ALTER TABLE no_acl_correlations ADD CONSTRAINT unique_correlation @@ -3684,4 +3688,24 @@ class AppModel extends Model ); } } + + public function removeDuplicateCorrelationEntries($table_name = 'default_correlations') + { + // If there are duplicate entries, the query creating the `unique_correlation` index will result in an integrity constraint violation. + // The query below cleans up potential duplicates before creating the constraint. + return $this->query(" + DELETE FROM `$table_name` WHERE id in ( + SELECT m_id FROM ( + SELECT MAX(corr_a.id) as m_id, CONCAT(corr_a.attribute_id, \" - \", corr_a.1_attribute_id, \" - \", corr_a.value_id) as uniq FROM `$table_name` corr_a + INNER JOIN `$table_name` corr_b on corr_a.attribute_id = corr_b.attribute_id + WHERE + corr_a.attribute_id = corr_b.attribute_id AND + corr_a.1_attribute_id = corr_b.1_attribute_id AND + corr_a.value_id = corr_b.value_id AND + corr_a.id <> corr_b.id + GROUP BY uniq + ) as c + ); + "); + } } From 896c18e66496aacaa1473d99519c633cd638613d Mon Sep 17 00:00:00 2001 From: Graham Williamson Date: Tue, 30 Aug 2022 22:40:36 +1000 Subject: [PATCH 5/7] fix: Removes unnecessary escape character Fixes a validation error - found unknown escape character --- app/webroot/doc/openapi.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/webroot/doc/openapi.yaml b/app/webroot/doc/openapi.yaml index 25f644713..37fbb5bef 100644 --- a/app/webroot/doc/openapi.yaml +++ b/app/webroot/doc/openapi.yaml @@ -4524,7 +4524,7 @@ components: properties: input: type: string - example: "cd $(git rev-parse --show-toplevel) && git checkout app\/composer.json 2>&1" + example: "cd $(git rev-parse --show-toplevel) && git checkout app/composer.json 2>&1" output: type: array items: From 687a558c0d70d7d9d3bc1b03b2667b60e1179198 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Wed, 31 Aug 2022 08:29:00 +0200 Subject: [PATCH 6/7] fix: bump db version and fix schema --- db_schema.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/db_schema.json b/db_schema.json index 10371914d..de43e9ffe 100644 --- a/db_schema.json +++ b/db_schema.json @@ -4864,12 +4864,12 @@ }, { "column_name": "value", - "is_nullable": "YES", - "data_type": "text", - "character_maximum_length": "65535", + "is_nullable": "NO", + "data_type": "varchar", + "character_maximum_length": "191", "numeric_precision": null, - "collation_name": "utf8mb4_unicode_ci", - "column_type": "text", + "collation_name": "utf8mb4_general_ci", + "column_type": "varchar(191)", "column_default": "NULL", "extra": "" }, @@ -9105,5 +9105,5 @@ "uuid": false } }, - "db_version": "93" + "db_version": "94" } \ No newline at end of file From 47674a80511f058ea35fff0a423b62d69e7139f9 Mon Sep 17 00:00:00 2001 From: Luciano Righetti Date: Wed, 31 Aug 2022 11:19:18 +0200 Subject: [PATCH 7/7] fix: schema inconsistency --- db_schema.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db_schema.json b/db_schema.json index de43e9ffe..0df1b959a 100644 --- a/db_schema.json +++ b/db_schema.json @@ -4868,9 +4868,9 @@ "data_type": "varchar", "character_maximum_length": "191", "numeric_precision": null, - "collation_name": "utf8mb4_general_ci", + "collation_name": "utf8mb4_unicode_ci", "column_type": "varchar(191)", - "column_default": "NULL", + "column_default": null, "extra": "" }, {