diff --git a/app/Model/AppModel.php b/app/Model/AppModel.php index 007d12572..e495d57c6 100644 --- a/app/Model/AppModel.php +++ b/app/Model/AppModel.php @@ -83,7 +83,7 @@ class AppModel extends Model 75 => false, 76 => true, 77 => false, 78 => false, 79 => false, 80 => false, 81 => false, 82 => false, 83 => false, 84 => false, 85 => false, 86 => false, 87 => false, 88 => false, 89 => false, 90 => false, 91 => false, 92 => false, - 93 => false, 94 => false, + 93 => false, 94 => false, 95 => true, ); const ADVANCED_UPDATES_DESCRIPTION = array( @@ -243,6 +243,9 @@ class AppModel extends Model "SHOW INDEX FROM default_correlations WHERE Key_name = 'unique_correlation';" ); if (empty($existing_index)) { + // If there are duplicate entries, the query creating the `unique_correlation` index will result in an integrity constraint violation. + // The query below cleans up potential duplicates before creating the constraint. + $this->removeDuplicateCorrelationEntries('default_correlations'); $this->query( "ALTER TABLE default_correlations ADD CONSTRAINT unique_correlation @@ -253,6 +256,7 @@ class AppModel extends Model "SHOW INDEX FROM no_acl_correlations WHERE Key_name = 'unique_correlation';" ); if (empty($existing_index)) { + $this->removeDuplicateCorrelationEntries('no_acl_correlations'); $this->query( "ALTER TABLE no_acl_correlations ADD CONSTRAINT unique_correlation @@ -1852,12 +1856,16 @@ class AppModel extends Model $this->__dropIndex('default_correlations', '1_org_id'); break; case 94: + $sqlArray[] = "UPDATE `over_correlating_values` SET `value` = SUBSTR(`value`, 1, 191);"; // truncate then migrate + $sqlArray[] = "ALTER TABLE `over_correlating_values` MODIFY `value` varchar(191) NOT NULL;"; + break; + case 95: $sqlArray[] = "ALTER TABLE `users` ADD COLUMN `notification_daily` tinyint(1) NOT NULL DEFAULT 0, ADD COLUMN `notification_weekly` tinyint(1) NOT NULL DEFAULT 0, ADD COLUMN `notification_monthly` tinyint(1) NOT NULL DEFAULT 0 ;"; - break; + break; case 'fixNonEmptySharingGroupID': $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;'; @@ -3691,4 +3699,24 @@ class AppModel extends Model ); } } + + public function removeDuplicateCorrelationEntries($table_name = 'default_correlations') + { + // If there are duplicate entries, the query creating the `unique_correlation` index will result in an integrity constraint violation. + // The query below cleans up potential duplicates before creating the constraint. + return $this->query(" + DELETE FROM `$table_name` WHERE id in ( + SELECT m_id FROM ( + SELECT MAX(corr_a.id) as m_id, CONCAT(corr_a.attribute_id, \" - \", corr_a.1_attribute_id, \" - \", corr_a.value_id) as uniq FROM `$table_name` corr_a + INNER JOIN `$table_name` corr_b on corr_a.attribute_id = corr_b.attribute_id + WHERE + corr_a.attribute_id = corr_b.attribute_id AND + corr_a.1_attribute_id = corr_b.1_attribute_id AND + corr_a.value_id = corr_b.value_id AND + corr_a.id <> corr_b.id + GROUP BY uniq + ) as c + ); + "); + } } diff --git a/app/Model/Correlation.php b/app/Model/Correlation.php index 29071b16e..61f11af35 100644 --- a/app/Model/Correlation.php +++ b/app/Model/Correlation.php @@ -899,10 +899,7 @@ class Correlation extends AppModel } } - $overCorrelatingValues = array_flip($this->OverCorrelatingValue->find('column', [ - 'conditions' => ['value' => array_keys($valuesToCheck)], - 'fields' => ['value'], - ])); + $overCorrelatingValues = array_flip($this->OverCorrelatingValue->findOverCorrelatingValues(array_keys($valuesToCheck))); unset($valuesToCheck); foreach ($attributes as &$attribute) { @@ -911,6 +908,7 @@ class Correlation extends AppModel } else { $values = [$attribute['value']]; } + $values = $this->OverCorrelatingValue->truncateValues($values); if (isset($overCorrelatingValues[$values[0]])) { $attribute['over_correlation'] = true; diff --git a/app/Model/OverCorrelatingValue.php b/app/Model/OverCorrelatingValue.php index 066076a7b..7bf75b29a 100644 --- a/app/Model/OverCorrelatingValue.php +++ b/app/Model/OverCorrelatingValue.php @@ -9,6 +9,29 @@ class OverCorrelatingValue extends AppModel 'Containable' ); + public function beforeValidate($options = array()) + { + $this->data['OverCorrelatingValue']['value'] = self::truncate($this->data['OverCorrelatingValue']['value']); + return true; + } + + public function beforeSave($options = array()) + { + $this->data['OverCorrelatingValue']['value'] = self::truncate($this->data['OverCorrelatingValue']['value']); + return true; + } + + public static function truncate(string $value): string + { + return mb_substr($value, 0, 191); + } + + public static function truncateValues(array $values): array + { + return array_map(function(string $value) { + return self::truncate($value); + }, $values); + } /** * @param string $value @@ -36,7 +59,7 @@ class OverCorrelatingValue extends AppModel { $this->deleteAll( [ - 'OverCorrelatingValue.value' => $value + 'OverCorrelatingValue.value' => self::truncate($value) ], false ); @@ -66,7 +89,17 @@ class OverCorrelatingValue extends AppModel public function checkValue($value) { - return $this->hasAny(['value' => $value]); + return $this->hasAny(['value' => self::truncate($value)]); + } + + public function findOverCorrelatingValues(array $values_to_check): array + { + $values_to_check_truncated = array_unique(self::truncateValues($values_to_check)); + $overCorrelatingValues = $this->find('column', [ + 'conditions' => ['value' => $values_to_check_truncated], + 'fields' => ['value'], + ]); + return $overCorrelatingValues; } public function generateOccurrencesRouter() @@ -110,8 +143,8 @@ class OverCorrelatingValue extends AppModel 'recursive' => -1, 'conditions' => [ 'OR' => [ - 'Attribute.value1' => $overCorrelation['OverCorrelatingValue']['value'], - 'Attribute.value2' => $overCorrelation['OverCorrelatingValue']['value'] + 'Attribute.value1 LIKE' => $overCorrelation['OverCorrelatingValue']['value'] . '%', + 'Attribute.value2 LIKE' => $overCorrelation['OverCorrelatingValue']['value'] . '%' ] ] ]); diff --git a/app/webroot/doc/openapi.yaml b/app/webroot/doc/openapi.yaml index 36c016bd5..4d31d3179 100644 --- a/app/webroot/doc/openapi.yaml +++ b/app/webroot/doc/openapi.yaml @@ -4524,7 +4524,7 @@ components: properties: input: type: string - example: "cd $(git rev-parse --show-toplevel) && git checkout app\/composer.json 2>&1" + example: "cd $(git rev-parse --show-toplevel) && git checkout app/composer.json 2>&1" output: type: array items: diff --git a/db_schema.json b/db_schema.json index de43e9ffe..0df1b959a 100644 --- a/db_schema.json +++ b/db_schema.json @@ -4868,9 +4868,9 @@ "data_type": "varchar", "character_maximum_length": "191", "numeric_precision": null, - "collation_name": "utf8mb4_general_ci", + "collation_name": "utf8mb4_unicode_ci", "column_type": "varchar(191)", - "column_default": "NULL", + "column_default": null, "extra": "" }, {