chg: [overCorrelatingValue] Truncated the `value` column

- We keep the unique constraint on the table
- Correlating values over the max. allowed size are truncated to fit the size requirement. That means large correlating values might be marked as over-correlating despite the fact they are not (as only the starting portion of the value is evaluated).

app/Model/AppModel.php

@@ -83,7 +83,7 @@ class AppModel extends Model
         75 => false, 76 => true, 77 => false, 78 => false, 79 => false, 80 => false,
         81 => false, 82 => false, 83 => false, 84 => false, 85 => false, 86 => false,
         87 => false, 88 => false, 89 => false, 90 => false, 91 => false, 92 => false,
-        93 => false,
+        93 => false, 94 => false,
     );
 
     const ADVANCED_UPDATES_DESCRIPTION = array(
@@ -1851,6 +1851,10 @@ class AppModel extends Model
                 $this->__dropIndex('default_correlations', '1_event_sharing_group_id');
                 $this->__dropIndex('default_correlations', '1_org_id');
                 break;
+            case 94:
+                $sqlArray[] = "UPDATE `over_correlating_values` SET `value` = SUBSTR(`value`, 1, 191);"; // truncate then migrate
+                $sqlArray[] = "ALTER TABLE `over_correlating_values` MODIFY `value` varchar(191) NOT NULL;";
+                break;
             case 'fixNonEmptySharingGroupID':
                 $sqlArray[] = 'UPDATE `events` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';
                 $sqlArray[] = 'UPDATE `attributes` SET `sharing_group_id` = 0 WHERE `distribution` != 4;';

app/Model/Correlation.php

@@ -899,10 +899,7 @@ class Correlation extends AppModel
             }
         }
 
-        $overCorrelatingValues = array_flip($this->OverCorrelatingValue->find('column', [
-            'conditions' => ['value' => array_keys($valuesToCheck)],
-            'fields' => ['value'],
-        ]));
+        $overCorrelatingValues = array_flip($this->OverCorrelatingValue->findOverCorrelatingValues(array_keys($valuesToCheck)));
         unset($valuesToCheck);
 
         foreach ($attributes as &$attribute) {
@@ -911,6 +908,7 @@ class Correlation extends AppModel
             } else {
                 $values = [$attribute['value']];
             }
+            $values = $this->OverCorrelatingValue->truncateValues($values);
 
             if (isset($overCorrelatingValues[$values[0]])) {
                 $attribute['over_correlation'] = true;

app/Model/OverCorrelatingValue.php

@@ -9,6 +9,29 @@ class OverCorrelatingValue extends AppModel
         'Containable'
     );
 
+    public function beforeValidate($options = array())
+    {
+        $this->data['OverCorrelatingValue']['value'] = self::truncate($this->data['OverCorrelatingValue']['value']);
+        return true;
+    }
+
+    public function beforeSave($options = array())
+    {
+        $this->data['OverCorrelatingValue']['value'] = self::truncate($this->data['OverCorrelatingValue']['value']);
+        return true;
+    }
+
+    public static function truncate(string $value): string
+    {
+        return mb_substr($value, 0, 191);
+    }
+
+    public static function truncateValues(array $values): array
+    {
+        return array_map(function(string $value) {
+            return self::truncate($value);
+        }, $values);
+    }
 
     /**
      * @param string $value
@@ -36,7 +59,7 @@ class OverCorrelatingValue extends AppModel
     {
         $this->deleteAll(
             [
-                'OverCorrelatingValue.value' => $value
+                'OverCorrelatingValue.value' => self::truncate($value)
             ],
             false
         );
@@ -66,7 +89,17 @@ class OverCorrelatingValue extends AppModel
 
     public function checkValue($value)
     {
-        return $this->hasAny(['value' => $value]);
+        return $this->hasAny(['value' => self::truncate($value)]);
+    }
+
+    public function findOverCorrelatingValues(array $values_to_check): array
+    {
+        $values_to_check_truncated = array_unique(self::truncateValues($values_to_check));
+        $overCorrelatingValues = $this->find('column', [
+            'conditions' => ['value' => $values_to_check_truncated],
+            'fields' => ['value'],
+        ]);
+        return $overCorrelatingValues;
     }
 
     public function generateOccurrencesRouter()
@@ -110,8 +143,8 @@ class OverCorrelatingValue extends AppModel
                 'recursive' => -1,
                 'conditions' => [
                     'OR' => [
-                        'Attribute.value1' => $overCorrelation['OverCorrelatingValue']['value'],
-                        'Attribute.value2' => $overCorrelation['OverCorrelatingValue']['value']
+                        'Attribute.value1 LIKE' => $overCorrelation['OverCorrelatingValue']['value'] . '%',
+                        'Attribute.value2 LIKE' => $overCorrelation['OverCorrelatingValue']['value'] . '%'
                     ]
                 ]
             ]);