Major speed boost to the correlation

- it seems that for some reason some conditions in the correlation lookup massacred the performance of the correlation
- doing that additional filter on a PHP level fixes it for now, but it would be interesting to investigate this further and potentially reuse the findings to improve other queries

- also fixed an issue with the indexing script failing on some fulltext fields if it has to fall back to regular indeces.

app/Model/AppModel.php

@@ -128,13 +128,13 @@ class AppModel extends Model {
 				break;
 			case 'indexTables':
 				$fieldsToIndex = array(
-					'attributes' => array(array('value1', 'FULLTEXT'), array('value2', 'FULLTEXT'), array('event_id', 'INDEX'), array('sharing_group_id', 'INDEX'), array('uuid', 'INDEX')),
+					'attributes' => array(array('value1', 'INDEX', '255'), array('value2', 'INDEX', '255'), array('event_id', 'INDEX'), array('sharing_group_id', 'INDEX'), array('uuid', 'INDEX')),
-					'correlations' =>  array(array('org_id', 'INDEX'), array('event_id', 'INDEX'), array('attribute_id', 'INDEX'), array('sharing_group_id', 'INDEX'), array('1_event_id', 'INDEX'), array('1_attribute_id', 'INDEX'), array('a_sharing_group_id', 'INDEX'), array('org_id', 'INDEX'), array('value', 'FULLTEXT')),
+					'correlations' =>  array(array('org_id', 'INDEX'), array('event_id', 'INDEX'), array('attribute_id', 'INDEX'), array('sharing_group_id', 'INDEX'), array('1_event_id', 'INDEX'), array('1_attribute_id', 'INDEX'), array('a_sharing_group_id', 'INDEX'), array('value', 'FULLTEXT')),
 					'events' => array(array('info', 'FULLTEXT'), array('sharing_group_id', 'INDEX'), array('org_id', 'INDEX'), array('orgc_id', 'INDEX'), array('uuid', 'INDEX')),
 					'event_tags' => array(array('event_id', 'INDEX'), array('tag_id', 'INDEX')),
 					'organisations' => array(array('uuid', 'INDEX'), array('name', 'FULLTEXT')),
 					'posts' => array(array('post_id', 'INDEX'), array('thread_id', 'INDEX')),
-					'shadow_attributes' => array(array('value1', 'FULLTEXT'), array('value2', 'FULLTEXT'), array('old_id', 'INDEX'), array('event_id', 'INDEX'), array('uuid', 'INDEX'), array('event_org_id', 'INDEX'), array('event_uuid', 'INDEX')),
+					'shadow_attributes' => array(array('value1', 'INDEX', '255'), array('value2', 'INDEX', '255'), array('old_id', 'INDEX'), array('event_id', 'INDEX'), array('uuid', 'INDEX'), array('event_org_id', 'INDEX'), array('event_uuid', 'INDEX')),
 					'sharing_groups' => array(array('org_id', 'INDEX'), array('sync_user_id', 'INDEX'), array('uuid', 'INDEX'), array('organisation_uuid', 'INDEX')),
 					'sharing_group_orgs' => array(array('sharing_group_id', 'INDEX'), array('org_id', 'INDEX')),
 					'sharing_group_servers' => array(array('sharing_group_id', 'INDEX'), array('server_id', 'INDEX')),
@@ -159,7 +159,10 @@ class AppModel extends Model {
 					$table_data = $this->query("SHOW TABLE STATUS WHERE Name = '" . $table . "'");
 					if ($downgrade && $table_data[0]['TABLES']['Engine'] !== 'MyISAM') $downgradeThis = true;
 					foreach ($fields as $field) {
-						$sqlArray[] = 'ALTER TABLE `' . $table . '` ADD ' . ($downgradeThis ? 'INDEX' : $field[1]) . ' `' . $field[0] . '` (`' . $field[0] . '`)';
+						$extra = '';
+						$this->__dropIndex($table, $field[0]);
+						if (isset($field[2])) $extra = ' (' . $field[2] . ')';
+						$sqlArray[] = 'ALTER TABLE `' . $table . '` ADD ' . ($downgradeThis ? 'INDEX' : $field[1]) . ' `' . $field[0] . '` (`' . $field[0] . '`' . $extra . ')';
 					}
 				}
 				break;

app/Model/Attribute.php

@@ -1133,20 +1133,18 @@ class Attribute extends AppModel {
 			$fields = array('value1', 'value2');
 			$correlatingValues = array($a['value1']);
 			if (!empty($a['value2'])) $correlatingValues[] = $a['value2'];
-			if ($full) $temp = array('Attribute.id >' => $a['id']);
-			else $temp = array();
 			foreach ($correlatingValues as $k => $cV) {
 				$correlatingAttributes[$k] = $this->find('all', array(
 						'conditions' => array(
-							'OR' => array(
-								'Attribute.value1' => $cV,
-								'Attribute.value2' => $cV
-							),
 							'AND' => array(
+								'OR' => array(
+										'Attribute.value1' => $cV,
+										'Attribute.value2' => $cV
+								),
 								'Attribute.type !=' => $this->nonCorrelatingTypes,
-								'Attribute.id !=' => $a['id'],
+								//'Attribute.id !=' => $a['id'],
-								$temp,
+								//$temp,
-								'Attribute.event_id !=' => $a['event_id']
+								//'Attribute.event_id !=' => $a['event_id']
 							),
 						),
 						'recursive => -1',
@@ -1154,6 +1152,11 @@ class Attribute extends AppModel {
 						'contain' => array('Event' => array('fields' => array('Event.id', 'Event.date', 'Event.info', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id'))),
 						'order' => array(),
 				));
+				foreach ($correlatingAttributes[$k] as $key => &$correlatingAttribute) {
+					if ($correlatingAttribute['Attribute']['id'] == $a['id']) unset($correlatingAttributes[$k][$key]);
+					else if ($correlatingAttribute['Attribute']['event_id'] == $a['event_id']) unset($correlatingAttributes[$k][$key]);
+					else if ($full && $correlatingAttribute['Attribute']['id'] <= $a['id']) unset($correlatingAttributes[$k][$key]);
+				}
 			}
 			$correlations = array();
 			foreach ($correlatingAttributes as $k => $cA) {