mirror of https://github.com/MISP/MISP
Major speed boost to the correlation
- it seems that for some reason some conditions in the correlation lookup massacred the performance of the correlation - doing that additional filter on a PHP level fixes it for now, but it would be interesting to investigate this further and potentially reuse the findings to improve other queries - also fixed an issue with the indexing script failing on some fulltext fields if it has to fall back to regular indeces.pull/903/head
parent
7f97a49b45
commit
97533ab272
|
@ -128,13 +128,13 @@ class AppModel extends Model {
|
|||
break;
|
||||
case 'indexTables':
|
||||
$fieldsToIndex = array(
|
||||
'attributes' => array(array('value1', 'FULLTEXT'), array('value2', 'FULLTEXT'), array('event_id', 'INDEX'), array('sharing_group_id', 'INDEX'), array('uuid', 'INDEX')),
|
||||
'correlations' => array(array('org_id', 'INDEX'), array('event_id', 'INDEX'), array('attribute_id', 'INDEX'), array('sharing_group_id', 'INDEX'), array('1_event_id', 'INDEX'), array('1_attribute_id', 'INDEX'), array('a_sharing_group_id', 'INDEX'), array('org_id', 'INDEX'), array('value', 'FULLTEXT')),
|
||||
'attributes' => array(array('value1', 'INDEX', '255'), array('value2', 'INDEX', '255'), array('event_id', 'INDEX'), array('sharing_group_id', 'INDEX'), array('uuid', 'INDEX')),
|
||||
'correlations' => array(array('org_id', 'INDEX'), array('event_id', 'INDEX'), array('attribute_id', 'INDEX'), array('sharing_group_id', 'INDEX'), array('1_event_id', 'INDEX'), array('1_attribute_id', 'INDEX'), array('a_sharing_group_id', 'INDEX'), array('value', 'FULLTEXT')),
|
||||
'events' => array(array('info', 'FULLTEXT'), array('sharing_group_id', 'INDEX'), array('org_id', 'INDEX'), array('orgc_id', 'INDEX'), array('uuid', 'INDEX')),
|
||||
'event_tags' => array(array('event_id', 'INDEX'), array('tag_id', 'INDEX')),
|
||||
'organisations' => array(array('uuid', 'INDEX'), array('name', 'FULLTEXT')),
|
||||
'posts' => array(array('post_id', 'INDEX'), array('thread_id', 'INDEX')),
|
||||
'shadow_attributes' => array(array('value1', 'FULLTEXT'), array('value2', 'FULLTEXT'), array('old_id', 'INDEX'), array('event_id', 'INDEX'), array('uuid', 'INDEX'), array('event_org_id', 'INDEX'), array('event_uuid', 'INDEX')),
|
||||
'shadow_attributes' => array(array('value1', 'INDEX', '255'), array('value2', 'INDEX', '255'), array('old_id', 'INDEX'), array('event_id', 'INDEX'), array('uuid', 'INDEX'), array('event_org_id', 'INDEX'), array('event_uuid', 'INDEX')),
|
||||
'sharing_groups' => array(array('org_id', 'INDEX'), array('sync_user_id', 'INDEX'), array('uuid', 'INDEX'), array('organisation_uuid', 'INDEX')),
|
||||
'sharing_group_orgs' => array(array('sharing_group_id', 'INDEX'), array('org_id', 'INDEX')),
|
||||
'sharing_group_servers' => array(array('sharing_group_id', 'INDEX'), array('server_id', 'INDEX')),
|
||||
|
@ -159,7 +159,10 @@ class AppModel extends Model {
|
|||
$table_data = $this->query("SHOW TABLE STATUS WHERE Name = '" . $table . "'");
|
||||
if ($downgrade && $table_data[0]['TABLES']['Engine'] !== 'MyISAM') $downgradeThis = true;
|
||||
foreach ($fields as $field) {
|
||||
$sqlArray[] = 'ALTER TABLE `' . $table . '` ADD ' . ($downgradeThis ? 'INDEX' : $field[1]) . ' `' . $field[0] . '` (`' . $field[0] . '`)';
|
||||
$extra = '';
|
||||
$this->__dropIndex($table, $field[0]);
|
||||
if (isset($field[2])) $extra = ' (' . $field[2] . ')';
|
||||
$sqlArray[] = 'ALTER TABLE `' . $table . '` ADD ' . ($downgradeThis ? 'INDEX' : $field[1]) . ' `' . $field[0] . '` (`' . $field[0] . '`' . $extra . ')';
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
|
|
@ -1133,20 +1133,18 @@ class Attribute extends AppModel {
|
|||
$fields = array('value1', 'value2');
|
||||
$correlatingValues = array($a['value1']);
|
||||
if (!empty($a['value2'])) $correlatingValues[] = $a['value2'];
|
||||
if ($full) $temp = array('Attribute.id >' => $a['id']);
|
||||
else $temp = array();
|
||||
foreach ($correlatingValues as $k => $cV) {
|
||||
$correlatingAttributes[$k] = $this->find('all', array(
|
||||
'conditions' => array(
|
||||
'OR' => array(
|
||||
'Attribute.value1' => $cV,
|
||||
'Attribute.value2' => $cV
|
||||
),
|
||||
'AND' => array(
|
||||
'OR' => array(
|
||||
'Attribute.value1' => $cV,
|
||||
'Attribute.value2' => $cV
|
||||
),
|
||||
'Attribute.type !=' => $this->nonCorrelatingTypes,
|
||||
'Attribute.id !=' => $a['id'],
|
||||
$temp,
|
||||
'Attribute.event_id !=' => $a['event_id']
|
||||
//'Attribute.id !=' => $a['id'],
|
||||
//$temp,
|
||||
//'Attribute.event_id !=' => $a['event_id']
|
||||
),
|
||||
),
|
||||
'recursive => -1',
|
||||
|
@ -1154,6 +1152,11 @@ class Attribute extends AppModel {
|
|||
'contain' => array('Event' => array('fields' => array('Event.id', 'Event.date', 'Event.info', 'Event.org_id', 'Event.distribution', 'Event.sharing_group_id'))),
|
||||
'order' => array(),
|
||||
));
|
||||
foreach ($correlatingAttributes[$k] as $key => &$correlatingAttribute) {
|
||||
if ($correlatingAttribute['Attribute']['id'] == $a['id']) unset($correlatingAttributes[$k][$key]);
|
||||
else if ($correlatingAttribute['Attribute']['event_id'] == $a['event_id']) unset($correlatingAttributes[$k][$key]);
|
||||
else if ($full && $correlatingAttribute['Attribute']['id'] <= $a['id']) unset($correlatingAttributes[$k][$key]);
|
||||
}
|
||||
}
|
||||
$correlations = array();
|
||||
foreach ($correlatingAttributes as $k => $cA) {
|
||||
|
|
Loading…
Reference in New Issue