MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

new: [diagnostic] Check if database index is unique

pull/6254/head
Jakub Onderka 2020-08-30 11:47:57 +02:00
parent 99d910fe75
commit 6f12dfc7df
3 changed files with 530 additions and 434 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

158
app/Model/Server.php
View File

@ -4752,9 +4752,9 @@ class Server extends AppModel
$dbActualIndexes = array(); $dbActualIndexes = array();
$dataSource = $this->getDataSource()->config['datasource']; $dataSource = $this->getDataSource()->config['datasource'];
if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') { if ($dataSource == 'Database/Mysql' || $dataSource == 'Database/MysqlObserver') {
$sqlGetTable = sprintf('SELECT TABLE_NAME FROM information_schema.tables WHERE table_schema = %s;', "'" . $this->getDataSource()->config['database'] . "'"); $sqlGetTable = sprintf('SELECT TABLE_NAME FROM information_schema.tables WHERE table_schema = %s ORDER BY TABLE_NAME;', "'" . $this->getDataSource()->config['database'] . "'");
$sqlResult = $this->query($sqlGetTable); $sqlResult = $this->query($sqlGetTable);
$tables = HASH::extract($sqlResult, '{n}.tables.TABLE_NAME'); $tables = Hash::extract($sqlResult, '{n}.tables.TABLE_NAME');
foreach ($tables as $table) { foreach ($tables as $table) {
$sqlSchema = sprintf( $sqlSchema = sprintf(
"SELECT %s "SELECT %s
@ -4878,53 +4878,120 @@ class Server extends AppModel
return $dbDiff; return $dbDiff;
} }
public function compareDBIndexes($actualIndex, $expectedIndex, $dbExpectedSchema) /**
* Returns `true` if given column for given table contains just unique values.
*
* @param string $tableName
* @param string $columnName
* @return bool
*/
private function checkIfColumnContainsJustUniqueValues($tableName, $columnName)
{
$db = $this->getDataSource();
$duplicates = $this->query(
sprintf('SELECT %s, COUNT(*) c FROM %s GROUP BY %s HAVING c > 1;',
$db->name($columnName), $db->name($tableName), $db->name($columnName))
);
return empty($duplicates);
}
private function generateSqlDropIndexQuery($tableName, $columnName)
{
return sprintf('DROP INDEX `%s` ON %s;',
$columnName,
$tableName
);
}
private function generateSqlIndexQuery(array $dbExpectedSchema, $tableName, $columnName, $shouldBeUnique = false, $defaultIndexKeylength = 255)
{
$columnData = Hash::extract($dbExpectedSchema['schema'][$tableName], "{n}[column_name=$columnName]");
if (empty($columnData)) {
throw new Exception("Index in db_schema.json is defined for `$tableName.$columnName`, but this column is not defined.");
}
$columnData = $columnData[0];
if ($columnData['data_type'] === 'varchar') {
$keyLength = sprintf('(%s)', $columnData['character_maximum_length'] < $defaultIndexKeylength ? $columnData['character_maximum_length'] : $defaultIndexKeylength);
} elseif ($columnData['data_type'] === 'text') {
$keyLength = sprintf('(%s)', $defaultIndexKeylength);
} else {
$keyLength = '';
}
return sprintf('CREATE%s INDEX `%s` ON `%s` (`%s`%s);',
$shouldBeUnique ? ' UNIQUE' : '',
$columnName,
$tableName,
$columnName,
$keyLength
);
}
public function compareDBIndexes(array $actualIndex, array $expectedIndex, array $dbExpectedSchema)
{ {
$defaultIndexKeylength = 255;
$allowedlistTables = array(); $allowedlistTables = array();
$indexDiff = array(); $indexDiff = array();
foreach($expectedIndex as $tableName => $indexes) { foreach ($expectedIndex as $tableName => $indexes) {
if (!array_key_exists($tableName, $actualIndex)) { if (!array_key_exists($tableName, $actualIndex)) {
continue; // If table does not exists, it is covered by the schema diagnostic continue; // If table does not exists, it is covered by the schema diagnostic
} elseif(in_array($tableName, $allowedlistTables)) { } elseif(in_array($tableName, $allowedlistTables)) {
continue; // Ignore allowedlisted tables continue; // Ignore allowedlisted tables
} else { } else {
$tableIndexDiff = array_diff($indexes, $actualIndex[$tableName]); // check for missing indexes $tableIndexDiff = array_diff(array_keys($indexes), array_keys($actualIndex[$tableName])); // check for missing indexes
if (count($tableIndexDiff) > 0) { foreach ($tableIndexDiff as $columnDiff) {
foreach($tableIndexDiff as $columnDiff) { $shouldBeUnique = $indexes[$columnDiff];
$columnData = Hash::extract($dbExpectedSchema['schema'][$tableName], sprintf('{n}[column_name=%s]', $columnDiff))[0]; if ($shouldBeUnique && !$this->checkIfColumnContainsJustUniqueValues($tableName, $columnDiff)) {
$message = sprintf(__('Column `%s` should be indexed'), $columnDiff);
if ($columnData['data_type'] == 'varchar') {
$keyLength = sprintf('(%s)', $columnData['character_maximum_length'] < $defaultIndexKeylength ? $columnData['character_maximum_length'] : $defaultIndexKeylength);
} elseif ($columnData['data_type'] == 'text') {
$keyLength = sprintf('(%s)', $defaultIndexKeylength);
} else {
$keyLength = '';
}
$sql = sprintf('CREATE INDEX `%s` ON `%s` (`%s`%s);',
$columnDiff,
$tableName,
$columnDiff,
$keyLength
);
$indexDiff[$tableName][$columnDiff] = array( $indexDiff[$tableName][$columnDiff] = array(
'message' => $message, 'message' => __('Column `%s` should be unique indexed, but contains duplicate values', $columnDiff),
'sql' => $sql 'sql' => '',
); );
continue;
} }
$message = __('Column `%s` should be indexed', $columnDiff);
$indexDiff[$tableName][$columnDiff] = array(
'message' => $message,
'sql' => $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $columnDiff, $shouldBeUnique),
);
} }
$tableIndexDiff = array_diff($actualIndex[$tableName], $indexes); // check for additional indexes $tableIndexDiff = array_diff(array_keys($actualIndex[$tableName]), array_keys($indexes)); // check for additional indexes
if (count($tableIndexDiff) > 0) { foreach ($tableIndexDiff as $columnDiff) {
foreach($tableIndexDiff as $columnDiff) { $message = __('Column `%s` is indexed but should not', $columnDiff);
$message = sprintf(__('Column `%s` is indexed but should not'), $columnDiff); $indexDiff[$tableName][$columnDiff] = array(
$sql = sprintf('DROP INDEX `%s` ON %s;', 'message' => $message,
$columnDiff, 'sql' => $this->generateSqlDropIndexQuery($tableName, $columnDiff),
$tableName );
); }
$indexDiff[$tableName][$columnDiff] = array( foreach ($indexes as $column => $unique) {
'message' => $message, if (isset($actualIndex[$tableName][$column]) && $actualIndex[$tableName][$column] != $unique) {
'sql' => $sql if ($actualIndex[$tableName][$column]) {
); $sql = $this->generateSqlDropIndexQuery($tableName, $column);
$sql .= '<br>' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, false);
$message = __('Column `%s` has unique index, but should be non unique', $column);
$indexDiff[$tableName][$column] = array(
'message' => $message,
'sql' => $sql,
);
} else {
if (!$this->checkIfColumnContainsJustUniqueValues($tableName, $column)) {
$message = __('Column `%s` should be unique index, but contains duplicate values', $column);
$indexDiff[$tableName][$column] = array(
'message' => $message,
'sql' => '',
);
continue;
}
$sql = $this->generateSqlDropIndexQuery($tableName, $column);
$sql .= '<br>' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, true);
$message = __('Column `%s` should be unique index', $column);
$indexDiff[$tableName][$column] = array(
'message' => $message,
'sql' => $sql,
);
}
} }
} }
} }
@ -4932,16 +4999,27 @@ class Server extends AppModel
return $indexDiff; return $indexDiff;
} }
/**
* Returns indexes for given schema and table in array, where key is column name and value is `true` if
* index is index is unique, `false` otherwise.
*
* @param string $database
* @param string $table
* @return array
*/
public function getDatabaseIndexes($database, $table) public function getDatabaseIndexes($database, $table)
{ {
$sqlTableIndex = sprintf( $sqlTableIndex = sprintf(
"SELECT DISTINCT TABLE_NAME, COLUMN_NAME FROM information_schema.statistics WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s';", "SELECT DISTINCT TABLE_NAME, COLUMN_NAME, NON_UNIQUE FROM information_schema.statistics WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s';",
$database, $database,
$table $table
); );
$sqlTableIndexResult = $this->query($sqlTableIndex); $sqlTableIndexResult = $this->query($sqlTableIndex);
$tableIndex = Hash::extract($sqlTableIndexResult, '{n}.statistics.COLUMN_NAME'); $output = [];
return $tableIndex; foreach ($sqlTableIndexResult as $index) {
$output[$index['statistics']['COLUMN_NAME']] = $index['statistics']['NON_UNIQUE'] == 0;
}
return $output;
} }
public function writeableDirsDiagnostics(&$diagnostic_errors) public function writeableDirsDiagnostics(&$diagnostic_errors)

8
app/View/Elements/healthElements/db_indexes_diagnostic.ctp
View File

@ -1,12 +1,12 @@
<div> <div>
<label for="toggleTableDBIndexes" style="display: inline-block;"> <label for="toggleTableDBIndexes" style="display: inline-block;">
<input type="checkbox" id="toggleTableDBIndexes" class="form-input" checked></input> <input type="checkbox" id="toggleTableDBIndexes" class="form-input" checked>
<?php echo __('Show database indexes') ?> <?php echo __('Show database indexes') ?>
</label> </label>
</div> </div>
<div id="containerDBIndexes" class="" style="max-height: 800px; overflow-y: auto; padding: 5px;"> <div id="containerDBIndexes" class="" style="max-height: 800px; overflow-y: auto; padding: 5px;">
<?php if(empty($diagnostic)): ?> <?php if(empty($diagnostic)): ?>
<span class="label label-success"><?php echo __('Index diagnostic:'); ?><i class="fa fa-check"></i></span> <span class="label label-success"><?php echo __('Index diagnostic:'); ?> <i class="fa fa-check"></i></span>
<?php else: ?> <?php else: ?>
<div class="alert alert-warning"> <div class="alert alert-warning">
<strong><?php echo __('Notice'); ?></strong> <strong><?php echo __('Notice'); ?></strong>
@ -30,7 +30,7 @@
?> ?>
<?php foreach($columnArray as $columnName): ?> <?php foreach($columnArray as $columnName): ?>
<?php <?php
$columnIndexed = !empty($indexes[$tableName]) && in_array($columnName, $indexes[$tableName]); $columnIndexed = isset($indexes[$tableName][$columnName]);
$warningArray = isset($diagnostic[$tableName][$columnName]); $warningArray = isset($diagnostic[$tableName][$columnName]);
if ($warningArray) { if ($warningArray) {
$columnCount++; $columnCount++;
@ -74,4 +74,4 @@
message += "<div class=\"well\"><kbd>" + sqlQuery + "</kbd></div>" message += "<div class=\"well\"><kbd>" + sqlQuery + "</kbd></div>"
openPopover(clicked, message, undefined, 'left'); openPopover(clicked, message, undefined, 'left');
} }
</script> </script>

798
db_schema.json
View File

@ -6775,397 +6775,415 @@
] ]
}, },
"indexes": { "indexes": {
"admin_settings": [ "admin_settings": {
"id" "id": true
], },
"attribute_tags": [ "attributes": {
"id", "id": true,
"attribute_id", "uuid": true,
"event_id", "event_id": false,
"tag_id" "sharing_group_id": false,
], "type": false,
"attributes": [ "category": false,
"id", "value1": false,
"uuid", "value2": false,
"event_id", "object_id": false,
"sharing_group_id", "object_relation": false,
"type", "deleted": false,
"category", "first_seen": false,
"value1", "last_seen": false
"value2", },
"object_id", "attribute_tags": {
"object_relation", "id": true,
"deleted", "attribute_id": false,
"first_seen", "event_id": false,
"last_seen" "tag_id": false
], },
"bruteforces": [], "bruteforces": [],
"cake_sessions": [ "cake_sessions": {
"id", "id": true,
"expires" "expires": false
], },
"correlations": [ "correlations": {
"id", "id": true,
"event_id", "event_id": false,
"1_event_id", "1_event_id": false,
"attribute_id", "attribute_id": false,
"1_attribute_id" "1_attribute_id": false
], },
"dashboards": [ "dashboards": {
"id", "id": true,
"name", "name": false,
"uuid", "uuid": false,
"user_id", "user_id": false,
"restrict_to_org_id", "restrict_to_org_id": false,
"restrict_to_permission_flag" "restrict_to_permission_flag": false
], },
"decaying_model_mappings": [ "decaying_models": {
"id", "id": true,
"model_id" "uuid": false,
], "name": false,
"decaying_models": [ "org_id": false,
"id", "enabled": false,
"uuid", "all_orgs": false,
"name", "version": false
"org_id", },
"enabled", "decaying_model_mappings": {
"all_orgs", "id": true,
"version" "model_id": false
], },
"event_blacklists": [ "events": {
"id", "id": true,
"event_uuid", "uuid": true,
"event_orgc" "info": false,
], "sharing_group_id": false,
"event_delegations": [ "org_id": false,
"id", "orgc_id": false,
"org_id", "extends_uuid": false
"event_id" },
], "event_blacklists": {
"event_graph": [ "id": true,
"id", "event_uuid": false,
"event_id", "event_orgc": false
"user_id", },
"org_id", "event_delegations": {
"timestamp" "id": true,
], "org_id": false,
"event_locks": [ "event_id": false
"id", },
"event_id", "event_graph": {
"user_id", "id": true,
"timestamp" "event_id": false,
], "user_id": false,
"event_tags": [ "org_id": false,
"id", "timestamp": false
"event_id", },
"tag_id" "event_locks": {
], "id": true,
"events": [ "event_id": false,
"id", "user_id": false,
"uuid", "timestamp": false
"info", },
"sharing_group_id", "event_tags": {
"org_id", "id": true,
"orgc_id", "event_id": false,
"extends_uuid" "tag_id": false
], },
"favourite_tags": [ "favourite_tags": {
"id", "id": true,
"user_id", "user_id": false,
"tag_id" "tag_id": false
], },
"feeds": [ "feeds": {
"id", "id": true,
"input_source", "input_source": false
"orgc_id" },
], "fuzzy_correlate_ssdeep": {
"fuzzy_correlate_ssdeep": [ "id": true,
"id", "chunk": false,
"chunk", "attribute_id": false
"attribute_id" },
], "galaxies": {
"galaxies": [ "id": true,
"id", "name": false,
"name", "uuid": false,
"uuid", "type": false,
"type", "namespace": false
"namespace" },
], "galaxy_clusters": {
"galaxy_clusters": [ "id": true,
"id", "value": false,
"value", "uuid": false,
"uuid", "collection_uuid": false,
"collection_uuid", "galaxy_id": false,
"galaxy_id", "version": false,
"version", "tag_name": false,
"tag_name", "type": false
"type" },
], "galaxy_cluster_blocklists": {
"galaxy_elements": [ "id": true,
"id", "cluster_uuid": false,
"key", "cluster_orgc": false
"value", },
"galaxy_cluster_id" "galaxy_cluster_relations": {
], "id": true,
"galaxy_reference": [ "galaxy_cluster_id": false,
"id", "referenced_galaxy_cluster_id": false,
"galaxy_cluster_id", "referenced_galaxy_cluster_type": false,
"referenced_galaxy_cluster_id", "galaxy_cluster_uuid": false,
"referenced_galaxy_cluster_value", "sharing_group_id": false,
"referenced_galaxy_cluster_type" "default": false
], },
"inbox": [ "galaxy_cluster_relation_tags": {
"id", "id": true,
"title", "galaxy_cluster_relation_id": false,
"type", "tag_id": false
"uuid", },
"user_agent_sha256", "galaxy_elements": {
"ip", "id": true,
"timestamp" "key": false,
], "value": false,
"jobs": [ "galaxy_cluster_id": false
"id" },
], "galaxy_reference": {
"logs": [ "id": true,
"id" "referenced_galaxy_cluster_id": false,
], "referenced_galaxy_cluster_value": false,
"news": [ "referenced_galaxy_cluster_type": false
"id" },
], "inbox": {
"noticelist_entries": [ "id": true,
"id", "title": false,
"noticelist_id" "type": false,
], "uuid": false,
"noticelists": [ "user_agent_sha256": false,
"id", "ip": false,
"name", "timestamp": false
"geographical_area" },
], "jobs": {
"notification_logs": [ "id": true
"id", },
"org_id", "logs": {
"type" "id": true
], },
"object_references": [ "news": {
"id", "id": true
"source_uuid", },
"referenced_uuid", "noticelists": {
"timestamp", "id": true,
"object_id", "name": false,
"referenced_id", "geographical_area": false
"relationship_type" },
], "noticelist_entries": {
"object_relationships": [ "id": true,
"id", "noticelist_id": false
"name" },
], "notification_logs": {
"object_template_elements": [ "id": true,
"id", "org_id": false,
"object_relation", "type": false
"type" },
], "objects": {
"object_templates": [ "id": true,
"id", "name": false,
"user_id", "template_uuid": false,
"org_id", "template_version": false,
"uuid", "meta-category": false,
"name", "event_id": false,
"meta-category" "uuid": false,
], "timestamp": false,
"objects": [ "distribution": false,
"id", "sharing_group_id": false,
"name", "first_seen": false,
"template_uuid", "last_seen": false
"template_version", },
"meta-category", "object_references": {
"event_id", "id": true,
"uuid", "source_uuid": false,
"timestamp", "referenced_uuid": false,
"distribution", "timestamp": false,
"sharing_group_id", "object_id": false,
"first_seen", "referenced_id": false,
"last_seen" "relationship_type": false
], },
"org_blacklists": [ "object_relationships": {
"id" "id": true,
], "name": false
"organisations": [ },
"id", "object_templates": {
"uuid", "id": true,
"name" "user_id": false,
], "org_id": false,
"posts": [ "uuid": false,
"id", "name": false,
"post_id", "meta-category": false
"thread_id" },
], "object_template_elements": {
"regexp": [ "id": true,
"id" "object_relation": false,
], "type": false
"rest_client_histories": [ },
"id", "organisations": {
"org_id", "id": true,
"user_id", "uuid": false,
"timestamp" "name": false
], },
"roles": [ "org_blacklists": {
"id" "id": true
], },
"servers": [ "posts": {
"id", "id": true,
"org_id", "post_id": false,
"priority", "thread_id": false
"remote_org_id" },
], "regexp": {
"shadow_attribute_correlations": [ "id": true
"id", },
"org_id", "rest_client_histories": {
"attribute_id", "id": true,
"a_sharing_group_id", "org_id": false,
"event_id", "user_id": false,
"1_event_id", "timestamp": false
"sharing_group_id", },
"1_shadow_attribute_id" "roles": {
], "id": true
"shadow_attributes": [ },
"id", "servers": {
"event_id", "id": true,
"event_uuid", "org_id": false,
"event_org_id", "priority": false,
"uuid", "remote_org_id": false
"old_id", },
"value1", "shadow_attributes": {
"value2", "id": true,
"type", "event_id": false,
"category", "event_uuid": false,
"first_seen", "event_org_id": false,
"last_seen" "uuid": false,
], "old_id": false,
"sharing_group_orgs": [ "value1": false,
"id", "value2": false,
"org_id", "type": false,
"sharing_group_id" "category": false,
], "first_seen": false,
"sharing_group_servers": [ "last_seen": false
"id", },
"server_id", "shadow_attribute_correlations": {
"sharing_group_id" "id": true,
], "org_id": false,
"sharing_groups": [ "attribute_id": false,
"id", "a_sharing_group_id": false,
"uuid", "event_id": false,
"org_id", "1_event_id": false,
"sync_user_id", "sharing_group_id": false,
"organisation_uuid" "1_shadow_attribute_id": false
], },
"sightingdb_orgs": [ "sharing_groups": {
"id", "id": true,
"sightingdb_id", "uuid": true,
"org_id" "org_id": false,
], "sync_user_id": false,
"sightingdbs": [ "organisation_uuid": false
"id", },
"name", "sharing_group_orgs": {
"owner", "id": true,
"host", "org_id": false,
"port" "sharing_group_id": false
], },
"sightings": [ "sharing_group_servers": {
"id", "id": true,
"attribute_id", "server_id": false,
"event_id", "sharing_group_id": false
"org_id", },
"uuid", "sightingdbs": {
"source", "id": true,
"type" "name": false,
], "owner": false,
"tag_collection_tags": [ "host": false,
"id", "port": false
"tag_collection_id", },
"tag_id" "sightingdb_orgs": {
], "id": true,
"tag_collections": [ "sightingdb_id": false,
"id", "org_id": false
"uuid", },
"user_id", "sightings": {
"org_id" "id": true,
], "attribute_id": false,
"tags": [ "event_id": false,
"id", "org_id": false,
"name", "uuid": false,
"org_id", "source": false,
"user_id", "type": false
"numerical_value" },
], "tags": {
"tasks": [ "id": true,
"id" "name": false,
], "org_id": false,
"taxonomies": [ "user_id": false,
"id" "numerical_value": false
], },
"taxonomy_entries": [ "tag_collections": {
"id", "id": true,
"taxonomy_predicate_id", "uuid": false,
"numerical_value" "user_id": false,
], "org_id": false
"taxonomy_predicates": [ },
"id", "tag_collection_tags": {
"taxonomy_id", "id": true,
"numerical_value" "tag_collection_id": false,
], "tag_id": false
"template_element_attributes": [ },
"id" "tasks": {
], "id": true
"template_element_files": [ },
"id" "taxonomies": {
], "id": true,
"template_element_texts": [ "enabled": false
"id" },
], "taxonomy_entries": {
"template_elements": [ "id": true,
"id" "taxonomy_predicate_id": false,
], "numerical_value": false
"template_tags": [ },
"id" "taxonomy_predicates": {
], "id": true,
"templates": [ "taxonomy_id": false,
"id" "numerical_value": false
], },
"threads": [ "templates": {
"id", "id": true
"user_id", },
"event_id", "template_elements": {
"org_id", "id": true
"sharing_group_id" },
], "template_element_attributes": {
"threat_levels": [ "id": true
"id" },
], "template_element_files": {
"user_settings": [ "id": true
"id", },
"setting", "template_element_texts": {
"user_id", "id": true
"timestamp" },
], "template_tags": {
"users": [ "id": true
"id", },
"email", "threads": {
"org_id", "id": true,
"server_id" "user_id": false,
], "event_id": false,
"warninglist_entries": [ "org_id": false,
"id", "sharing_group_id": false
"warninglist_id" },
], "threat_levels": {
"warninglist_types": [ "id": true
"id" },
], "users": {
"warninglists": [ "id": true,
"id" "email": false,
], "org_id": false,
"whitelist": [ "server_id": false
"id" },
] "user_settings": {
"id": true,
"setting": false,
"user_id": false,
"timestamp": false
},
"warninglists": {
"id": true
},
"warninglist_entries": {
"id": true,
"warninglist_id": false
},
"warninglist_types": {
"id": true
},
"whitelist": {
"id": true
}
}, },
"db_version": "55" "db_version": "55"
} }