MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge branch 'sqlIndexDiagnostic' into 2.4

pull/5488/head
iglocska 2019-12-19 10:33:00 +01:00
parent 3fc02548f6 8f6cdc73e1
commit 53482ff76c
No known key found for this signature in database
GPG Key ID: BEA224F1FEF113AC
7 changed files with 130 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
app/Console/Command/AdminShell.php
View File

@ -516,14 +516,15 @@ class AdminShell extends AppShell
public function dumpCurrentDatabaseSchema()
{
$dbActualSchema = $this->Server->getActualDBSchema()['schema'];
$dbActualSchema = $this->Server->getActualDBSchema();
$dbVersion = $this->AdminSetting->find('first', array(
'conditions' => array('setting' => 'db_version')
));
if (!empty($dbVersion) && !empty($dbActualSchema)) {
if (!empty($dbVersion) && !empty($dbActualSchema['schema'])) {
$dbVersion = $dbVersion['AdminSetting']['value'];
$data = array(
'schema' => $dbActualSchema,
'schema' => $dbActualSchema['schema'],
'indexes' => $dbActualSchema['indexes'],
'db_version' => $dbVersion
);
$file = new File(ROOT . DS . 'db_schema.json', true);

4
app/Controller/ServersController.php
View File

@ -2221,6 +2221,7 @@ misp.direct_call(relative_path, body)
} else {
$this->set('checkedTableColumn', $dbSchemaDiagnostics['checked_table_column']);
$this->set('dbSchemaDiagnostics', $dbSchemaDiagnostics['diagnostic']);
$this->set('dbIndexDiagnostics', $dbSchemaDiagnostics['diagnostic_index']);
$this->set('expectedDbVersion', $dbSchemaDiagnostics['expected_db_version']);
$this->set('actualDbVersion', $dbSchemaDiagnostics['actual_db_version']);
$this->set('error', $dbSchemaDiagnostics['error']);
@ -2228,9 +2229,10 @@ misp.direct_call(relative_path, body)
$this->set('updateFailNumberReached', $dbSchemaDiagnostics['update_fail_number_reached']);
$this->set('updateLocked', $dbSchemaDiagnostics['update_locked']);
$this->set('dataSource', $dbSchemaDiagnostics['dataSource']);
$this->set('columnPerTable', $dbSchemaDiagnostics['columnPerTable']);
$this->set('indexes', $dbSchemaDiagnostics['indexes']);
$this->render('/Elements/healthElements/db_schema_diagnostic');
}
}
public function viewDeprecatedFunctionUse()

52
app/Model/Server.php
View File

@ -4351,20 +4351,30 @@ class Server extends AppModel
'actual_db_version' => $actualDbVersion,
'checked_table_column' => array(),
'diagnostic' => array(),
'diagnostic_index' => array(),
'expected_db_version' => '?',
'error' => '',
'update_locked' => $this->isUpdateLocked(),
'remaining_lock_time' => $this->getLockRemainingTime(),
'update_fail_number_reached' => $this->UpdateFailNumberReached()
'update_fail_number_reached' => $this->UpdateFailNumberReached(),
'indexes' => array()
);
if ($dataSource == 'Database/Mysql') {
$dbActualSchema = $this->getActualDBSchema();
$dbExpectedSchema = $this->getExpectedDBSchema();
if ($dbExpectedSchema !== false) {
$db_schema_comparison = $this->compareDBSchema($dbActualSchema['schema'], $dbExpectedSchema['schema']);
$db_indexes_comparison = $this->compareDBIndexes($dbActualSchema['indexes'], $dbExpectedSchema['indexes']);
$schemaDiagnostic['checked_table_column'] = $dbActualSchema['column'];
$schemaDiagnostic['diagnostic'] = $db_schema_comparison;
$schemaDiagnostic['diagnostic_index'] = $db_indexes_comparison;
$schemaDiagnostic['expected_db_version'] = $dbExpectedSchema['db_version'];
foreach($dbActualSchema['schema'] as $tableName => $tableMetas) {
foreach($tableMetas as $tableMeta) {
$schemaDiagnostic['columnPerTable'][$tableName][] = $tableMeta['column_name'];
}
}
$schemaDiagnostic['indexes'] = $dbActualSchema['indexes'];
} else {
$schemaDiagnostic['error'] = sprintf('Diagnostic not available as the expected schema file could not be loaded');
}
@ -4501,6 +4511,7 @@ class Server extends AppModel
)
){
$dbActualSchema = array();
$dbActualIndexes = array();
$dataSource = $this->getDataSource()->config['datasource'];
if ($dataSource == 'Database/Mysql') {
$sqlGetTable = sprintf('SELECT TABLE_NAME FROM information_schema.tables WHERE table_schema = %s;', "'" . $this->getDataSource()->config['database'] . "'");
@ -4515,12 +4526,13 @@ class Server extends AppModel
foreach ($sqlResult as $column_schema) {
$dbActualSchema[$table][] = $column_schema['columns'];
}
$dbActualIndexes[$table] = $this->getDatabaseIndexes($this->getDataSource()->config['database'], $table);
}
}
else if ($dataSource == 'Database/Postgres') {
return array('Database/Postgres' => array('description' => __('Can\'t check database schema for Postgres database type')));
}
return array('schema' => $dbActualSchema, 'column' => $tableColumnNames);
return array('schema' => $dbActualSchema, 'column' => $tableColumnNames, 'indexes' => $dbActualIndexes);
}
public function compareDBSchema($dbActualSchema, $dbExpectedSchema)
@ -4627,6 +4639,42 @@ class Server extends AppModel
return $dbDiff;
}
public function compareDBIndexes($actualIndex, $expectedIndex)
{
$indexDiff = array();
foreach($expectedIndex as $tableName => $indexes) {
if (!array_key_exists($tableName, $actualIndex)) {
// If table does not exists, it is covered by the schema diagnostic
} else {
$tableIndexDiff = array_diff($indexes, $actualIndex[$tableName]); // check for missing indexes
if (count($tableIndexDiff) > 0) {
foreach($tableIndexDiff as $columnDiff) {
$indexDiff[$tableName][$columnDiff] = sprintf(__('Column `%s` should be indexed'), $columnDiff);
}
}
$tableIndexDiff = array_diff($actualIndex[$tableName], $indexes); // check for additional indexes
if (count($tableIndexDiff) > 0) {
foreach($tableIndexDiff as $columnDiff) {
$indexDiff[$tableName][$columnDiff] = sprintf(__('Column `%s` is indexed but should not'), $columnDiff);
}
}
}
}
return $indexDiff;
}
public function getDatabaseIndexes($database, $table)
{
$sqlTableIndex = sprintf(
"SELECT DISTINCT TABLE_NAME, COLUMN_NAME FROM information_schema.statistics WHERE TABLE_SCHEMA = '%s' AND TABLE_NAME = '%s';",
$database,
$table
);
$sqlTableIndexResult = $this->query($sqlTableIndex);
$tableIndex = Hash::extract($sqlTableIndexResult, '{n}.statistics.COLUMN_NAME');
return $tableIndex;
}
public function writeableDirsDiagnostics(&$diagnostic_errors)
{
App::uses('File', 'Utility');

62
app/View/Elements/healthElements/db_indexes_diagnostic.ctp Normal file
View File

@ -0,0 +1,62 @@
<div>
<label for="toggleTableDBIndexes" style="display: inline-block;">
<input type="checkbox" id="toggleTableDBIndexes" class="form-input" checked></input>
<?php echo __('Show database indexes') ?>
</label>
</div>
<div id="containerDBIndexes" class="" style="max-height: 800px; overflow-y: auto; padding: 5px;">
<?php if(empty($diagnostic)): ?>
<span class="label label-success"><?php echo __('Index diagnostic:'); ?><i class="fa fa-check"></i></span>
<?php else: ?>
<div class="alert alert-warning">
<strong><?php echo __('Notice'); ?></strong>
<?php echo __('The highlighted issues may be benign. if you are unsure, please open an issue and ask for clarification.'); ?>
</div>
<table id="tableDBIndexes" class="table table-condensed table-bordered">
<thead>
<tr>
<th>Table name</th>
<th>Column name</th>
<th>Indexed</th>
<th>Description</th>
</tr>
</thead>
<tbody>
<?php foreach($columnPerTable as $tableName => $columnArray): ?>
<?php
$columnCount = 0;
$rowHtml = '';
?>
<?php foreach($columnArray as $columnName): ?>
<?php
$columnIndexed = !empty($indexes[$tableName]) && in_array($columnName, $indexes[$tableName]);
$warning = isset($diagnostic[$tableName][$columnName]);
if ($warning) {
$columnCount++;
}
$rowHtml .= sprintf('%s%s%s%s%s',
sprintf('<tr class="%s">', $warning ? 'error' : 'indexInfo hidden'),
sprintf('<td>%s</td>', h($columnName)),
sprintf('<td><i class="bold fa %s"></i></td>', $columnIndexed ? 'green fa-check' : 'red fa-times'),
sprintf('<td>%s</td>', $warning ? h($diagnostic[$tableName][$columnName]) : ''),
'</tr>'
);
?>
<?php endforeach; ?>
<?php if ($columnCount > 0): ?>
<?php echo sprintf('<tr><td rowspan="%s" colspan="0" class="bold">%s</td></tr>', $columnCount+1, h($tableName)); ?>
<?php echo $rowHtml; ?>
<?php endif; ?>
<?php endforeach; ?>
</tbody>
</table>
<?php endif; ?>
</div>
<script>
$(document).ready(function() {
$('#toggleTableDBIndexes').change(function() {
$('#containerDBIndexes').toggle();
})
})
</script>

8
app/View/Elements/healthElements/db_schema_diagnostic.ctp
View File

@ -24,7 +24,6 @@
*/
function highlightAndSanitize($dirty, $toHighlight, $colorType = 'success')
{
if (is_array($dirty)) {
@ -168,7 +167,12 @@
__('DataSource: ') . h($dataSource),
__('DataSource: ') . h($dataSource),
$dataSource != 'Database/Mysql' ? 'times' : 'check'
)
);
echo $this->element('/healthElements/db_indexes_diagnostic', array(
'columnPerTable' => $columnPerTable,
'diagnostic' => $dbIndexDiagnostics,
'indexes' => $indexes
));
?>
<script>
var dbSchemaDiagnostics = <?php echo json_encode($dbSchemaDiagnostics); ?>;

5
app/View/Elements/healthElements/diagnostics.ctp
View File

@ -237,7 +237,10 @@
'remainingLockTime' => $dbSchemaDiagnostics['remaining_lock_time'],
'updateFailNumberReached' => $dbSchemaDiagnostics['update_fail_number_reached'],
'updateLocked' => $dbSchemaDiagnostics['update_locked'],
'dataSource' => $dbSchemaDiagnostics['dataSource']
'dataSource' => $dbSchemaDiagnostics['dataSource'],
'columnPerTable' => $dbSchemaDiagnostics['columnPerTable'],
'dbIndexDiagnostics' => $dbSchemaDiagnostics['diagnostic_index'],
'indexes' => $dbSchemaDiagnostics['indexes'],
)); ?>
</div>
<h3><?= __("Redis info") ?></h3>

2
db_schema.json
View File

File diff suppressed because one or more lines are too long