mirror of https://github.com/MISP/MISP
Merge remote-tracking branch 'origin/develop' into feature-workflows-2
Sami Mokaddem
commit
49575533ad
|
|
@ -16,8 +16,9 @@ App::uses('JsonTool', 'Tools');
|
|||
*/
|
||||
class AdminShell extends AppShell
|
||||
{
|
||||
public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Allowedlist', 'Server', 'Organisation', 'AdminSetting', 'Galaxy', 'Taxonomy', 'Warninglist', 'Noticelist', 'ObjectTemplate', 'Bruteforce', 'Role', 'Feed', 'SharingGroupBlueprint');
|
||||
|
||||
public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Allowedlist', 'Server', 'Organisation', 'AdminSetting', 'Galaxy', 'Taxonomy', 'Warninglist', 'Noticelist', 'ObjectTemplate', 'Bruteforce', 'Role', 'Feed', 'SharingGroupBlueprint', 'Correlation');
|
||||
public $tasks = array('ConfigLoad');
|
||||
|
||||
public function getOptionParser()
|
||||
{
|
||||
$parser = parent::getOptionParser();
|
||||
|
|
@ -99,6 +100,7 @@ class AdminShell extends AppShell
|
|||
|
||||
public function jobGenerateCorrelation()
|
||||
{
|
||||
$this->ConfigLoad->execute();
|
||||
if (empty($this->args[0])) {
|
||||
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Generate correlation'] . PHP_EOL);
|
||||
}
|
||||
|
|
@ -1186,4 +1188,37 @@ class AdminShell extends AppShell
|
|||
);
|
||||
$this->out($message);
|
||||
}
|
||||
|
||||
public function truncateTable()
|
||||
{
|
||||
$this->ConfigLoad->execute();
|
||||
if (empty($this->args[0])) {
|
||||
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Truncate table correlation'] . PHP_EOL);
|
||||
}
|
||||
$userId = $this->args[0];
|
||||
if ($userId) {
|
||||
$user = $this->User->getAuthUser($userId);
|
||||
} else {
|
||||
$user = [
|
||||
'id' => 0,
|
||||
'email' => 'SYSTEM',
|
||||
'Organisation' => [
|
||||
'name' => 'SYSTEM'
|
||||
]
|
||||
];
|
||||
}
|
||||
if (empty($this->args[1])) {
|
||||
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Truncate table correlation'] . PHP_EOL);
|
||||
}
|
||||
if (!empty($this->args[2])) {
|
||||
$jobId = $this->args[2];
|
||||
}
|
||||
$table = trim($this->args[1]);
|
||||
$this->Correlation->truncate($user, $table);
|
||||
if ($jobId) {
|
||||
$this->Job->saveField('progress', 100);
|
||||
$this->Job->saveField('date_modified', date("Y-m-d H:i:s"));
|
||||
$this->Job->saveField('message', __('Database truncated: ' . $table));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -84,10 +84,24 @@ class AttributesController extends AppController
|
|||
'fields' => ['Orgc.id', 'Orgc.name', 'Orgc.uuid'],
|
||||
]);
|
||||
$orgTable = Hash::combine($orgTable, '{n}.Orgc.id', '{n}.Orgc');
|
||||
$sgids = $this->Attribute->SharingGroup->authorizedIds($this->Auth->user());
|
||||
foreach ($attributes as &$attribute) {
|
||||
if (isset($orgTable[$attribute['Event']['orgc_id']])) {
|
||||
$attribute['Event']['Orgc'] = $orgTable[$attribute['Event']['orgc_id']];
|
||||
}
|
||||
$temp = $this->Attribute->Correlation->getRelatedAttributes(
|
||||
$this->Auth->user(),
|
||||
$sgids,
|
||||
$attribute['Attribute'],
|
||||
[],
|
||||
true
|
||||
);
|
||||
foreach ($temp as &$t) {
|
||||
$t['info'] = $t['Event']['info'];
|
||||
$t['org_id'] = $t['Event']['org_id'];
|
||||
$t['date'] = $t['Event']['date'];
|
||||
}
|
||||
$attribute['Event']['RelatedAttribute'][$attribute['Attribute']['id']] = $temp;
|
||||
}
|
||||
|
||||
list($attributes, $sightingsData) = $this->__searchUI($attributes);
|
||||
|
|
@ -1582,13 +1596,19 @@ class AttributesController extends AppController
|
|||
$attribute['Event']['Org'] = $orgTable[$attribute['Event']['org_id']];
|
||||
}
|
||||
if (isset($filters['includeCorrelations'])) {
|
||||
$attribute['Event']['RelatedAttribute'][$attribute['Attribute']['id']] = $this->Attribute->Correlation->getRelatedAttributes(
|
||||
$temp = $this->Attribute->Correlation->getRelatedAttributes(
|
||||
$this->Auth->user(),
|
||||
$sgids,
|
||||
$attribute['Attribute'],
|
||||
[],
|
||||
true
|
||||
);
|
||||
foreach ($temp as &$t) {
|
||||
$t['info'] = $t['Event']['info'];
|
||||
$t['org_id'] = $t['Event']['org_id'];
|
||||
$t['date'] = $t['Event']['date'];
|
||||
}
|
||||
$attribute['Event']['RelatedAttribute'][$attribute['Attribute']['id']] = $temp;
|
||||
}
|
||||
}
|
||||
if ($this->_isRest()) {
|
||||
|
|
@ -1905,36 +1925,38 @@ class AttributesController extends AppController
|
|||
|
||||
public function generateCorrelation()
|
||||
{
|
||||
$this->request->allowMethod(['post']);
|
||||
if ($this->request->is('post')) {
|
||||
if (!Configure::read('MISP.background_jobs')) {
|
||||
$k = $this->Attribute->generateCorrelation();
|
||||
$this->Flash->success(__('All done. %s attributes processed.', $k));
|
||||
$this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
|
||||
} else {
|
||||
/** @var Job $job */
|
||||
$job = ClassRegistry::init('Job');
|
||||
$jobId = $job->createJob(
|
||||
'SYSTEM',
|
||||
Job::WORKER_DEFAULT,
|
||||
'generate correlation',
|
||||
'All attributes',
|
||||
'Job created.'
|
||||
);
|
||||
|
||||
if (!Configure::read('MISP.background_jobs')) {
|
||||
$k = $this->Attribute->generateCorrelation();
|
||||
$this->Flash->success(__('All done. %s attributes processed.', $k));
|
||||
$this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
|
||||
} else {
|
||||
/** @var Job $job */
|
||||
$job = ClassRegistry::init('Job');
|
||||
$jobId = $job->createJob(
|
||||
'SYSTEM',
|
||||
Job::WORKER_DEFAULT,
|
||||
'generate correlation',
|
||||
'All attributes',
|
||||
'Job created.'
|
||||
);
|
||||
|
||||
$this->Attribute->getBackgroundJobsTool()->enqueue(
|
||||
BackgroundJobsTool::DEFAULT_QUEUE,
|
||||
BackgroundJobsTool::CMD_ADMIN,
|
||||
[
|
||||
'jobGenerateCorrelation',
|
||||
$this->Attribute->getBackgroundJobsTool()->enqueue(
|
||||
BackgroundJobsTool::DEFAULT_QUEUE,
|
||||
BackgroundJobsTool::CMD_ADMIN,
|
||||
[
|
||||
'jobGenerateCorrelation',
|
||||
$jobId
|
||||
],
|
||||
true,
|
||||
$jobId
|
||||
],
|
||||
true,
|
||||
$jobId
|
||||
);
|
||||
);
|
||||
|
||||
$this->Flash->success(__('Job queued. You can view the progress if you navigate to the active jobs view (Administration -> Jobs).'));
|
||||
$this->redirect(array('controller' => 'pages', 'action' => 'display', 'administration'));
|
||||
$this->Flash->success(__('Job queued. You can view the progress if you navigate to the active jobs view (Administration -> Jobs).'));
|
||||
$this->redirect(Router::url($this->referer(), true));
|
||||
}
|
||||
} else {
|
||||
$this->render('ajax/recorrelationConfirmation');
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -111,4 +111,93 @@ class CorrelationsController extends AppController
|
|||
]);
|
||||
}
|
||||
}
|
||||
|
||||
public function switchEngine(string $engine)
|
||||
{
|
||||
$this->loadModel('Server');
|
||||
if (!isset($this->Correlation->validEngines[$engine])) {
|
||||
throw new MethodNotAllowedException(__('Not a valid engine choice. Please make sure you pass one of the following: ', implode(', ', array_keys($this->Correlation->validEngines))));
|
||||
}
|
||||
if ($this->request->is('post')) {
|
||||
$setting = $this->Server->getSettingData('MISP.correlation_engine');
|
||||
$result = $this->Server->serverSettingsEditValue($this->Auth->user(), $setting, $engine);
|
||||
if ($result === true) {
|
||||
$message = __('Engine switched.');
|
||||
if ($this->_isRest()) {
|
||||
return $this->RestResponse->saveSuccessResponse('Correlations', 'switchEngine', false, $this->response->type(), $message);
|
||||
} else {
|
||||
$this->Flash->success($message);
|
||||
$this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'correlations']);
|
||||
}
|
||||
} else {
|
||||
$message = __('Couldn\'t switch to the requested engine.');
|
||||
if ($this->_isRest()) {
|
||||
return $this->RestResponse->saveFailResponse('Correlations', 'switchEngine', false, $message, $this->response->type());
|
||||
} else {
|
||||
$this->Flash->error($message);
|
||||
$this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'correlations']);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
$this->set('engine', $engine);
|
||||
$this->render('ajax/switch_engine_confirmation');
|
||||
}
|
||||
}
|
||||
|
||||
public function truncate(string $engine)
|
||||
{
|
||||
if (!isset($this->Correlation->validEngines[$engine])) {
|
||||
throw new MethodNotAllowedException(__('Not a valid engine choice. Please make sure you pass one of the following: ', implode(', ', array_keys($this->Correlation->validEngines))));
|
||||
}
|
||||
if ($this->request->is('post')) {
|
||||
if (!Configure::read('MISP.background_jobs')) {
|
||||
$result = $this->Correlation->truncate($this->Auth->user(), $engine);
|
||||
$message = $result ? __('Table truncated.') : __('Could not truncate table');
|
||||
if ($this->_isRest()) {
|
||||
if ($result) {
|
||||
$this->RestResponse->saveSuccessResponse('Correlations', 'truncate', false, $this->response->type(), $message);
|
||||
} else {
|
||||
$this->RestResponse->saveFailResponse('Correlations', 'truncate', false, $message, $this->response->type());
|
||||
}
|
||||
} else {
|
||||
$this->Flash->{$result ? 'success' : 'error'}($message);
|
||||
$this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'correlations']);
|
||||
}
|
||||
} else {
|
||||
$job = ClassRegistry::init('Job');
|
||||
$jobId = $job->createJob(
|
||||
'SYSTEM',
|
||||
Job::WORKER_DEFAULT,
|
||||
'truncate table',
|
||||
$this->Correlation->validEngines[$engine],
|
||||
'Job created.'
|
||||
);
|
||||
|
||||
$this->Correlation->Attribute->getBackgroundJobsTool()->enqueue(
|
||||
BackgroundJobsTool::DEFAULT_QUEUE,
|
||||
BackgroundJobsTool::CMD_ADMIN,
|
||||
[
|
||||
'truncateTable',
|
||||
$this->Auth->user('id'),
|
||||
$engine,
|
||||
$jobId
|
||||
],
|
||||
true,
|
||||
$jobId
|
||||
);
|
||||
|
||||
$message = __('Job queued. You can view the progress if you navigate to the active jobs view (Administration -> Jobs).');
|
||||
if ($this->_isRest()) {
|
||||
return $this->RestResponse->saveSuccessResponse('Correlations', 'truncate', false, $this->response->type(), $message);
|
||||
} else {
|
||||
$this->Flash->success($message);
|
||||
$this->redirect(['controller' => 'servers', 'action' => 'serverSettings', 'correlations']);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
$this->set('engine', $engine);
|
||||
$this->set('table_name', $this->Correlation->validEngines[$engine]);
|
||||
$this->render('ajax/truncate_confirmation');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1072,6 +1072,11 @@ class ServersController extends AppController
|
|||
$diagnostic_errors = 0;
|
||||
App::uses('File', 'Utility');
|
||||
App::uses('Folder', 'Utility');
|
||||
if ($tab === 'correlations') {
|
||||
$this->loadModel('Correlation');
|
||||
$correlation_metrics = $this->Correlation->collectMetrics();
|
||||
$this->set('correlation_metrics', $correlation_metrics);
|
||||
}
|
||||
if ($tab === 'files') {
|
||||
$files = $this->Server->grabFiles();
|
||||
$this->set('files', $files);
|
||||
|
|
|
|||
|
|
@ -82,7 +82,7 @@ class AppModel extends Model
|
|||
69 => false, 70 => false, 71 => true, 72 => true, 73 => false, 74 => false,
|
||||
75 => false, 76 => true, 77 => false, 78 => false, 79 => false, 80 => false,
|
||||
81 => false, 82 => false, 83 => false, 84 => false, 85 => false, 86 => false,
|
||||
87 => false, 88 => false, 89 => false,
|
||||
87 => false, 88 => false, 89 => false, 90 => false,
|
||||
);
|
||||
|
||||
const ADVANCED_UPDATES_DESCRIPTION = array(
|
||||
|
|
@ -228,6 +228,9 @@ class AppModel extends Model
|
|||
$dbUpdateSuccess = $this->__generateCorrelations();
|
||||
break;
|
||||
case 89:
|
||||
$this->__retireOldCorrelationEngine();
|
||||
break;
|
||||
case 90:
|
||||
$this->Workflow = Classregistry::init('Workflow');
|
||||
$this->Workflow->enableDefaultModules();
|
||||
break;
|
||||
|
|
@ -1770,7 +1773,7 @@ class AppModel extends Model
|
|||
$sqlArray[] = 'ALTER TABLE `users` ADD `external_auth_required` tinyint(1) NOT NULL DEFAULT 0;';
|
||||
$sqlArray[] = 'ALTER TABLE `users` ADD `external_auth_key` text COLLATE utf8_bin;';
|
||||
break;
|
||||
case 89:
|
||||
case 90:
|
||||
$sqlArray[] = "CREATE TABLE IF NOT EXISTS `workflows` (
|
||||
`id` int(11) NOT NULL AUTO_INCREMENT,
|
||||
`uuid` varchar(40) COLLATE utf8_bin NOT NULL ,
|
||||
|
|
@ -2227,7 +2230,7 @@ class AppModel extends Model
|
|||
'fields' => ['id', 'value'],
|
||||
]);
|
||||
if (count($db_version) > 1) {
|
||||
// we rgan into a bug where we have more than one db_version entry. This bug happened in some rare circumstances around 2.4.50-2.4.57
|
||||
// we ran into a bug where we have more than one db_version entry. This bug happened in some rare circumstances around 2.4.50-2.4.57
|
||||
foreach ($db_version as $k => $v) {
|
||||
if ($k > 0) {
|
||||
$this->AdminSetting->delete($v['AdminSetting']['id']);
|
||||
|
|
@ -3578,4 +3581,61 @@ class AppModel extends Model
|
|||
}
|
||||
return $this->_eventManager;
|
||||
}
|
||||
|
||||
private function __retireOldCorrelationEngine($user = null) {
|
||||
if ($user === null) {
|
||||
$user = [
|
||||
'id' => 0,
|
||||
'email' => 'SYSTEM',
|
||||
'Organisation' => [
|
||||
'name' => 'SYSTEM'
|
||||
]
|
||||
];
|
||||
}
|
||||
$this->Correlation = ClassRegistry::init('Correlation');
|
||||
$this->Attribute = ClassRegistry::init('Attribute');
|
||||
if (!Configure::read('MISP.background_jobs')) {
|
||||
$this->Correlation->truncate($user, 'Legacy');
|
||||
$this->Attribute->generateCorrelation();
|
||||
} else {
|
||||
$job = ClassRegistry::init('Job');
|
||||
$jobId = $job->createJob(
|
||||
'SYSTEM',
|
||||
Job::WORKER_DEFAULT,
|
||||
'truncate table',
|
||||
$this->Correlation->validEngines['Legacy'],
|
||||
'Job created.'
|
||||
);
|
||||
$this->Correlation->Attribute->getBackgroundJobsTool()->enqueue(
|
||||
BackgroundJobsTool::DEFAULT_QUEUE,
|
||||
BackgroundJobsTool::CMD_ADMIN,
|
||||
[
|
||||
'truncateTable',
|
||||
0,
|
||||
'Legacy',
|
||||
$jobId
|
||||
],
|
||||
true,
|
||||
$jobId
|
||||
);
|
||||
$jobId = $job->createJob(
|
||||
'SYSTEM',
|
||||
Job::WORKER_DEFAULT,
|
||||
'generate correlation',
|
||||
'All attributes',
|
||||
'Job created.'
|
||||
);
|
||||
|
||||
$this->Attribute->getBackgroundJobsTool()->enqueue(
|
||||
BackgroundJobsTool::DEFAULT_QUEUE,
|
||||
BackgroundJobsTool::CMD_ADMIN,
|
||||
[
|
||||
'jobGenerateCorrelation',
|
||||
$jobId
|
||||
],
|
||||
true,
|
||||
$jobId
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1601,6 +1601,9 @@ class Attribute extends AppModel
|
|||
$attributes = $this->find('all', $query);
|
||||
foreach ($attributes as $attribute) {
|
||||
$attribute['Attribute']['event_id'] = $eventId;
|
||||
if ($full) {
|
||||
$this->Correlation->beforeSaveCorrelation($attribute['Attribute']);
|
||||
}
|
||||
$this->Correlation->afterSaveCorrelation($attribute['Attribute'], $full);
|
||||
}
|
||||
$fetchedAttributes = count($attributes);
|
||||
|
|
|
|||
|
|
@ -136,7 +136,7 @@ class NoAclCorrelationBehavior extends ModelBehavior
|
|||
$max_correlations = Configure::read('MISP.max_correlations_per_event') ?: 5000;
|
||||
$source = $primary ? '' : '1_';
|
||||
$prefix = $primary ? '1_' : '';
|
||||
$correlations = $this->Correlation->find('all', array(
|
||||
$correlations = $this->Correlation->find('all', [
|
||||
'fields' => [
|
||||
$source . 'attribute_id',
|
||||
$prefix . 'attribute_id',
|
||||
|
|
@ -166,7 +166,7 @@ class NoAclCorrelationBehavior extends ModelBehavior
|
|||
],
|
||||
'order' => false,
|
||||
'limit' => $max_correlations
|
||||
));
|
||||
]);
|
||||
return $correlations;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -31,6 +31,12 @@ class Correlation extends AppModel
|
|||
]
|
||||
);
|
||||
|
||||
public $validEngines = [
|
||||
'Default' => 'default_correlations',
|
||||
'NoAcl' => 'no_acl_correlations',
|
||||
'Legacy' => 'correlations'
|
||||
];
|
||||
|
||||
public $actsAs = array(
|
||||
'Containable'
|
||||
);
|
||||
|
|
@ -255,7 +261,7 @@ class Correlation extends AppModel
|
|||
$this->runBeforeSaveCorrelation($attribute);
|
||||
}
|
||||
|
||||
private function __cachedGetContainData($scope, $id): array
|
||||
private function __cachedGetContainData($scope, $id)
|
||||
{
|
||||
if (!empty($this->getContainRules($scope))) {
|
||||
if (empty($this->__tempContainCache[$scope][$id])) {
|
||||
|
|
@ -345,6 +351,8 @@ class Correlation extends AppModel
|
|||
'Event.disable_correlation' => 0,
|
||||
'Attribute.deleted' => 0,
|
||||
];
|
||||
$correlationLimit = $this->OverCorrelatingValue->getLimit();
|
||||
|
||||
$correlatingAttributes = $this->Attribute->find('all', [
|
||||
'conditions' => $conditions,
|
||||
'recursive' => -1,
|
||||
|
|
@ -352,9 +360,11 @@ class Correlation extends AppModel
|
|||
'contain' => $this->getContainRules(),
|
||||
'order' => [],
|
||||
'callbacks' => 'before', // memory leak fix
|
||||
// let's fetch the limit +1 - still allows us to detect overcorrelations, but we'll also never need more
|
||||
'limit' => empty($correlationLimit) ? null : ($correlationLimit+1)
|
||||
]);
|
||||
|
||||
// Let's check if we don't have a case of an over-correlating attribute
|
||||
$correlationLimit = $this->OverCorrelatingValue->getLimit();
|
||||
$count = count($correlatingAttributes);
|
||||
if ($count > $correlationLimit) {
|
||||
// If we have more correlations for the value than the limit, set the block entry and stop the correlation process
|
||||
|
|
@ -372,9 +382,7 @@ class Correlation extends AppModel
|
|||
$value = $cV;
|
||||
}
|
||||
if ($a['Attribute']['id'] > $b['Attribute']['id']) {
|
||||
if (!$full) {
|
||||
$correlations[] = $this->__createCorrelationEntry($value, $a, $b);
|
||||
}
|
||||
$correlations[] = $this->__createCorrelationEntry($value, $a, $b);
|
||||
} else {
|
||||
$correlations[] = $this->__createCorrelationEntry($value, $b, $a);
|
||||
}
|
||||
|
|
@ -878,4 +886,81 @@ class Correlation extends AppModel
|
|||
}
|
||||
return $attribute;
|
||||
}
|
||||
|
||||
public function collectMetrics()
|
||||
{
|
||||
$results['engine'] = $this->getCorrelationModelName();
|
||||
$results['db'] = [
|
||||
'Default' => [
|
||||
'name' => __('Default correlation engine'),
|
||||
'tables' => [
|
||||
'default_correlations' => [
|
||||
'id_limit' => 4294967295
|
||||
],
|
||||
'correlation_values' => [
|
||||
'id_limit' => 4294967295
|
||||
]
|
||||
]
|
||||
],
|
||||
'NoAcl' => [
|
||||
'name' => __('No ACL correlation engine'),
|
||||
'tables' => [
|
||||
'no_acl_correlations' => [
|
||||
'id_limit' => 4294967295
|
||||
],
|
||||
'correlation_values' => [
|
||||
'id_limit' => 4294967295
|
||||
]
|
||||
]
|
||||
],
|
||||
'Legacy' => [
|
||||
'name' => __('Legacy correlation engine (< 2.4.160)'),
|
||||
'tables' => [
|
||||
'correlations' => [
|
||||
'id_limit' => 2147483647
|
||||
]
|
||||
]
|
||||
]
|
||||
];
|
||||
$results['over_correlations'] = $this->OverCorrelatingValue->find('count');
|
||||
$this->CorrelationExclusion = ClassRegistry::init('CorrelationExclusion');
|
||||
$results['excluded_correlations'] = $this->CorrelationExclusion->find('count');
|
||||
foreach ($results['db'] as &$result) {
|
||||
foreach ($result['tables'] as $table_name => &$table_data) {
|
||||
$size_metrics = $this->query(sprintf('show table status like \'%s\';', $table_name));
|
||||
if (!empty($size_metrics)) {
|
||||
$table_data['size_on_disk'] = $this->query(
|
||||
//'select FILE_SIZE from information_schema.innodb_sys_tablespaces where FILENAME like \'%/' . $table_name . '.ibd\';'
|
||||
sprintf(
|
||||
'select TABLE_NAME, ROUND((DATA_LENGTH + INDEX_LENGTH)) AS size FROM information_schema.TABLES where TABLE_SCHEMA="%s" AND TABLE_NAME="%s"',
|
||||
$this->getDataSource()->config['database'],
|
||||
$table_name
|
||||
)
|
||||
)[0][0]['size'];
|
||||
$last_id = $this->query(sprintf('select max(id) as max_id from %s;', $table_name));
|
||||
$table_data['row_count'] = $size_metrics[0]['TABLES']['Rows'];
|
||||
$table_data['last_id'] = $last_id[0][0]['max_id'];
|
||||
$table_data['id_saturation'] = round(100 * $table_data['last_id'] / $table_data['id_limit'], 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
return $results;
|
||||
}
|
||||
|
||||
public function truncate(array $user, string $engine)
|
||||
{
|
||||
$table = $this->validEngines[$engine];
|
||||
$result = $this->query('truncate table ' . $table);
|
||||
if ($result !== true) {
|
||||
$this->loadLog()->createLogEntry(
|
||||
$user,
|
||||
'truncate',
|
||||
'Correlation',
|
||||
0,
|
||||
'Could not truncate table ' . $table,
|
||||
'Errors: ' . json_encode($result)
|
||||
);
|
||||
}
|
||||
return $result === true;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1938,7 +1938,7 @@ class Event extends AppModel
|
|||
if (!empty($options['includeRelatedTags'])) {
|
||||
$event = $this->includeRelatedTags($event, $options);
|
||||
}
|
||||
$event['RelatedShadowAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id'], true);
|
||||
//$event['RelatedShadowAttribute'] = $this->getRelatedAttributes($user, $event['Event']['id'], true);
|
||||
}
|
||||
$shadowAttributeByOldId = [];
|
||||
if (!empty($event['ShadowAttribute'])) {
|
||||
|
|
|
|||
|
|
@ -304,6 +304,20 @@ class MispObject extends AppModel
|
|||
$object = $this->data['Object'];
|
||||
$this->Attribute->Correlation->updateContainedCorrelations($object, 'object');
|
||||
}
|
||||
if ($this->data['Object']['deleted']) {
|
||||
$attributes_to_delete = $this->Attribute->find('all', [
|
||||
'recursive' => -1,
|
||||
'conditions' => [
|
||||
'Attribute.object_id' => $this->id,
|
||||
'Attribute.deleted' => 0
|
||||
]
|
||||
]);
|
||||
foreach ($attributes_to_delete as &$attribute_to_delete) {
|
||||
$attribute_to_delete['Attribute']['deleted'] = 1;
|
||||
unset($attribute_to_delete['Attribute']['timestamp']);
|
||||
}
|
||||
$this->Attribute->saveMany($attributes_to_delete);
|
||||
}
|
||||
$workflowErrors = [];
|
||||
$logging = [
|
||||
'model' => 'Object',
|
||||
|
|
@ -1015,6 +1029,9 @@ class MispObject extends AppModel
|
|||
$objectId = $this->id;
|
||||
if (!empty($object['Object']['Attribute'])) {
|
||||
foreach ($object['Object']['Attribute'] as $attribute) {
|
||||
if (!empty($object['Object']['deleted'])) {
|
||||
$attribute['deleted'] = 1;
|
||||
}
|
||||
$this->Attribute->captureAttribute($attribute, $eventId, $user, $objectId, false, $parentEvent);
|
||||
}
|
||||
}
|
||||
|
|
@ -1099,6 +1116,9 @@ class MispObject extends AppModel
|
|||
}
|
||||
if (!empty($object['Attribute'])) {
|
||||
foreach ($object['Attribute'] as $attribute) {
|
||||
if (!empty($object['deleted'])) {
|
||||
$attribute['deleted'] = 1;
|
||||
}
|
||||
$result = $this->Attribute->editAttribute($attribute, $event, $user, $object['id'], false, $force);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3160,13 +3160,6 @@ class Server extends AppModel
|
|||
$tableIndexDiff = array_diff(array_keys($indexes), array_keys($actualIndex[$tableName])); // check for missing indexes
|
||||
foreach ($tableIndexDiff as $columnDiff) {
|
||||
$shouldBeUnique = $indexes[$columnDiff];
|
||||
if ($shouldBeUnique && !$this->checkIfColumnContainsJustUniqueValues($tableName, $columnDiff)) {
|
||||
$indexDiff[$tableName][$columnDiff] = array(
|
||||
'message' => __('Column `%s` should be unique indexed, but contains duplicate values', $columnDiff),
|
||||
'sql' => '',
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
$message = __('Column `%s` should be indexed', $columnDiff);
|
||||
$indexDiff[$tableName][$columnDiff] = array(
|
||||
|
|
@ -3194,15 +3187,6 @@ class Server extends AppModel
|
|||
'sql' => $sql,
|
||||
);
|
||||
} else {
|
||||
if (!$this->checkIfColumnContainsJustUniqueValues($tableName, $column)) {
|
||||
$message = __('Column `%s` should be unique index, but contains duplicate values', $column);
|
||||
$indexDiff[$tableName][$column] = array(
|
||||
'message' => $message,
|
||||
'sql' => '',
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
$sql = $this->generateSqlDropIndexQuery($tableName, $column);
|
||||
$sql .= '<br>' . $this->generateSqlIndexQuery($dbExpectedSchema, $tableName, $column, true);
|
||||
|
||||
|
|
@ -7479,6 +7463,7 @@ class Server extends AppModel
|
|||
'Get IPs for user ID' => 'MISP/app/Console/cake Admin UserIP [user_id]',
|
||||
'Get user ID for user IP' => 'MISP/app/Console/cake Admin IPUser [ip]',
|
||||
'Generate correlation' => 'MISP/app/Console/cake Admin jobGenerateCorrelation [job_id]',
|
||||
'Truncate correlation table' => 'MISP/app/Console/cake Admin truncateTable [user_id] [correlation_engine_name] [job_id]',
|
||||
'Purge correlation' => 'MISP/app/Console/cake Admin jobPurgeCorrelation [job_id]',
|
||||
'Generate shadow attribute correlation' => 'MISP/app/Console/cake Admin jobGenerateShadowAttributeCorrelation [job_id]',
|
||||
'Update MISP' => 'MISP/app/Console/cake Admin updateMISP',
|
||||
|
|
|
|||
|
|
@ -0,0 +1,26 @@
|
|||
<div class="confirmation">
|
||||
<?php
|
||||
echo $this->Form->create('Attribute', ['style' => 'margin:0px;', 'id' => 'PromptForm', 'url' => $baseurl . '/attributes/generateCorrelation']);
|
||||
$message = __('Recorrelate instance', );
|
||||
$buttonTitle = __('Recorrelate');
|
||||
?>
|
||||
<legend><?= $message ?></legend>
|
||||
<div style="padding-left:5px;padding-right:5px;padding-bottom:5px;">
|
||||
<?php
|
||||
echo '<p>' . __('Are you sure you wish to start a recorrelation for the currently active correlation engine?') . '</p>';
|
||||
echo '<p>' . __('Depending on the system and the amount of attributes, this might take a long time.') . '</p>';
|
||||
?>
|
||||
<table>
|
||||
<tr>
|
||||
<td style="vertical-align:top">
|
||||
<button role="button" tabindex="0" aria-label="<?= $buttonTitle ?>" title="<?= $buttonTitle ?>" id="PromptYesButton" class="btn btn-primary"><?= __('Yes') ?></button>
|
||||
</td>
|
||||
<td style="width:100%;"></td>
|
||||
<td style="vertical-align:top;">
|
||||
<span role="button" tabindex="0" aria-label="<?= __('Cancel');?>" title="<?= __('Cancel');?>" class="btn btn-inverse" id="PromptNoButton" onclick="cancelPrompt()"><?= __('No');?></span>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
<?= $this->Form->end(); ?>
|
||||
</div>
|
||||
|
|
@ -2,7 +2,6 @@
|
|||
|
||||
$modules = isset($modules) ? $modules : null;
|
||||
$cortex_modules = isset($cortex_modules) ? $cortex_modules : null;
|
||||
|
||||
echo '<div class="index">';
|
||||
echo $this->element('/genericElements/IndexTable/index_table', [
|
||||
'data' => [
|
||||
|
|
|
|||
|
|
@ -0,0 +1,25 @@
|
|||
<div class="confirmation">
|
||||
<?php
|
||||
echo $this->Form->create('Correlation', ['style' => 'margin:0px;', 'id' => 'PromptForm', 'url' => $baseurl . '/correlations/switchEngine/' . urlencode($engine)]);
|
||||
$message = __('Switch Engine');
|
||||
$buttonTitle = __('Switch');
|
||||
?>
|
||||
<legend><?= $message ?></legend>
|
||||
<div style="padding-left:5px;padding-right:5px;padding-bottom:5px;">
|
||||
<?php
|
||||
echo '<p>' . __('Are you sure you want to switch to the given correlation engine (' . h($engine) . ')? If so, it is highly recommended that you recorrelate afterwards.') . '</p>';
|
||||
?>
|
||||
<table>
|
||||
<tr>
|
||||
<td style="vertical-align:top">
|
||||
<button role="button" tabindex="0" aria-label="<?= $buttonTitle ?>" title="<?= $buttonTitle ?>" id="PromptYesButton" class="btn btn-primary"><?= __('Yes') ?></button>
|
||||
</td>
|
||||
<td style="width:100%;"></td>
|
||||
<td style="vertical-align:top;">
|
||||
<span role="button" tabindex="0" aria-label="<?= __('Cancel');?>" title="<?= __('Cancel');?>" class="btn btn-inverse" id="PromptNoButton" onclick="cancelPrompt()"><?= __('No');?></span>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
<?= $this->Form->end(); ?>
|
||||
</div>
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
<div class="confirmation">
|
||||
<?php
|
||||
echo $this->Form->create('Correlation', ['style' => 'margin:0px;', 'id' => 'PromptForm', 'url' => $baseurl . '/correlations/truncate/' . urlencode($engine)]);
|
||||
$message = __('Truncate database of the associated correlation engine');
|
||||
$buttonTitle = __('Truncate');
|
||||
?>
|
||||
<legend><?= $message ?></legend>
|
||||
<div style="padding-left:5px;padding-right:5px;padding-bottom:5px;">
|
||||
<?php
|
||||
echo '<p>' . __('Are you sure you want to truncate the correlation table (' . h($table_name) . ') used by the disabled correlation engine ' . h($engine) . '?') . '</p>';
|
||||
?>
|
||||
<table>
|
||||
<tr>
|
||||
<td style="vertical-align:top">
|
||||
<button role="button" tabindex="0" aria-label="<?= $buttonTitle ?>" title="<?= $buttonTitle ?>" id="PromptYesButton" class="btn btn-primary"><?= __('Yes') ?></button>
|
||||
</td>
|
||||
<td style="width:100%;"></td>
|
||||
<td style="vertical-align:top;">
|
||||
<span role="button" tabindex="0" aria-label="<?= __('Cancel');?>" title="<?= __('Cancel');?>" class="btn btn-inverse" id="PromptNoButton" onclick="cancelPrompt()"><?= __('No');?></span>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
<?= $this->Form->end(); ?>
|
||||
</div>
|
||||
|
|
@ -57,11 +57,18 @@
|
|||
);
|
||||
}
|
||||
}
|
||||
if (!empty($object['over_correlation']) || !empty($object['correlation_exclusion'])) {
|
||||
if (!empty($object['correlation_exclusion'])) {
|
||||
echo sprintf(
|
||||
'<span class="bold red">%s</span> ',
|
||||
'<span class="bold red" title="%s">%s</span> ',
|
||||
__('The attribute value matches a correlation exclusion rule defined by a site-administrator for this instance. Click the magnifying glass to search for all occurrences of the value.'),
|
||||
__('Excluded.')
|
||||
);
|
||||
} else if (!empty($object['over_correlation'])) {
|
||||
echo sprintf(
|
||||
'<span class="bold red" title="%s">%s</span> ',
|
||||
__('The instance threshold for the maximum number of correlations for the given attribute value has been exceeded. Click the magnifying glass to search for all occurrences of the value.'),
|
||||
__('Too many correlations.')
|
||||
);
|
||||
}
|
||||
echo $this->Html->link(
|
||||
'',
|
||||
|
|
|
|||
|
|
@ -1,18 +1,20 @@
|
|||
<?php
|
||||
|
||||
$object = Hash::extract($row, $field['data']['object']['value_path']);
|
||||
$event = Hash::extract($row, 'Event');
|
||||
if (!empty($object['RelatedAttribute'])) {
|
||||
$event['RelatedAttribute'] = array($object['id'] => $object['RelatedAttribute']);
|
||||
}
|
||||
|
||||
if (!empty($event['RelatedAttribute'][$object['id']])) {
|
||||
|
||||
echo '<ul class="inline" style="margin:0">';
|
||||
echo $this->element('Events/View/attribute_correlations', array(
|
||||
'scope' => $field['data']['scope'],
|
||||
'object' => $object,
|
||||
'event' => $event,
|
||||
));
|
||||
echo '</ul>';
|
||||
}
|
||||
$object = Hash::extract($row, $field['data']['object']['value_path']);
|
||||
$event = Hash::extract($row, 'Event');
|
||||
if (!empty($object['RelatedAttribute'])) {
|
||||
$event['RelatedAttribute'] = array($object['id'] => $object['RelatedAttribute']);
|
||||
}
|
||||
foreach ($event['RelatedAttribute'] as $k => &$ra) {
|
||||
if (!empty($ra['Event'])) {
|
||||
$ra['info'] = $ra['Event']['info'];
|
||||
$ra['org_id'] = $ra['Event']['org_id'];
|
||||
}
|
||||
}
|
||||
echo sprintf(
|
||||
'<ul class="inline" style="margin:0">%s</ul>',
|
||||
$this->element('Events/View/attribute_correlations', [
|
||||
'scope' => $field['data']['scope'],
|
||||
'object' => $object,
|
||||
'event' => $event,
|
||||
])
|
||||
);
|
||||
|
|
|
|||
|
|
@ -0,0 +1,45 @@
|
|||
<?php
|
||||
echo '<div style="border:1px solid #dddddd; margin-top:1px; width:100%; padding:10px">';
|
||||
echo sprintf(
|
||||
'<p>%s</p><p>%s</p>',
|
||||
__('This is the correlation management interface. Its goal is to provide youwith information about the currently used correlation engine as well as the data stores of currently dormant engines.'),
|
||||
__('You will also find management tools for the various engines below, make sure that you keep an eye on the disk requirements as well as the exhaustion of IDs and recorrelate the instance when needed.')
|
||||
);
|
||||
echo sprintf(
|
||||
'<div style="width:300px;">%s</div>',
|
||||
$this->element(
|
||||
'/healthElements/correlations_generic_data',
|
||||
[
|
||||
'correlation_metrics' => $correlation_metrics
|
||||
]
|
||||
)
|
||||
);
|
||||
$currentEngineData = $correlation_metrics['db'][$correlation_metrics['engine']];
|
||||
unset($correlation_metrics['db'][$correlation_metrics['engine']]);
|
||||
|
||||
echo sprintf(
|
||||
'<hr /><h2 class="overflow label label-success">%s</h2><div style="width:800px;">%s<div>%s</div></div>',
|
||||
__('Active engine: %s', $currentEngineData['name']),
|
||||
$this->element('/healthElements/correlations_table', ['currentEngineData' => $currentEngineData]),
|
||||
sprintf(
|
||||
'<div class="btn btn-primary" onClick="simplePopup(\'%s\');">Recorrelate</div>',
|
||||
$baseurl . '/attributes/generateCorrelation'
|
||||
)
|
||||
);
|
||||
foreach ($correlation_metrics['db'] as $engine => $engineData) {
|
||||
echo sprintf(
|
||||
'<hr /><h2 class="overflow label">%s</h2><div style="width:800px;">%s<div>%s %s</div></div>',
|
||||
__('Dormant engine: %s', $engineData['name']),
|
||||
$this->element('/healthElements/correlations_table', ['currentEngineData' => $engineData]),
|
||||
$engine === 'Legacy' ? '' : sprintf(
|
||||
'<div class="btn btn-primary" onClick="simplePopup(\'%s\');">Activate engine</div>',
|
||||
$baseurl . '/correlations/switchEngine/' . h($engine)
|
||||
),
|
||||
sprintf(
|
||||
'<div class="btn btn-danger" onClick="simplePopup(\'%s\');">Truncate</div>',
|
||||
$baseurl . '/correlations/truncate/' . h($engine)
|
||||
)
|
||||
);
|
||||
}
|
||||
echo '</div>';
|
||||
?>
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
<?php
|
||||
|
||||
$rows = '';
|
||||
$rows .= sprintf(
|
||||
'<tr><td><a href="%s">%s</a></td><td style="text-align:right;">%s</td></tr>',
|
||||
$baseurl . '/correlations/overCorrelations',
|
||||
__('Over correlations'),
|
||||
h($correlation_metrics['over_correlations'])
|
||||
);
|
||||
$rows .= sprintf(
|
||||
'<tr><td><a href="%s">%s</a></td><td style="text-align:right;">%s</td></tr>',
|
||||
$baseurl . '/correlation_exclusions/index',
|
||||
__('Excluded correlations'),
|
||||
h($correlation_metrics['excluded_correlations'])
|
||||
);
|
||||
echo sprintf(
|
||||
'<table class="meta_table table table-striped table-condensed"><tr><th>Field</th><th style="text-align:right;">Value</th></tr>%s</table>',
|
||||
$rows
|
||||
);
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
<?php
|
||||
$engine_contents = sprintf(
|
||||
'<tr><th title="%s">%s</th><th title="%s">%s</th><th title="%s">%s</th><th title="%s">%s</th></tr>',
|
||||
__('Table name'),
|
||||
__('Table'),
|
||||
__('Number of entries in the table'),
|
||||
__('# of rows'),
|
||||
__('The table\'s size in MB on disk'),
|
||||
__('Size on disk'),
|
||||
__('The saturation of the ID space of the given table. Be careful, reaching the limit will block further correlations from being created - make sure you recorrelate in time or extend the ID space by changing the column type.'),
|
||||
__('ID space saturation'),
|
||||
);
|
||||
foreach ($currentEngineData['tables'] as $table_name => $table_data) {
|
||||
$engine_contents .= sprintf(
|
||||
'<tr><td>%s</td><td>%s</td><td title="%s">%s</td><td title="%s">%s</td></tr>',
|
||||
h($table_name),
|
||||
h($table_data['row_count']),
|
||||
h($table_data['size_on_disk']) . ' B',
|
||||
h(round($table_data['size_on_disk']/1024/1024), 2) . ' MB',
|
||||
sprintf(
|
||||
"Last inserted correlation ID: %s\nHighest possible ID: %s",
|
||||
h($table_data['last_id']),
|
||||
h($table_data['id_limit']),
|
||||
),
|
||||
h($table_data['id_saturation']) . '%'
|
||||
|
||||
);
|
||||
}
|
||||
echo sprintf(
|
||||
'<table class="meta_table table table-striped table-condensed">%s</table>',
|
||||
$engine_contents
|
||||
);
|
||||
|
|
@ -15,7 +15,7 @@
|
|||
foreach ($tabs as $k => $tab) {
|
||||
$data['children'][0]['children'][] = array(
|
||||
'html' => sprintf(
|
||||
__('%s settings%s'),
|
||||
__('%s %s'),
|
||||
ucfirst(h($k)),
|
||||
($tab['errors'] == 0) ? '' : sprintf(
|
||||
' (<span>%s%s</span>)',
|
||||
|
|
@ -27,6 +27,13 @@
|
|||
'active' => $k == $active_tab
|
||||
);
|
||||
}
|
||||
|
||||
$data['children'][0]['children'][] = array(
|
||||
'url' => $baseurl . '/servers/serverSettings/correlations',
|
||||
'html' => __('Correlations'),
|
||||
'active' => $active_tab === 'correlations'
|
||||
);
|
||||
|
||||
$data['children'][0]['children'][] = array(
|
||||
'url' => $baseurl . '/servers/serverSettings/diagnostics',
|
||||
'html' => sprintf(
|
||||
|
|
|
|||
|
|
@ -13,6 +13,8 @@
|
|||
echo $this->element('healthElements/workers');
|
||||
} else if ($tab === 'files') {
|
||||
echo $this->element('healthElements/files');
|
||||
} else if ($tab === 'correlations') {
|
||||
echo $this->element('healthElements/correlations');
|
||||
} else {
|
||||
echo $this->element('healthElements/overview');
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
Subproject commit 2330c1760250dd4b92623fab1b4f3ffa1523ec49
|
||||
Subproject commit bfda561f5f29a8cca573e22789fb58252ad36c93
|
||||
|
|
@ -1 +1 @@
|
|||
Subproject commit 50f61a03beb9aac4256a82348b5cb2f4ed6d1932
|
||||
Subproject commit 9b9c8389616b516a2d5cbe41a4407b2e9d7f24d3
|
||||
|
|
@ -1 +1 @@
|
|||
Subproject commit f4fb812c37a201b93f8b07ddddc293ee1d5110bd
|
||||
Subproject commit fc12a106f5481e466275799de71e29dd7cf764f7
|
||||
|
|
@ -1 +1 @@
|
|||
Subproject commit f270e406bcad4744b81cb02fc7b39cba8c0d5e99
|
||||
Subproject commit fc5599114f92926838b519f322b81a061039fe0e
|
||||
Loading…
Reference in New Issue