MISP
/
MISP
mirror of https://github.com/MISP/MISP
2
2
Fork 0
Code Issues Releases Wiki Activity

Merge pull request #8572 from JakubOnderka/correlation-value-transaction 

chg: [correlation] Faster saving correlations
pull/8489/head
Jakub Onderka 2022-09-10 20:44:29 +02:00 committed by GitHub
parent f155818167 b98be56f41
commit f8b5b22aea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 179 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
app/Console/Command/AdminShell.php
View File

@ -22,8 +22,6 @@ class AdminShell extends AppShell
'Role', 'Feed', 'SharingGroupBlueprint', 'Correlation', 'OverCorrelatingValue'
];
public $tasks = ['ConfigLoad'];
public function getOptionParser()
{
$parser = parent::getOptionParser();
@ -105,7 +103,6 @@ class AdminShell extends AppShell
public function jobGenerateCorrelation()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Generate correlation'] . PHP_EOL);
}
@ -116,7 +113,6 @@ class AdminShell extends AppShell
public function jobGenerateOccurrences()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Generate over-correlation occurrences'] . PHP_EOL);
}
@ -138,7 +134,6 @@ class AdminShell extends AppShell
public function jobGenerateShadowAttributeCorrelation()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Generate shadow attribute correlation'] . PHP_EOL);
}
@ -152,14 +147,12 @@ class AdminShell extends AppShell
public function updateMISP()
{
$this->ConfigLoad->execute();
$status = array('branch' => '2.4');
echo $this->Server->update($status) . PHP_EOL;
}
public function updateAfterPull()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Update after pull'] . PHP_EOL);
}
@ -186,7 +179,6 @@ class AdminShell extends AppShell
$this->error('This method does nothing when SimpleBackgroundJobs are enabled.');
}
$this->ConfigLoad->execute();
$this->Server->restartWorkers();
echo PHP_EOL . 'Workers restarted.' . PHP_EOL;
}
@ -197,7 +189,6 @@ class AdminShell extends AppShell
$this->error('This method does nothing when SimpleBackgroundJobs are enabled.');
}
$this->ConfigLoad->execute();
if (empty($this->args[0]) || !is_numeric($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['worker_management_tasks']['data']['Restart a worker'] . PHP_EOL);
}
@ -223,7 +214,6 @@ class AdminShell extends AppShell
$this->error('This method does nothing when SimpleBackgroundJobs are enabled.');
}
$this->ConfigLoad->execute();
if (empty($this->args[0]) || !is_numeric($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['worker_management_tasks']['data']['Kill a worker'] . PHP_EOL);
}
@ -244,7 +234,6 @@ class AdminShell extends AppShell
$this->error('This method does nothing when SimpleBackgroundJobs are enabled.');
}
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['worker_management_tasks']['data']['Start a worker'] . PHP_EOL);
}
@ -276,7 +265,6 @@ class AdminShell extends AppShell
public function updateGalaxies()
{
$this->ConfigLoad->execute();
// The following is 7.x upwards only
//$value = $this->args[0] ?? $this->args[0] ?? 0;
$value = empty($this->args[0]) ? null : $this->args[0];
@ -347,7 +335,6 @@ class AdminShell extends AppShell
public function updateNoticeLists()
{
$this->ConfigLoad->execute();
$result = $this->Noticelist->update();
if ($result) {
echo 'Notice lists updated' . PHP_EOL;
@ -359,7 +346,6 @@ class AdminShell extends AppShell
# FIXME: Fails to pass userId/orgId properly, global update works.
public function updateObjectTemplates()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Update object templates'] . PHP_EOL);
} else {
@ -392,7 +378,6 @@ class AdminShell extends AppShell
public function jobUpgrade24()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Job upgrade'] . PHP_EOL);
}
@ -410,7 +395,6 @@ class AdminShell extends AppShell
public function prune_update_logs()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Prune update logs'] . PHP_EOL);
}
@ -429,7 +413,6 @@ class AdminShell extends AppShell
public function getWorkers()
{
$this->ConfigLoad->execute();
$result = $this->Server->workerDiagnostics($workerIssueCount);
$query = 'all';
if (!empty($this->args[0])) {
@ -501,7 +484,6 @@ class AdminShell extends AppShell
public function setDatabaseVersion()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Set database version'] . PHP_EOL);
} else {
@ -580,7 +562,6 @@ class AdminShell extends AppShell
public function setDefaultRole()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || !is_numeric($this->args[0])) {
$roles = $this->Role->find('list', array(
'fields' => array('id', 'name')
@ -615,7 +596,6 @@ class AdminShell extends AppShell
*/
public function change_authkey()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
echo 'MISP apikey command line tool' . PHP_EOL . 'To assign a new random API key for a user: ' . APP . 'Console/cake Admin change_authkey [user_email]' . PHP_EOL . 'To assign a fixed API key: ' . APP . 'Console/cake Admin change_authkey [user_email] [authkey]' . PHP_EOL;
die();
@ -646,7 +626,6 @@ class AdminShell extends AppShell
public function recoverSinceLastSuccessfulUpdate()
{
$this->ConfigLoad->execute();
$this->loadModel('Log');
$logs = $this->Log->find('all', array(
'conditions' => array(
@ -685,7 +664,6 @@ class AdminShell extends AppShell
public function cleanCaches()
{
$this->ConfigLoad->execute();
echo 'Cleaning caches...' . PHP_EOL;
$this->Server->cleanCacheFiles();
echo '...caches lost in time, like tears in rain.' . PHP_EOL;
@ -693,7 +671,6 @@ class AdminShell extends AppShell
public function resetSyncAuthkeys()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
echo sprintf(
__("MISP mass sync authkey reset command line tool" . PHP_EOL . "Usage: %sConsole/cake Admin resetSyncAuthkeys [user_id]" . PHP_EOL), APP
@ -719,7 +696,6 @@ class AdminShell extends AppShell
public function purgeFeedEvents()
{
$this->ConfigLoad->execute();
if (
(empty($this->args[0]) || !is_numeric($this->args[0])) ||
(empty($this->args[1]) || !is_numeric($this->args[1]))
@ -759,7 +735,6 @@ class AdminShell extends AppShell
*/
public function UserIP()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get IPs for user ID'] . PHP_EOL);
}
@ -787,7 +762,6 @@ class AdminShell extends AppShell
*/
public function IPUser()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Get user ID for user IP'] . PHP_EOL);
}
@ -1207,7 +1181,6 @@ class AdminShell extends AppShell
public function truncateTable()
{
$this->ConfigLoad->execute();
if (!isset($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_admin_tasks']['data']['Truncate table correlation'] . PHP_EOL);
}

3
app/Console/Command/AppShell.php
View File

@ -32,9 +32,10 @@ class AppShell extends Shell
public function initialize()
{
parent::initialize();
$this->ConfigLoad = $this->Tasks->load('ConfigLoad');
$this->ConfigLoad->execute();
parent::initialize();
}
public function perform()

3
app/Console/Command/AuthkeyShell.php
View File

@ -10,13 +10,10 @@ class AuthkeyShell extends AppShell {
public $uses = array('User', 'Log');
public $tasks = array('ConfigLoad');
public function main()
{
$this->err('This method is deprecated. Next time please use `cake user change_authkey [user] [authkey]` command.');
$this->ConfigLoad->execute();
if (!isset($this->args[0]) || empty($this->args[0])) echo 'MISP authkey reset command line tool.' . PHP_EOL . 'To assign a new authkey for a user:' . PHP_EOL . APP . 'Console/cake Authkey [email] [auth_key | optional]' . PHP_EOL;
else {
// get the users that need their password hashed

1
app/Console/Command/BaseurlShell.php
View File

@ -13,7 +13,6 @@ class BaseurlShell extends AppShell {
{
$this->err('This method is deprecated. Next time please use `cake admin setSetting MISP.baseurl [baseurl]` command.');
$this->ConfigLoad->execute();
$baseurl = $this->args[0];
$result = $this->Server->testBaseURL($baseurl);
if (true !== $result) {

13
app/Console/Command/EventShell.php
View File

@ -15,7 +15,6 @@ require_once 'AppShell.php';
class EventShell extends AppShell
{
public $uses = array('Event', 'Post', 'Attribute', 'Job', 'User', 'Task', 'Allowedlist', 'Server', 'Organisation', 'Correlation', 'Tag');
public $tasks = array('ConfigLoad');
public function getOptionParser()
{
@ -121,7 +120,6 @@ class EventShell extends AppShell
public function doPublish()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Do publish'] . PHP_EOL);
}
@ -157,7 +155,6 @@ class EventShell extends AppShell
public function correlateValue()
{
$this->ConfigLoad->execute();
$value = $this->args[0];
if (!empty($this->args[1])) {
@ -182,7 +179,6 @@ class EventShell extends AppShell
public function cache()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Cache event'] . PHP_EOL);
}
@ -223,7 +219,6 @@ class EventShell extends AppShell
private function __runCaching($user, $typeData, $id, $export_type, $subType = '')
{
$this->ConfigLoad->execute();
$export_type = strtolower($typeData['type']);
$final = $this->{$typeData['scope']}->restSearch($user, $typeData['params']['returnFormat'], $typeData['params'], false, $id);
$dir = new Folder(APP . 'tmp/cached_exports/' . $export_type, true, 0750);
@ -240,7 +235,6 @@ class EventShell extends AppShell
public function cachebro()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1])) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Cache bro'] . PHP_EOL);
}
@ -281,7 +275,6 @@ class EventShell extends AppShell
public function alertemail()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Alert email'] . PHP_EOL);
}
@ -314,7 +307,6 @@ class EventShell extends AppShell
public function postsemail()
{
$this->ConfigLoad->execute();
if (
empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) ||
empty($this->args[3]) || empty($this->args[4]) || empty($this->args[5])
@ -348,7 +340,6 @@ class EventShell extends AppShell
public function enqueueCaching()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Enqueue caching'] . PHP_EOL);
}
@ -405,7 +396,6 @@ class EventShell extends AppShell
public function publish()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[2]) || empty($this->args[3])) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Publish event'] . PHP_EOL);
}
@ -465,7 +455,6 @@ class EventShell extends AppShell
public function publish_galaxy_clusters()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || !array_key_exists(3, $this->args)) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Publish Galaxy clusters'] . PHP_EOL);
}
@ -492,7 +481,6 @@ class EventShell extends AppShell
public function enrichment()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Run enrichment'] . PHP_EOL);
}
@ -588,7 +576,6 @@ class EventShell extends AppShell
public function recoverEvent()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1])) {
die('Usage: ' . $this->Server->command_line_functions['event_management_tasks']['data']['Recover event'] . PHP_EOL);
}

1
app/Console/Command/LiveShell.php
View File

@ -10,7 +10,6 @@ class LiveShell extends AppShell {
public function main()
{
$this->ConfigLoad->execute();
$live = $this->args[0];
if ($live != 0 && $live != 1) {
echo 'Invalid parameters. Usage: /var/www/MISP/app/Console/cake Live [0|1]';

2
app/Console/Command/PasswordShell.php
View File

@ -14,7 +14,6 @@ class PasswordShell extends AppShell {
{
$this->err('This method is deprecated. Next time please use `cake user change_pw [user] [password]` command.');
$this->ConfigLoad->execute();
if (!isset($this->args[0]) || empty($this->args[0]) || !isset($this->args[1]) || empty($this->args[1])) echo 'MISP password reset command line tool.' . PHP_EOL . 'To assign a new password for a user:' . PHP_EOL . APP . 'Console/cake Password [email] [password]' . PHP_EOL;
else {
// get the users that need their password hashed
@ -42,7 +41,6 @@ class PasswordShell extends AppShell {
public function getOptionParser()
{
$this->ConfigLoad->execute();
$parser = parent::getOptionParser();
$parser->addOption('override_password_change', array(
'short' => 'o',

5
app/Console/Command/ServerShell.php
View File

@ -57,7 +57,6 @@ class ServerShell extends AppShell
public function pullAll()
{
$this->ConfigLoad->execute();
if (empty($this->args[0])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['PullAll'] . PHP_EOL);
}
@ -367,7 +366,6 @@ class ServerShell extends AppShell
public function enqueuePull()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Enqueue pull'] . PHP_EOL);
}
@ -430,7 +428,6 @@ class ServerShell extends AppShell
public function enqueueFeedFetch()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Enqueue feed fetch'] . PHP_EOL);
}
@ -480,7 +477,6 @@ class ServerShell extends AppShell
public function enqueueFeedCache()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Enqueue feed cache'] . PHP_EOL);
}
@ -537,7 +533,6 @@ class ServerShell extends AppShell
public function enqueuePush()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2])) {
die('Usage: ' . $this->Server->command_line_functions['console_automation_tasks']['data']['Enqueue push'] . PHP_EOL);
}

3
app/Console/Command/WorkflowShell.php
View File

@ -6,11 +6,9 @@ require_once 'AppShell.php';
class WorkflowShell extends AppShell {
public $uses = ['Job', 'Workflow'];
public $tasks = ['ConfigLoad'];
public function executeWorkflowForTrigger()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || empty($this->args[3])) {
die(__('Invalid number of arguments.'));
}
@ -39,7 +37,6 @@ class WorkflowShell extends AppShell {
public function walkGraph()
{
$this->ConfigLoad->execute();
if (empty($this->args[0]) || empty($this->args[1]) || empty($this->args[2]) || empty($this->args[3])) {
die(__('Invalid number of arguments.'));
}

32
app/Model/AppModel.php
View File

@ -48,6 +48,9 @@ class AppModel extends Model
/** @var AttachmentTool|null */
private $attachmentTool;
/** @var Workflow|null */
private $Workflow;
// deprecated, use $db_changes
// major -> minor -> hotfix -> requires_logout
const OLD_DB_CHANGES = array(
@ -3736,22 +3739,6 @@ class AppModel extends Model
return $dataSourceName === 'Database/Mysql' || $dataSourceName === 'Database/MysqlObserver' || $dataSourceName === 'Database/MysqlExtended' || $dataSource instanceof Mysql;
}
public function getCorrelationModelName()
{
if (!empty(Configure::read('MISP.correlation_engine'))) {
return Configure::read('MISP.correlation_engine');
}
return 'Default';
}
public function loadCorrelationModel()
{
if (!empty(Configure::read('MISP.correlation_engine'))) {
return ClassRegistry::init(Configure::read('MISP.correlation_engine'));
}
return ClassRegistry::init('Correlation');
}
/**
* executeTrigger
*
@ -3763,9 +3750,6 @@ class AppModel extends Model
*/
public function executeTrigger($trigger_id, array $data=[], array &$blockingErrors=[], array $logging=[]): bool
{
if ($this->Workflow === null) {
$this->Workflow = ClassRegistry::init('Workflow');
}
if ($this->isTriggerCallable($trigger_id)) {
$success = $this->Workflow->executeWorkflowForTriggerRouter($trigger_id, $data, $blockingErrors, $logging);
if (!empty($logging) && empty($success)) {
@ -3787,13 +3771,6 @@ class AppModel extends Model
$this->Workflow->checkTriggerListenedTo($trigger_id);
}
public function addPendingLogEntry($logEntry)
{
$logEntries = Configure::read('pendingLogEntries');
$logEntries[] = $logEntry;
Configure::write('pendingLogEntries', $logEntries);
}
/**
* Use different CakeEventManager to fix memory leak
* @return CakeEventManager
@ -3808,7 +3785,8 @@ class AppModel extends Model
return $this->_eventManager;
}
private function __retireOldCorrelationEngine($user = null) {
private function __retireOldCorrelationEngine($user = null)
{
if ($user === null) {
$user = [
'id' => 0,

5
app/Model/Attribute.php
View File

@ -107,18 +107,19 @@ class Attribute extends AppModel
const UPLOAD_DEFINITIONS = ['attachment'];
// skip Correlation for the following types
const NON_CORRELATING_TYPES = array(
const NON_CORRELATING_TYPES = [
'comment',
'http-method',
'aba-rtn',
'gender',
'counter',
'float',
'port',
'nationality',
'cortex',
'boolean',
'anonymised'
);
];
const PRIMARY_ONLY_CORRELATING_TYPES = array(
'ip-src|port',

22
app/Model/Behavior/DefaultCorrelationBehavior.php
View File

@ -58,12 +58,18 @@ class DefaultCorrelationBehavior extends ModelBehavior
return self::TABLE_NAME;
}
/**
* @param Model $Model
* @param string $value
* @param array $a
* @param array $b
* @return array
*/
public function createCorrelationEntry(Model $Model, $value, $a, $b)
{
$valueId = $this->Correlation->CorrelationValue->getValueId($value);
if ($this->deadlockAvoidance) {
return [
'value_id' => $valueId,
'value_id' => $value,
'1_event_id' => $a['Event']['id'],
'1_object_id' => $a['Attribute']['object_id'],
'1_attribute_id' => $a['Attribute']['id'],
@ -87,7 +93,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
];
} else {
return [
(int) $valueId,
$value,
(int) $a['Event']['id'],
(int) $a['Attribute']['object_id'],
(int) $a['Attribute']['id'],
@ -112,7 +118,7 @@ class DefaultCorrelationBehavior extends ModelBehavior
}
}
public function saveCorrelations(Model $Model, $correlations)
public function saveCorrelations(Model $Model, array $correlations)
{
$fields = [
'value_id',
@ -138,14 +144,16 @@ class DefaultCorrelationBehavior extends ModelBehavior
'object_sharing_group_id'
];
$this->Correlation->CorrelationValue->replaceValueWithId($correlations, $this->deadlockAvoidance ? 'value_id' : 0);
if ($this->deadlockAvoidance) {
return $this->Correlation->saveMany($correlations, array(
return $this->Correlation->saveMany($correlations, [
'atomic' => false,
'callbacks' => false,
'deep' => false,
'validate' => false,
'fieldList' => $fields
));
'fieldList' => $fields,
]);
} else {
$db = $this->Correlation->getDataSource();
// Split to chunks datasource is is enabled

98
app/Model/Behavior/NoAclCorrelationBehavior.php
View File

@ -1,15 +1,12 @@
<?php
App::uses('AppModel', 'Model');
App::uses('RandomTool', 'Tools');
/**
* ACL-less correlation behaviour for end-point instances
*/
class NoAclCorrelationBehavior extends ModelBehavior
{
private $__tableName = 'no_acl_correlations';
const TABLE_NAME = 'no_acl_correlations';
private $__config = [
'AttributeFetcher' => [
@ -31,26 +28,36 @@ class NoAclCorrelationBehavior extends ModelBehavior
]
];
public $Correlation = null;
/** @var Correlation */
public $Correlation;
/** @var bool */
private $deadlockAvoidance = false;
public function setup(Model $Model, $settings = []) {
$Model->useTable = $this->__tableName;
public function setup(Model $Model, $settings = [])
{
$Model->useTable = self::TABLE_NAME;
$this->Correlation = $Model;
$this->deadlockAvoidance = $settings['deadlockAvoidance'];
}
public function getTableName(Model $Model)
{
return $this->__tableName;
return self::TABLE_NAME;
}
public function createCorrelationEntry(Model $Model, $value, $a, $b) {
$value_id = $this->Correlation->CorrelationValue->getValueId($value);
/**
* @param Model $Model
* @param string $value
* @param array $a
* @param array $b
* @return array|int[]
*/
public function createCorrelationEntry(Model $Model, $value, $a, $b)
{
if ($this->deadlockAvoidance) {
return [
'value_id' => $value_id,
'value_id' => $value,
'1_event_id' => $a['Event']['id'],
'1_attribute_id' => $a['Attribute']['id'],
'event_id' => $b['Event']['id'],
@ -58,7 +65,7 @@ class NoAclCorrelationBehavior extends ModelBehavior
];
} else {
return [
(int) $value_id,
$value,
(int) $a['Event']['id'],
(int) $a['Attribute']['id'],
(int) $b['Event']['id'],
@ -67,7 +74,12 @@ class NoAclCorrelationBehavior extends ModelBehavior
}
}
public function saveCorrelations(Model $Model, $correlations)
/**
* @param Model $Model
* @param array $correlations
* @return bool
*/
public function saveCorrelations(Model $Model, array $correlations)
{
$fields = [
'value_id',
@ -77,29 +89,32 @@ class NoAclCorrelationBehavior extends ModelBehavior
'attribute_id'
];
$this->Correlation->CorrelationValue->replaceValueWithId($correlations, $this->deadlockAvoidance ? 'value_id' : 0);
if ($this->deadlockAvoidance) {
return $this->Correlation->saveMany($correlations, array(
return $this->Correlation->saveMany($correlations, [
'atomic' => false,
'callbacks' => false,
'deep' => false,
'validate' => false,
'fieldList' => $fields
));
'fieldList' => $fields,
]);
} else {
$db = $this->Correlation->getDataSource();
// Split to chunks datasource is is enabled
if (count($correlations) > 100) {
foreach (array_chunk($correlations, 100) as $chunk) {
$db->insertMulti('no_acl_correlations', $fields, $chunk);
$db->insertMulti(self::TABLE_NAME, $fields, $chunk);
}
return true;
} else {
return $db->insertMulti('no_acl_correlations', $fields, $correlations);
return $db->insertMulti(self::TABLE_NAME, $fields, $correlations);
}
}
}
public function runBeforeSaveCorrelation(Model $Model, $attribute) {
public function runBeforeSaveCorrelation(Model $Model, $attribute)
{
// (update-only) clean up the relation of the old value: remove the existing relations related to that attribute, we DO have a reference, the id
// ==> DELETE FROM no_acl_correlations WHERE 1_attribute_id = $a_id OR attribute_id = $a_id; */
// first check if it's an update
@ -172,10 +187,10 @@ class NoAclCorrelationBehavior extends ModelBehavior
public function runGetAttributesRelatedToEvent(Model $Model, $user, $id)
{
$temp_correlations = $this->__collectCorrelations($user, $id, false);
$temp_correlations_1 = $this->__collectCorrelations($user, $id, true);
$correlations = [];
$event_ids = [];
$temp_correlations = $this->__collectCorrelations($user, $id, false);
foreach ($temp_correlations as $temp_correlation) {
$correlations[] = [
'id' => $temp_correlation['Correlation']['event_id'],
@ -185,7 +200,9 @@ class NoAclCorrelationBehavior extends ModelBehavior
];
$event_ids[$temp_correlation['Correlation']['event_id']] = true;
}
foreach ($temp_correlations_1 as $temp_correlation) {
$temp_correlations = $this->__collectCorrelations($user, $id, true);
foreach ($temp_correlations as $temp_correlation) {
$correlations[] = [
'id' => $temp_correlation['Correlation']['1_event_id'],
'attribute_id' => $temp_correlation['Correlation']['1_attribute_id'],
@ -224,6 +241,15 @@ class NoAclCorrelationBehavior extends ModelBehavior
return $relatedAttributes;
}
/**
* @param Correlation $Model
* @param $user
* @param $sgids
* @param array $attribute
* @param array $fields
* @param bool $includeEventData
* @return array
*/
public function runGetRelatedAttributes(Model $Model, $user, $sgids, $attribute, $fields = [], $includeEventData = false)
{
// LATER getRelatedAttributes($attribute) this might become a performance bottleneck
@ -293,7 +319,7 @@ class NoAclCorrelationBehavior extends ModelBehavior
]);
if (!empty($includeEventData)) {
$results = [];
foreach ($relatedAttributes as $k => $attribute) {
foreach ($relatedAttributes as $attribute) {
$temp = $attribute['Attribute'];
$temp['Event'] = $attribute['Event'];
$results[] = $temp;
@ -315,19 +341,24 @@ class NoAclCorrelationBehavior extends ModelBehavior
// ii. Event has a sharing group that the user is accessible to view
// b. Attribute:
// i. Attribute has a distribution of 5 (inheritance of the event, for this the event check has to pass anyway)
// ii. Atttibute has a distribution between 1-3 (community only, connected communities, all orgs)
// ii. Attribute has a distribution between 1-3 (community only, connected communities, all orgs)
// iii. Attribute has a sharing group that the user is accessible to view
$primaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, true);
$secondaryEventIds = $this->__filterRelatedEvents($Model, $user, $eventId, false);
return array_unique(array_merge($primaryEventIds,$secondaryEventIds));
$primaryEventIds = $this->__filterRelatedEvents($Model, $eventId, true);
$secondaryEventIds = $this->__filterRelatedEvents($Model, $eventId, false);
return array_unique(array_merge($primaryEventIds, $secondaryEventIds), SORT_REGULAR);
}
private function __filterRelatedEvents(Model $Model, array $user, int $eventId, bool $primary)
/**
* @param Correlation $Model
* @param int $eventId
* @param bool $primary
* @return array
*/
private function __filterRelatedEvents(Model $Model, int $eventId, bool $primary)
{
$current = $primary ? '' : '1_';
$prefix = $primary ? '1_' : '';
$correlations = $Model->find('all', [
return $Model->find('column', [
'recursive' => -1,
'fields' => [
$prefix . 'event_id'
@ -337,8 +368,6 @@ class NoAclCorrelationBehavior extends ModelBehavior
],
'unique' => true,
]);
$eventIds = Hash::extract($correlations, '{n}.Correlation.' . $prefix . 'event_id');
return $eventIds;
}
public function updateContainedCorrelations(
@ -368,6 +397,11 @@ class NoAclCorrelationBehavior extends ModelBehavior
}
}
/**
* @param Correlation $Model
* @param string $value
* @return void
*/
public function purgeByValue(Model $Model, string $value)
{
$valueIds = $Model->CorrelationValue->find('column', [

56
app/Model/Correlation.php
View File

@ -6,7 +6,8 @@ App::uses('AppModel', 'Model');
* @property Event $Event
* @property CorrelationValue $CorrelationValue
* @method saveCorrelations(array $correlations)
* @method runBeforeSaveCorrelation
* @method createCorrelationEntry(string $value, array $a, array $b)
* @method runBeforeSaveCorrelation(array $attribute)
* @method fetchRelatedEventIds(array $user, int $eventId, array $sgids)
* @method getFieldRules
* @method getContainRules($filter = null)
@ -56,9 +57,6 @@ class Correlation extends AppModel
/** @var array */
private $cidrListCache;
/** @var string */
private $__correlationEngine;
private $__tempContainCache = [];
/** @var OverCorrelatingValue */
@ -67,10 +65,10 @@ class Correlation extends AppModel
public function __construct($id = false, $table = null, $ds = null)
{
parent::__construct($id, $table, $ds);
$this->__correlationEngine = $this->getCorrelationModelName();
$deadlockAvoidance = Configure::check('MISP.deadlock_avoidance') ? Configure::read('MISP.deadlock_avoidance') : false;
$correlationEngine = $this->getCorrelationModelName();
$deadlockAvoidance = Configure::read('MISP.deadlock_avoidance') ?: false;
// load the currently used correlation engine
$this->Behaviors->load($this->__correlationEngine . 'Correlation', ['deadlockAvoidance' => $deadlockAvoidance]);
$this->Behaviors->load($correlationEngine . 'Correlation', ['deadlockAvoidance' => $deadlockAvoidance]);
// getTableName() needs to be implemented by the engine - this points us to the table to be used
$this->useTable = $this->getTableName();
$this->advancedCorrelationEnabled = (bool)Configure::read('MISP.enable_advanced_correlations');
@ -183,17 +181,6 @@ class Correlation extends AppModel
return $correlatingAttributes;
}
/**
* @param string $value
* @param array $a Attribute A
* @param array $b Attribute B
* @return array
*/
private function __createCorrelationEntry($value, $a, $b)
{
return $this->createCorrelationEntry($value, $a, $b);
}
public function correlateValue($value, $jobId = false)
{
$correlatingAttributes = $this->__getMatchingAttributes($value);
@ -216,7 +203,7 @@ class Correlation extends AppModel
if ($correlatingAttribute['Attribute']['event_id'] === $correlatingAttribute2['Attribute']['event_id']) {
continue;
}
$correlations[] = $this->__createCorrelationEntry($value, $correlatingAttribute, $correlatingAttribute2);
$correlations[] = $this->createCorrelationEntry($value, $correlatingAttribute, $correlatingAttribute2);
}
$extraCorrelations = $this->__addAdvancedCorrelations($correlatingAttribute);
if (!empty($extraCorrelations)) {
@ -224,8 +211,8 @@ class Correlation extends AppModel
if ($correlatingAttribute['Attribute']['event_id'] === $extraCorrelation['Attribute']['event_id']) {
continue;
}
$correlations[] = $this->__createCorrelationEntry($value, $correlatingAttribute, $extraCorrelation);
//$correlations = $this->__createCorrelationEntry($value, $extraCorrelation, $correlatingAttribute, $correlations);
$correlations[] = $this->createCorrelationEntry($value, $correlatingAttribute, $extraCorrelation);
//$correlations = $this->createCorrelationEntry($value, $extraCorrelation, $correlatingAttribute, $correlations);
}
}
if ($jobId && $k % 100 === 0) {
@ -252,17 +239,16 @@ class Correlation extends AppModel
}
}
public function correlateAttribute(array $attribute)
{
$this->runBeforeSaveCorrelation($attribute);
$this->afterSaveCorrelation($attribute);
}
public function beforeSaveCorrelation(array $attribute)
{
$this->runBeforeSaveCorrelation($attribute);
}
/**
* @param string $scope
* @param int $id
* @return false|array
*/
private function __cachedGetContainData($scope, $id)
{
if (!empty($this->getContainRules($scope))) {
@ -374,13 +360,13 @@ class Correlation extends AppModel
// If we have more correlations for the value than the limit, set the block entry and stop the correlation process
$this->OverCorrelatingValue->block($cV);
return true;
} else {
} else if ($count !== 0) {
// If we have fewer hits than the limit, proceed with the correlation, but first make sure we remove any existing blockers
$this->OverCorrelatingValue->unblock($cV);
}
foreach ($correlatingAttributes as $b) {
// On a full correlation, only correlate with attributes that have a higher ID to avoid duplicate correlations
if ($full && $b['Attribute']['id'] < $b['Attribute']['id']) {
if ($full && $a['Attribute']['id'] < $b['Attribute']['id']) {
continue;
}
if (isset($b['Attribute']['value1'])) {
@ -390,9 +376,9 @@ class Correlation extends AppModel
$value = $cV;
}
if ($a['Attribute']['id'] > $b['Attribute']['id']) {
$correlations[] = $this->__createCorrelationEntry($value, $a, $b);
$correlations[] = $this->createCorrelationEntry($value, $a, $b);
} else {
$correlations[] = $this->__createCorrelationEntry($value, $b, $a);
$correlations[] = $this->createCorrelationEntry($value, $b, $a);
}
}
}
@ -996,4 +982,12 @@ class Correlation extends AppModel
}
return $result === true;
}
/**
* @return string
*/
private function getCorrelationModelName()
{
return Configure::read('MISP.correlation_engine') ?: 'Default';
}
}

56
app/Model/CorrelationValue.php
View File

@ -5,6 +5,62 @@ class CorrelationValue extends AppModel
{
public $recursive = -1;
/**
* @param array $correlations
* @param string|int $valueIndex
* @return void
*/
public function replaceValueWithId(array &$correlations, $valueIndex)
{
$values = array_column($correlations, $valueIndex);
$valueIds = $this->getIds($values);
foreach ($correlations as &$correlation) {
$value = mb_substr($correlation[$valueIndex], 0, 191);
$correlation[$valueIndex] = $valueIds[$value];
}
}
/**
* @param array $values
* @return array Value in key, value ID in value
*/
private function getIds(array $values)
{
foreach ($values as &$value) {
$value = mb_substr($value, 0, 191);
}
$values = array_unique($values, SORT_REGULAR); // Remove duplicate values
$existingValues = $this->find('list', [
'recursive' => -1,
'callbacks' => false,
'fields' => ['value', 'id'],
'conditions' => [
'value' => $values,
],
]);
$notExistValues = array_diff($values, array_keys($existingValues));
if (!empty($notExistValues)) {
$this->getDataSource()->begin();
foreach ($notExistValues as $notExistValue) {
$this->create();
try {
$this->save(['value' => $notExistValue], [
'callbacks' => false,
'validate' => false,
]);
$existingValues[$notExistValue] = $this->id;
} catch (Exception $e) {
$existingValues[$notExistValue] = $this->getValueId($notExistValue);
}
}
$this->getDataSource()->commit();
}
return $existingValues;
}
/**
* @param string $value
* @return int

11
app/Model/OverCorrelatingValue.php
View File

@ -5,10 +5,6 @@ class OverCorrelatingValue extends AppModel
{
public $recursive = -1;
public $actsAs = array(
'Containable'
);
public function beforeValidate($options = array())
{
$this->data['OverCorrelatingValue']['value'] = self::truncate($this->data['OverCorrelatingValue']['value']);
@ -70,7 +66,7 @@ class OverCorrelatingValue extends AppModel
*/
public function getLimit()
{
return Configure::check('MISP.correlation_limit') ? Configure::read('MISP.correlation_limit') : 20;
return Configure::read('MISP.correlation_limit') ?: 20;
}
public function getOverCorrelations($query)
@ -94,12 +90,11 @@ class OverCorrelatingValue extends AppModel
public function findOverCorrelatingValues(array $values_to_check): array
{
$values_to_check_truncated = array_unique(self::truncateValues($values_to_check));
$overCorrelatingValues = $this->find('column', [
$values_to_check_truncated = array_unique(self::truncateValues($values_to_check), SORT_REGULAR);
return $this->find('column', [
'conditions' => ['value' => $values_to_check_truncated],
'fields' => ['value'],
]);
return $overCorrelatingValues;
}
public function generateOccurrencesRouter()

4
tests/testlive_comprehensive_local.py
View File

@ -536,6 +536,10 @@ class TestComprehensive(unittest.TestCase):
for event in (first, second):
check_response(self.admin_misp_connector.delete_event(event))
def test_correlations_noacl(self):
with MISPSetting(self.admin_misp_connector, {"MISP.correlation_engine": "NoAcl"}):
self.test_correlations()
def test_advanced_correlations(self):
with MISPSetting(self.admin_misp_connector, {"MISP.enable_advanced_correlations": True}):
first = create_simple_event()